From 20144ae130652073d71c8ff1562621a03a892df2 Mon Sep 17 00:00:00 2001
From: Tyler Hallada <tyler@hallada.net>
Date: Thu, 3 Mar 2022 01:33:36 -0500
Subject: [PATCH] Process plugins round-robin in a web worker queue

Speeds it up a lot!
---
 components/DataDirPicker.tsx | 14 ++++++-------
 pages/index.tsx              | 38 +++++++++++++++++++++---------------
 2 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/components/DataDirPicker.tsx b/components/DataDirPicker.tsx
index 0b9a903..936236e 100644
--- a/components/DataDirPicker.tsx
+++ b/components/DataDirPicker.tsx
@@ -1,6 +1,6 @@
 import React, { useContext, useEffect, useRef } from "react";
 
-import { WorkerContext } from "../pages/index";
+import { WorkersContext } from "../pages/index";
 import { useAppSelector, useAppDispatch } from "../lib/hooks";
 import {
   addPluginInOrder,
@@ -22,12 +22,12 @@ export const excludedPlugins = [
 type Props = {};
 
 const DataDirPicker: React.FC<Props> = () => {
-  const worker = useContext(WorkerContext);
+  const workers = useContext(WorkersContext);
   const dispatch = useAppDispatch();
   const plugins = useAppSelector((state) => state.plugins.plugins);
 
   const onDataDirButtonClick = async () => {
-    if (!worker) {
+    if (workers.length === 0) {
       return alert("Worker not loaded yet");
     }
     const dirHandle = await (
@@ -53,13 +53,13 @@ const DataDirPicker: React.FC<Props> = () => {
     }
     dispatch(setPending(plugins.length));
 
-    for (const plugin of plugins) {
+    plugins.forEach(async (plugin, index) => {
       const file = await plugin.getFile();
       console.log(file.lastModified);
       console.log(file.lastModifiedDate);
       const contents = new Uint8Array(await file.arrayBuffer());
       try {
-        worker.postMessage(
+        workers[index % workers.length].postMessage(
           {
             skipParsing: excludedPlugins.includes(plugin.name),
             filename: plugin.name,
@@ -71,7 +71,7 @@ const DataDirPicker: React.FC<Props> = () => {
       } catch (error) {
         console.error(error);
       }
-    }
+    });
   };
 
   return (
@@ -80,7 +80,7 @@ const DataDirPicker: React.FC<Props> = () => {
         To see all of the cell edits and conflicts for your current mod load
         order select your <code>Data</code> directory below to load the plugins.
       </p>
-      <button onClick={onDataDirButtonClick} disabled={!worker}>
+      <button onClick={onDataDirButtonClick} disabled={workers.length === 0}>
         {plugins.length === 0 ? "Open" : "Reload"} Skyrim Data directory
       </button>
     </>
diff --git a/pages/index.tsx b/pages/index.tsx
index 535a309..ee884ce 100644
--- a/pages/index.tsx
+++ b/pages/index.tsx
@@ -12,30 +12,36 @@ import {
 } from "../slices/plugins";
 import { setPluginsTxtAndApplyLoadOrder } from "../slices/pluginsTxt";
 
-export const WorkerContext = createContext<Worker | null>(null);
+export const WorkersContext = createContext<Worker[]>([]);
 
 const Home: NextPage = () => {
-  const [worker, setWorker] = useState<Worker | null>(null);
+  const [workers, setWorkers] = useState<Worker[]>([]);
   const dispatch = useAppDispatch();
 
   useEffect(() => {
-    async function loadWorker() {
+    async function loadWorkers(count: number) {
       const { default: Worker } = await import(
         "worker-loader?filename=static/[fullhash].worker.js!../workers/PluginsLoader.worker"
       );
-      const newWorker = new Worker();
-      newWorker.onmessage = (evt: { data: PluginFile }) => {
-        const { data } = evt;
-        dispatch(decrementPending(1));
-        console.log(data.parsed);
-        dispatch(addPluginInOrder(data));
-      };
-      setWorker(newWorker);
+      const newWorkers = [];
+      for (let i = 0; i < count; i++) {
+        const worker = new Worker();
+        worker.onmessage = (evt: { data: PluginFile }) => {
+          const { data } = evt;
+          dispatch(decrementPending(1));
+          console.log(data.parsed);
+          dispatch(addPluginInOrder(data));
+        };
+        newWorkers.push(worker);
+      }
+      setWorkers(newWorkers);
     }
-    loadWorker();
+    loadWorkers(window.navigator.hardwareConcurrency ?? 8);
     return () => {
-      if (worker) {
-        worker.terminate();
+      if (workers) {
+        for (const worker of workers) {
+          worker.terminate();
+        }
       }
     };
     // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -89,7 +95,7 @@ const Home: NextPage = () => {
         <meta name="twitter:site" content="@tyhallada" />
         <meta name="twitter:creator" content="@tyhallada" />
       </Head>
-      <WorkerContext.Provider value={worker}>
+      <WorkersContext.Provider value={workers}>
         <div
           style={{
             margin: 0,
@@ -113,7 +119,7 @@ const Home: NextPage = () => {
         >
           <Map />
         </div>
-      </WorkerContext.Provider>
+      </WorkersContext.Provider>
     </>
   );
 };