From 6dc6c60ec6edce03cf32a9d494818fea09a575d8 Mon Sep 17 00:00:00 2001
From: Kenneth Reitz <me@kennethreitz.org>
Date: Sun, 30 Nov 2025 01:50:14 -0500
Subject: [PATCH] Use concurrent worker pool for faster offline caching
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replace sequential batch downloads with a concurrent pool pattern
using 50 simultaneous workers. This dramatically speeds up the
caching of ~48,000 pages for offline use.

- Each worker pulls URLs from a shared queue
- Progress notifications sent every 100 pages
- Workers run in parallel using Promise.all()

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 kjvstudy_org/static/sw.js | 67 +++++++++++++++++++++++----------------
 1 file changed, 39 insertions(+), 28 deletions(-)

diff --git a/kjvstudy_org/static/sw.js b/kjvstudy_org/static/sw.js
index 76d3b88..da31beb 100644
--- a/kjvstudy_org/static/sw.js
+++ b/kjvstudy_org/static/sw.js
@@ -168,40 +168,51 @@ async function startBackgroundCaching() {
     status: `Downloading ${totalToCache.toLocaleString()} pages...`
   });
 
-  // Cache pages in batches
-  const BATCH_SIZE = 20;
-  const BATCH_DELAY = 200; // 0.2 seconds between batches
+  // Concurrent pool - keep N requests in flight at all times
+  const CONCURRENCY = 50; // Number of concurrent requests
+  const PROGRESS_INTERVAL = 100; // Notify every N completions
 
-  for (let i = 0; i < uncachedPages.length; i += BATCH_SIZE) {
-    const batch = uncachedPages.slice(i, i + BATCH_SIZE);
+  let nextIndex = 0;
+  let lastNotified = 0;
 
-    await Promise.all(
-      batch.map(async (url) => {
-        try {
-          const response = await fetch(url);
-          if (response.ok) {
-            await cache.put(url, response);
-            cachedCount++;
-          }
-        } catch (err) {
-          // Silent fail for individual pages
+  async function cacheUrl(url) {
+    try {
+      const response = await fetch(url);
+      if (response.ok) {
+        await cache.put(url, response);
+        cachedCount++;
+
+        // Notify progress periodically
+        if (cachedCount - lastNotified >= PROGRESS_INTERVAL) {
+          lastNotified = cachedCount;
+          notifyClients({
+            type: 'CACHE_PROGRESS',
+            cached: cachedCount,
+            total: totalToCache
+          });
         }
-      })
-    );
-
-    // Notify progress every batch
-    notifyClients({
-      type: 'CACHE_PROGRESS',
-      cached: cachedCount,
-      total: totalToCache
-    });
-
-    // Small delay between batches
-    if (i + BATCH_SIZE < uncachedPages.length) {
-      await new Promise(resolve => setTimeout(resolve, BATCH_DELAY));
+      }
+    } catch (err) {
+      // Silent fail for individual pages
     }
   }
 
+  async function worker() {
+    while (nextIndex < uncachedPages.length) {
+      const url = uncachedPages[nextIndex++];
+      await cacheUrl(url);
+    }
+  }
+
+  // Start concurrent workers
+  const workers = [];
+  for (let i = 0; i < Math.min(CONCURRENCY, uncachedPages.length); i++) {
+    workers.push(worker());
+  }
+
+  // Wait for all workers to complete
+  await Promise.all(workers);
+
   console.log('[SW] Background caching complete!', cachedCount, 'pages cached');
   cachingInProgress = false;