Use concurrent worker pool for faster offline caching

Replace sequential batch downloads with a concurrent pool pattern
using 50 simultaneous workers. This dramatically speeds up the
caching of ~48,000 pages for offline use.

- Each worker pulls URLs from a shared queue
- Progress notifications sent every 100 pages
- Workers run in parallel using Promise.all()

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-11-30 01:50:14 -05:00
parent 8f7a6097d7
commit 6dc6c60ec6
+39 -28
View File
@@ -168,40 +168,51 @@ async function startBackgroundCaching() {
status: `Downloading ${totalToCache.toLocaleString()} pages...`
});
// Cache pages in batches
const BATCH_SIZE = 20;
const BATCH_DELAY = 200; // 0.2 seconds between batches
// Concurrent pool - keep N requests in flight at all times
const CONCURRENCY = 50; // Number of concurrent requests
const PROGRESS_INTERVAL = 100; // Notify every N completions
for (let i = 0; i < uncachedPages.length; i += BATCH_SIZE) {
const batch = uncachedPages.slice(i, i + BATCH_SIZE);
let nextIndex = 0;
let lastNotified = 0;
await Promise.all(
batch.map(async (url) => {
try {
const response = await fetch(url);
if (response.ok) {
await cache.put(url, response);
cachedCount++;
}
} catch (err) {
// Silent fail for individual pages
async function cacheUrl(url) {
try {
const response = await fetch(url);
if (response.ok) {
await cache.put(url, response);
cachedCount++;
// Notify progress periodically
if (cachedCount - lastNotified >= PROGRESS_INTERVAL) {
lastNotified = cachedCount;
notifyClients({
type: 'CACHE_PROGRESS',
cached: cachedCount,
total: totalToCache
});
}
})
);
// Notify progress every batch
notifyClients({
type: 'CACHE_PROGRESS',
cached: cachedCount,
total: totalToCache
});
// Small delay between batches
if (i + BATCH_SIZE < uncachedPages.length) {
await new Promise(resolve => setTimeout(resolve, BATCH_DELAY));
}
} catch (err) {
// Silent fail for individual pages
}
}
async function worker() {
while (nextIndex < uncachedPages.length) {
const url = uncachedPages[nextIndex++];
await cacheUrl(url);
}
}
// Start concurrent workers
const workers = [];
for (let i = 0; i < Math.min(CONCURRENCY, uncachedPages.length); i++) {
workers.push(worker());
}
// Wait for all workers to complete
await Promise.all(workers);
console.log('[SW] Background caching complete!', cachedCount, 'pages cached');
cachingInProgress = false;