Batch cache reads and stream search results progressively (AI)

cache.js: replaces the per-directory getCacheEntry() point lookup
with getAllCacheEntries(), a single bulk read via IDBObjectStore
.getAll(). discovery.js's runCrawl() kicks this off in parallel with
the author/post-directory requests and passes the resulting Map into
resolveStub() as a synchronous in-memory lookup, cutting hundreds of
individual IndexedDB round-trips down to one.

hydrateArticles() now takes an optional onProgress callback;
loadSearch() uses it to render matches as hydration resolves instead
of blocking on the whole result set.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AXSVPS3UmmASiiprPfcnUd

README.md

@@ -366,8 +366,11 @@ articles it has — only ~20 articles' worth of content is ever fetched per page
   listing fetch too, whether or not the article has ever actually been hydrated. Hydration
   re-caches the same entry with the fetched content added, so once *any* visit renders an
   article, every later visit skips its content fetch as well, until its directory's `mtime`
-  changes. If IndexedDB is unavailable (private browsing, disabled storage), discovery
-  transparently falls back to fetching everything live on every visit.
+  changes. Indexing reads the entire cache once up front (`getAllCacheEntries()`, kicked off in
+  parallel with the author/post-directory requests) rather than a separate IndexedDB lookup per
+  directory, so a warm cache doesn't cost hundreds of individual reads. If IndexedDB is
+  unavailable (private browsing, disabled storage), discovery transparently falls back to
+  fetching everything live on every visit.
 - **Cache schema version** — the cached records store the full derived article shape, not raw
   content, so a code change to how articles are derived (in `cmjs/js/discovery.js`) can make old
   cached entries stale in *structure*, not just content — and a directory's `mtime` won't catch
@@ -387,6 +390,10 @@ articles it has — only ~20 articles' worth of content is ever fetched per page
 - **Prefetching** — once the current page has fully rendered, the next page's articles are
   hydrated in the background (fire-and-forget), so clicking "next" is usually an instant cache
   hit rather than a fresh round-trip.
+- **Progressive search** — search hydrates every indexed article (see above — it needs real
+  text, unlike the list view), but does so incrementally rather than in one blocking batch:
+  matches appear and the result list grows as hydration resolves, with a "still indexing…" note
+  until every article has been checked.
 - **Crawl health** — if every author's directory listing fails outright, or very few of the
   known post directories loaded, a small warning banner is shown above the article list instead
   of silently rendering an empty page (see `getCrawlStats()` / `renderCrawlHealthWarning()` in

cmjs/js/cache.js

@@ -56,10 +56,14 @@ export function isCacheAvailable() {
     return cacheAvailable;
 }
 
-// Returns the cached { path, mtime, article } record, or undefined on a miss
-// (or if the cache is unavailable).
-export async function getCacheEntry(path) {
-    return runRequest('readonly', store => store.get(path));
+// Returns every cached { path, mtime, article } record as a Map keyed by
+// path — one bulk read instead of a separate IndexedDB round-trip per
+// directory, which matters once there are hundreds of them: discovery.js
+// reads this once up front rather than doing a point lookup per directory
+// during indexing. Empty Map on a miss or if the cache is unavailable.
+export async function getAllCacheEntries() {
+    const records = await runRequest('readonly', store => store.getAll());
+    return new Map((records || []).map(record => [record.path, record]));
 }
 
 export async function setCacheEntry(path, entry) {

cmjs/js/discovery.js

@@ -1,7 +1,7 @@
 import { parseFrontmatter } from './frontmatter.js';
 import { BASE_URL } from './config.js';
 import { mapWithConcurrency } from './concurrency.js';
-import { getCacheEntry, setCacheEntry, pruneCache } from './cache.js';
+import { getAllCacheEntries, setCacheEntry, pruneCache } from './cache.js';
 import { t } from './i18n.js';
 
 // Caps simultaneous per-article requests — both the cheap per-directory
@@ -210,8 +210,11 @@ async function buildStub(dir) {
 // listing fetch entirely; otherwise builds a fresh stub from the listing
 // and caches that too, so a repeat visit skips the listing fetch even for
 // an article nothing has hydrated yet — not just already-viewed ones.
-async function resolveStub(dir) {
-    const cached = await getCacheEntry(dir.root);
+// `cacheSnapshot` is the whole cache read once up front (see runCrawl) —
+// an in-memory lookup here instead of a separate IndexedDB read per
+// directory, which matters once there are hundreds of them.
+async function resolveStub(dir, cacheSnapshot) {
+    const cached = cacheSnapshot.get(dir.root);
     if (cached && dir.mtime != null && cached.mtime === dir.mtime) return cached.article;
     const stub = await buildStub(dir);
     if (stub) await setCacheEntry(dir.root, { mtime: dir.mtime, article: stub });
@@ -261,9 +264,19 @@ export async function hydrateArticle(stub) {
     return stub;
 }
 
-// Concurrency-limited batch hydration for a page's worth of stubs.
-export function hydrateArticles(stubs) {
-    return mapWithConcurrency(stubs, MAX_CONCURRENT_REQUESTS, hydrateArticle);
+// Concurrency-limited batch hydration. For a page's worth of stubs this
+// just resolves once; `onProgress`, if given, is called after each item
+// resolves with (resolvedCount, total) — lets a caller hydrating a much
+// bigger batch (search hydrates every result, not just one page) show
+// results incrementally instead of one long wait for all of them.
+export function hydrateArticles(stubs, onProgress) {
+    let resolved = 0;
+    return mapWithConcurrency(stubs, MAX_CONCURRENT_REQUESTS, async stub => {
+        const article = await hydrateArticle(stub);
+        resolved += 1;
+        onProgress?.(resolved, stubs.length);
+        return article;
+    });
 }
 
 // Best-effort recency signal used only to decide crawl *order* (see below) —
@@ -279,6 +292,12 @@ function mtimeSortValue(dir) {
 async function runCrawl() {
     partialResolved = [];
 
+    // Independent of waves 1/2 below — kicked off in parallel with them and
+    // only awaited once wave 3 actually needs it, so the cache read's
+    // latency is hidden behind the (already-required) directory fetches
+    // rather than adding to the total.
+    const cacheSnapshotPromise = getAllCacheEntries();
+
     // Wave 1 (root) — always fetched live: the author directories.
     const authorEntries = await fetchJson(`${BASE_URL}articles/`);
     const authors = authorEntries.filter(entry => entry.type === 'directory' && !entry.name.startsWith('.'));
@@ -318,6 +337,7 @@ async function runCrawl() {
     // large tree is still indexing in the background (see
     // onArticlesProgress). Purely a priority order, not the display order.
     const crawlOrder = [...slugDirs].sort((a, b) => mtimeSortValue(b) - mtimeSortValue(a));
+    const cacheSnapshot = await cacheSnapshotPromise;
 
     // Wave 3 — per slug directory, cache-aware and concurrency-limited: a
     // directory whose mtime matches its cached entry (stub or hydrated)
@@ -327,7 +347,7 @@ async function runCrawl() {
     let loaded = 0;
     let resolvedCount = 0;
     await mapWithConcurrency(crawlOrder, MAX_CONCURRENT_REQUESTS, async dir => {
-        const stub = await resolveStub(dir);
+        const stub = await resolveStub(dir, cacheSnapshot);
         if (stub) {
             loaded += 1;
             partialResolved.push(stub);

cmjs/js/views.js

@@ -349,7 +349,7 @@ export const loadTagView = (tag, page = 1) => loadArticleList({ type: 'tag', val
 export async function loadSearch(query) {
     showListView();
     window.scrollTo(0, 0);
-    currentGeneration += 1; // invalidates any still-running loadArticleList progress listener
+    const generation = ++currentGeneration;
     const view = document.getElementById('list-view');
     view.innerHTML = `<p class="loading">${t('searching')}</p>`;
     const input = document.getElementById('search-input');
@@ -357,19 +357,43 @@ export async function loadSearch(query) {
     try {
         await ensureTags();
         const needle = query.toLocaleLowerCase(getLocale());
+        const matchesQuery = article => [article.title, article.description, article.author, article.authorName, article.category, ...article.tags.map(tag => tagLabels.get(tag) || tag)]
+            .some(value => String(value || '').toLocaleLowerCase(getLocale()).includes(needle));
+
         const stubs = await discoverArticles();
+        if (generation !== currentGeneration) return;
         renderFilterNavigation(stubs);
+
         // Unlike the list view, search needs every article's real title and
         // description — text that only exists in its markdown content — so
         // it hydrates the whole set rather than just a page's worth. Most
         // of that is typically already cached from earlier browsing/search.
-        const allArticles = await hydrateArticles(stubs);
-        const articles = allArticles.filter(article => [article.title, article.description, article.author, article.authorName, article.category, ...article.tags.map(tag => tagLabels.get(tag) || tag)]
-            .some(value => String(value || '').toLocaleLowerCase(getLocale()).includes(needle)));
-        view.innerHTML = `<h2 class="category-heading">${t('searchHeading', { query })}</h2>`;
-        if (!articles.length) view.insertAdjacentHTML('beforeend', `<p class="empty">${t('searchNoResults')}</p>`);
-        else articles.forEach(article => view.appendChild(renderCard(article)));
-    } catch (error) { view.innerHTML = `<p class="error">${t('searchError', { message: error.message })}</p>`; }
+        // Hydrating is progressive (like indexing) so results appear as
+        // they're found instead of one long wait for the whole site's
+        // content — hydrateArticle() mutates stubs in place, so checking
+        // each stub's own `hydrated` flag is enough to know what's ready,
+        // without tracking a separately-growing list.
+        const renderResults = () => {
+            if (generation !== currentGeneration) return;
+            const hydratedSoFar = stubs.filter(stub => stub.hydrated);
+            const done = hydratedSoFar.length === stubs.length;
+            const matches = hydratedSoFar.filter(matchesQuery);
+            view.innerHTML = `<h2 class="category-heading">${t('searchHeading', { query })}</h2>`;
+            if (matches.length) {
+                matches.forEach(article => view.appendChild(renderCard(article)));
+                if (!done) view.insertAdjacentHTML('beforeend', `<p class="loading">${t('indexingBackground')}</p>`);
+            } else {
+                view.insertAdjacentHTML('beforeend', done ? `<p class="empty">${t('searchNoResults')}</p>` : `<p class="loading">${t('searching')}</p>`);
+            }
+        };
+
+        await hydrateArticles(stubs, resolvedCount => {
+            if (resolvedCount === 1 || resolvedCount % 5 === 0 || resolvedCount === stubs.length) renderResults();
+        });
+        renderResults(); // guaranteed-final render — also covers the empty-site case, where the loop above never calls back at all
+    } catch (error) {
+        if (generation === currentGeneration) view.innerHTML = `<p class="error">${t('searchError', { message: error.message })}</p>`;
+    }
 }
 
 function buildTOC(content) {