Stream the article list as it's indexed instead of blocking on the whole tree (AI)

discoverArticles() now dedupes concurrent callers into one shared
crawl and exposes onArticlesProgress() for incremental results;
loadArticleList() (and everything built on it: pagination, author/
category/tag filters, the sidebar counts) renders as soon as the
first articles resolve and keeps updating while the rest crawl in
the background, with a "still indexing…" note until done. The crawl
also visits the most-recently-touched directories first (mtime,
best-effort) so early results tend to already be close to correct;
final display order is unchanged (always the authoritative `date`).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AXSVPS3UmmASiiprPfcnUd

README.md

@@ -353,6 +353,14 @@ read its `.tag`/`.date` markers) and its Markdown content.
 - **Concurrency** — `MAX_CONCURRENT_REQUESTS` in `cmjs/js/discovery.js` (default `10`) caps how
   many post directories are crawled at once; raise it for a fast/local nginx, lower it for a
   slow or shared/production instance.
+- **Progressive rendering** — on a large site, the article list doesn't wait for the entire
+  tree to finish crawling before showing anything: `discoverArticles()` streams incremental
+  results via `onArticlesProgress()`, so the list (and pagination, and the sidebar tag/year
+  counts) render as soon as the first articles are indexed and keep filling in while the rest
+  crawl in the background — with a small "still indexing…" note shown until it's done. The
+  crawl also visits directories with the most recent `mtime` first (a best-effort heuristic, not
+  the final display order — that's always the authoritative `.date` marker), so the first
+  results tend to already be close to correct.
 - **Crawl health** — if every author's directory listing fails outright, or very few of the
   known post directories loaded, a small warning banner is shown above the article list instead
   of silently rendering an empty page (see `getCrawlStats()` / `renderCrawlHealthWarning()` in

cmjs/js/discovery.js

@@ -94,8 +94,41 @@ export async function fetchTagRegistry() {
 }
 
 let articleCache = null;
+let crawlPromise = null;
 let crawlStats = { total: 0, loaded: 0, authorsTotal: 0, authorsFailed: 0 };
 
+// Raw (unfiltered/unsorted) article records resolved so far by the crawl
+// currently or most recently in flight — lets a late subscriber (a view
+// that mounts mid-crawl) immediately see what's already available instead
+// of waiting for the next progress tick.
+let partialResolved = [];
+const progressListeners = new Set();
+
+function sortAndFilterPublished(rawArticles) {
+    return rawArticles
+        .filter(article => article?.published && article.date)
+        .sort((a, b) => String(b.date).localeCompare(String(a.date)) || a.hash.localeCompare(b.hash));
+}
+
+// Subscribes to incremental crawl progress so a large site's article list
+// can render as soon as the first articles are indexed instead of blocking
+// on the whole tree. `listener(articles, done)` is called immediately with
+// whatever has resolved so far, again as more of the crawl resolves, and a
+// final time with `done: true`. Returns an unsubscribe function — always
+// call it once the caller stops rendering (e.g. the user navigated away),
+// otherwise a background crawl keeps calling a listener for a view that's
+// no longer showing.
+export function onArticlesProgress(listener) {
+    progressListeners.add(listener);
+    if (partialResolved.length) listener(sortAndFilterPublished(partialResolved), false);
+    return () => progressListeners.delete(listener);
+}
+
+function notifyProgress(done) {
+    const snapshot = sortAndFilterPublished(partialResolved);
+    progressListeners.forEach(listener => listener(snapshot, done));
+}
+
 // How many of the last crawl's slug directories actually produced an article
 // vs. how many were candidates, plus how many authors' listings failed
 // outright — lets the UI show a "something's badly wrong" indicator instead
@@ -177,8 +210,18 @@ async function buildArticle(dir) {
     }
 }
 
-export async function discoverArticles() {
-    if (articleCache) return articleCache;
+// Best-effort recency signal used only to decide crawl *order* (see below) —
+// never for display order, which always uses the authoritative `date`
+// marker once the crawl finishes. `Date.parse` handles both the RFC-1123
+// string nginx's JSON autoindex emits and the ISO string the HTML-fallback
+// parser produces; an unparseable/missing mtime sorts last.
+function mtimeSortValue(dir) {
+    const parsed = dir.mtime ? Date.parse(dir.mtime) : NaN;
+    return Number.isNaN(parsed) ? -Infinity : parsed;
+}
+
+async function runCrawl() {
+    partialResolved = [];
 
     // Wave 1 (root) — always fetched live: the author directories.
     const authorEntries = await fetchJson(`${BASE_URL}articles/`);
@@ -214,31 +257,53 @@ export async function discoverArticles() {
     }));
     const slugDirs = slugDirLists.flat();
 
+    // Crawl the likeliest-recent directories first so the first page of
+    // results tends to already be close to correct while the rest of a
+    // large tree is still indexing in the background (see
+    // onArticlesProgress). Purely a priority order, not the display order.
+    const crawlOrder = [...slugDirs].sort((a, b) => mtimeSortValue(b) - mtimeSortValue(a));
+
     // Wave 3+4 — per slug directory, cache-aware and concurrency-limited: a
     // directory whose mtime matches its cached entry skips both the listing
     // fetch (for .tag/.date files) and the markdown content fetch entirely.
     let loaded = 0;
-    const articles = await mapWithConcurrency(slugDirs, MAX_CONCURRENT_REQUESTS, async dir => {
+    let resolvedCount = 0;
+    await mapWithConcurrency(crawlOrder, MAX_CONCURRENT_REQUESTS, async dir => {
         const cached = await getCacheEntry(dir.root);
-        if (cached && dir.mtime != null && cached.mtime === dir.mtime) {
-            if (cached.article) loaded += 1;
-            return cached.article;
-        }
-        const article = await buildArticle(dir);
+        const cacheHit = cached && dir.mtime != null && cached.mtime === dir.mtime;
+        const article = cacheHit ? cached.article : await buildArticle(dir);
+        if (!cacheHit && article) await setCacheEntry(dir.root, { mtime: dir.mtime, article });
         if (article) {
             loaded += 1;
-            await setCacheEntry(dir.root, { mtime: dir.mtime, article });
+            partialResolved.push(article);
         }
+        // Throttled rather than on every single item — with a large tree
+        // that would mean hundreds of list re-renders for little benefit.
+        resolvedCount += 1;
+        if (resolvedCount === 1 || resolvedCount % 5 === 0 || resolvedCount === crawlOrder.length) notifyProgress(false);
         return article;
     });
 
     await pruneCache(slugDirs.map(dir => dir.root));
     crawlStats = { total: slugDirs.length, loaded, authorsTotal: authors.length, authorsFailed };
 
-    articleCache = articles
-        .filter(article => article?.published && article.date)
-        .sort((a, b) => String(b.date).localeCompare(String(a.date)) || a.hash.localeCompare(b.hash));
-    return articleCache;
+    const finalArticles = sortAndFilterPublished(partialResolved);
+    notifyProgress(true);
+    return finalArticles;
+}
+
+// Cached after the first successful crawl; concurrent callers during the
+// crawl share the same in-flight promise rather than each starting their
+// own duplicate crawl. See onArticlesProgress() for incremental results.
+export function discoverArticles() {
+    if (articleCache) return Promise.resolve(articleCache);
+    if (!crawlPromise) {
+        crawlPromise = runCrawl().then(
+            result => { articleCache = result; crawlPromise = null; return result; },
+            error => { crawlPromise = null; throw error; },
+        );
+    }
+    return crawlPromise;
 }
 
 export async function findArticle(hash) {

cmjs/js/views.js

@@ -1,11 +1,17 @@
 import { templates } from './templates.js';
-import { discoverArticles, fetchAuthorProfile, fetchTagRegistry, findArticle, getCrawlStats, resolveMediaUrl } from './discovery.js';
+import { discoverArticles, fetchAuthorProfile, fetchTagRegistry, findArticle, getCrawlStats, onArticlesProgress, resolveMediaUrl } from './discovery.js';
 import { parseFrontmatter } from './frontmatter.js';
 import { t, getLocale } from './i18n.js';
 
 const PAGE_SIZE = 20;
 let tagLabels = new Map();
 
+// Bumped at the start of every list-rendering view. A progress listener
+// checks its own snapshot of this before touching the DOM, so a background
+// crawl's late update can't render into a view the user has since
+// navigated away from (e.g. clicked into a different filter/page).
+let currentGeneration = 0;
+
 async function ensureTags() {
     if (!tagLabels.size) tagLabels = new Map((await fetchTagRegistry()).map(tag => [tag.id, tag.label]));
 }
@@ -186,7 +192,10 @@ function renderCard(article) {
 // (checked regardless of site size, since discovery.js now silently skips
 // individual author failures — see the wave-2 comment in discovery.js) and,
 // for larger sites, a near-zero loaded/total ratio at the article level.
-function renderCrawlHealthWarning(view) {
+// Only meaningful once the crawl has actually finished (`done`) — mid-crawl
+// stats are incomplete and would misreport a healthy crawl as broken.
+function renderCrawlHealthWarning(view, done) {
+    if (!done) return;
     const { total, loaded, authorsTotal, authorsFailed } = getCrawlStats();
     const authorsBroken = authorsTotal > 0 && authorsFailed === authorsTotal;
     const articlesBroken = total >= 5 && loaded / total < 0.1;
@@ -199,42 +208,68 @@ function renderCrawlHealthWarning(view) {
     view.appendChild(warning);
 }
 
+// Renders one snapshot of the (possibly still-growing) article list into
+// `view` — shared by the live progress updates and the final render, so a
+// large site's first page shows up as soon as it's indexed instead of
+// blocking on the whole tree (see onArticlesProgress() in discovery.js).
+function renderArticleListBody(view, articles, filter, page, done) {
+    view.innerHTML = '';
+    renderCrawlHealthWarning(view, done);
+    if (filter) {
+        const heading = document.createElement('h2');
+        heading.className = 'category-heading';
+        heading.textContent = filterLabel(filter);
+        view.appendChild(heading);
+    }
+    if (!articles.length) {
+        const message = document.createElement('p');
+        message.className = done ? 'empty' : 'loading';
+        message.textContent = done ? t('emptyArticleList') : t('loadingArticles');
+        view.appendChild(message);
+        return;
+    }
+    const pages = Math.max(1, Math.ceil(articles.length / PAGE_SIZE));
+    const clampedPage = Math.max(1, Math.min(page, pages));
+    articles.slice((clampedPage - 1) * PAGE_SIZE, clampedPage * PAGE_SIZE).forEach(article => view.appendChild(renderCard(article)));
+    if (pages > 1) view.appendChild(buildPaginationNav(clampedPage, pages, filter));
+    if (!done) {
+        const indexing = document.createElement('p');
+        indexing.className = 'loading';
+        indexing.textContent = t('indexingBackground');
+        view.appendChild(indexing);
+    }
+}
+
 export async function loadArticleList(filter = null, page = 1) {
     showListView();
     window.scrollTo(0, 0);
     clearSearchInput();
     const view = document.getElementById('list-view');
     view.innerHTML = `<p class="loading">${t('loadingArticles')}</p>`;
-    try {
-        await ensureTags();
-        const allArticles = await discoverArticles();
+    const generation = ++currentGeneration;
+
+    const applyFilter = allArticles => {
         renderFilterNavigation(allArticles, filter);
-        let articles = allArticles;
-        if (filter) articles = articles.filter(article => {
+        if (!filter) return allArticles;
+        return allArticles.filter(article => {
             if (filter.type === 'tag') return article.tags.includes(filter.value);
             return String(article[filter.type] || '').toLowerCase() === filter.value.toLowerCase();
         });
-        view.innerHTML = '';
-        renderCrawlHealthWarning(view);
-        if (filter) {
-            const heading = document.createElement('h2');
-            heading.className = 'category-heading';
-            heading.textContent = filterLabel(filter);
-            view.appendChild(heading);
-        }
-        if (!articles.length) {
-            const empty = document.createElement('p');
-            empty.className = 'empty';
-            empty.textContent = t('emptyArticleList');
-            view.appendChild(empty);
-            return;
-        }
-        const pages = Math.ceil(articles.length / PAGE_SIZE);
-        page = Math.max(1, Math.min(page, pages));
-        articles.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE).forEach(article => view.appendChild(renderCard(article)));
-        if (pages > 1) view.appendChild(buildPaginationNav(page, pages, filter));
+    };
+
+    const unsubscribe = onArticlesProgress((partialArticles, done) => {
+        if (generation !== currentGeneration) return;
+        renderArticleListBody(view, applyFilter(partialArticles), filter, page, done);
+    });
+    try {
+        await ensureTags();
+        const allArticles = await discoverArticles();
+        if (generation !== currentGeneration) return;
+        renderArticleListBody(view, applyFilter(allArticles), filter, page, true);
     } catch (error) {
-        view.innerHTML = `<p class="error">${t('articlesLoadError', { message: error.message })}</p>`;
+        if (generation === currentGeneration) view.innerHTML = `<p class="error">${t('articlesLoadError', { message: error.message })}</p>`;
+    } finally {
+        unsubscribe();
     }
 }
 
@@ -281,6 +316,7 @@ export const loadTagView = (tag, page = 1) => loadArticleList({ type: 'tag', val
 export async function loadSearch(query) {
     showListView();
     window.scrollTo(0, 0);
+    currentGeneration += 1; // invalidates any still-running loadArticleList progress listener
     const view = document.getElementById('list-view');
     view.innerHTML = `<p class="loading">${t('searching')}</p>`;
     const input = document.getElementById('search-input');
@@ -318,6 +354,7 @@ function buildTOC(content) {
 export async function loadArticle(hash) {
     showArticleView();
     window.scrollTo(0, 0);
+    currentGeneration += 1; // invalidates any still-running loadArticleList progress listener
     clearSearchInput();
     const view = document.getElementById('article-view');
     view.innerHTML = `<p class="loading">${t('loadingGeneric')}</p>`;

cmjs/strings/cs.json

@@ -24,6 +24,7 @@
   "loadingGeneric": "Načítám…",
   "emptyArticleList": "Nebyly nalezeny žádné články.",
   "articlesLoadError": "Články se nepodařilo načíst: {message}",
+  "indexingBackground": "Zbytek článků se stále indexuje na pozadí…",
 
   "searching": "Hledám…",
   "searchHeading": "Hledání: „{query}“",

cmjs/strings/en.json

@@ -24,6 +24,7 @@
   "loadingGeneric": "Loading…",
   "emptyArticleList": "No articles found.",
   "articlesLoadError": "Failed to load articles: {message}",
+  "indexingBackground": "Still indexing the rest of the articles…",
 
   "searching": "Searching…",
   "searchHeading": "Search: \"{query}\"",

cmjs/strings/sk.json

@@ -24,6 +24,7 @@
   "loadingGeneric": "Načítavam…",
   "emptyArticleList": "Neboli nájdené žiadne články.",
   "articlesLoadError": "Články sa nepodarilo načítať: {message}",
+  "indexingBackground": "Zvyšok článkov sa stále indexuje na pozadí…",
 
   "searching": "Hľadám…",
   "searchHeading": "Hľadanie: „{query}“",