From 469a19388efebb2fdc20ab4d504c581c757b3812 Mon Sep 17 00:00:00 2001 From: AI Date: Fri, 11 Sep 2026 12:06:39 +0200 Subject: [PATCH] Add cache schema versioning and isolate per-author crawl failures (AI) Bumps the IndexedDB DB name on schema changes so a code deploy can't serve stale-shaped cached articles. One author's directory listing failing no longer takes the whole site down; a health banner now covers that all-authors-failed case too, not just the low-ratio one. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01AXSVPS3UmmASiiprPfcnUd --- README.md | 9 +++++++++ cmjs/js/cache.js | 8 +++++++- cmjs/js/discovery.js | 39 ++++++++++++++++++++++++++------------- cmjs/js/views.js | 14 +++++++++++--- 4 files changed, 53 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 201433d..2f5b6bc 100644 --- a/README.md +++ b/README.md @@ -282,9 +282,18 @@ read its `.tag`/`.date` markers) and its Markdown content. by directory path and invalidated by comparing the live `mtime` against the cached one. If IndexedDB is unavailable (private browsing, disabled storage), the crawler transparently falls back to fetching everything live on every visit. +- **Cache schema version** — the cached records store the full derived article shape, not raw + content, so a code change to how articles are derived (in `cmjs/js/discovery.js`) can make old + cached entries stale in *structure*, not just content — and a directory's `mtime` won't catch + that. Bump `CACHE_SCHEMA_VERSION` in `cmjs/js/cache.js` whenever that shape changes; it's baked + into the IndexedDB database name, so a bump invalidates every cached entry on the next deploy. - **Concurrency** — `MAX_CONCURRENT_REQUESTS` in `cmjs/js/discovery.js` (default `10`) caps how many post directories are crawled at once; raise it for a fast/local nginx, lower it for a slow or shared/production instance. +- **Crawl health** — if every author's directory listing fails outright, or very few of the + known post directories loaded, a small warning banner is shown above the article list instead + of silently rendering an empty page (see `getCrawlStats()` / `renderCrawlHealthWarning()` in + `cmjs/js/discovery.js` / `cmjs/js/views.js`). - **Design rationale** — the full reasoning (wave-based crawling, recency prioritisation, HTML-autoindex fallback, and long-term "deploy and forget" constraints) is documented in [`.claude/nginx-sitemap-crawler-plan.md`](.claude/nginx-sitemap-crawler-plan.md), which ships diff --git a/cmjs/js/cache.js b/cmjs/js/cache.js index 5a8dbe6..705090a 100644 --- a/cmjs/js/cache.js +++ b/cmjs/js/cache.js @@ -5,7 +5,13 @@ // silently becomes a no-op so discovery.js gets live-fetch behaviour for free // without needing its own availability branching (plan doc R6). -const DB_NAME = 'cmjs-cache'; +// Bump this whenever discovery.js's derived article record shape changes. +// The version is baked into the DB name so a code deploy invalidates every +// cached entry immediately, instead of stale-shaped records lingering until +// each directory's mtime happens to change — which, on a "deploy and +// forget" site, could be years. +const CACHE_SCHEMA_VERSION = 1; +const DB_NAME = `cmjs-cache-v${CACHE_SCHEMA_VERSION}`; const STORE_NAME = 'slugDirs'; let cacheAvailable = true; diff --git a/cmjs/js/discovery.js b/cmjs/js/discovery.js index 21bfb18..8364258 100644 --- a/cmjs/js/discovery.js +++ b/cmjs/js/discovery.js @@ -88,11 +88,12 @@ export async function fetchTagRegistry() { } let articleCache = null; -let crawlStats = { total: 0, loaded: 0 }; +let crawlStats = { total: 0, loaded: 0, authorsTotal: 0, authorsFailed: 0 }; // How many of the last crawl's slug directories actually produced an article -// vs. how many were candidates — lets the UI show a "something's badly wrong" -// indicator instead of silently rendering an empty/near-empty list forever. +// vs. how many were candidates, plus how many authors' listings failed +// outright — lets the UI show a "something's badly wrong" indicator instead +// of silently rendering an empty/near-empty list forever. export function getCrawlStats() { return { ...crawlStats }; } @@ -181,17 +182,29 @@ export async function discoverArticles() { // current page): each author's slug directories, with their mtimes. This // is what builds the flat, cross-author list that drives global sorting // and pagination — see the plan doc's "keep the flat feed" decision. + // One author's listing failing (network blip, deleted mid-crawl) is + // silently skipped rather than failing every other author's list too — + // consistent with this module's usual per-item failure handling (see + // buildArticle) — but is still counted, so a *total* breakage (wrong + // origin, CORS) remains visible via getCrawlStats() instead of just + // silently producing an empty article list. + let authorsFailed = 0; const slugDirLists = await Promise.all(authors.map(async author => { const authorRoot = `${BASE_URL}articles/${encodeURIComponent(author.name)}/`; - const entries = await fetchJson(authorRoot); - return entries - .filter(entry => entry.type === 'directory' && entry.name !== '_history') - .map(entry => ({ - author: author.name, - slug: entry.name, - mtime: entry.mtime ?? null, - root: `${authorRoot}${encodeURIComponent(entry.name)}/`, - })); + try { + const entries = await fetchJson(authorRoot); + return entries + .filter(entry => entry.type === 'directory' && entry.name !== '_history') + .map(entry => ({ + author: author.name, + slug: entry.name, + mtime: entry.mtime ?? null, + root: `${authorRoot}${encodeURIComponent(entry.name)}/`, + })); + } catch { + authorsFailed += 1; + return []; + } })); const slugDirs = slugDirLists.flat(); @@ -214,7 +227,7 @@ export async function discoverArticles() { }); await pruneCache(slugDirs.map(dir => dir.root)); - crawlStats = { total: slugDirs.length, loaded }; + crawlStats = { total: slugDirs.length, loaded, authorsTotal: authors.length, authorsFailed }; articleCache = articles .filter(article => article?.published && article.date) diff --git a/cmjs/js/views.js b/cmjs/js/views.js index 9dd9df3..bfb1485 100644 --- a/cmjs/js/views.js +++ b/cmjs/js/views.js @@ -171,12 +171,20 @@ function renderCard(article) { // Near-total crawl failure (wrong origin after a migration, CORS broken, // autoindex disabled) should be visible rather than silently rendering an // empty/near-empty list forever — see .claude/nginx-sitemap-crawler-plan.md L3. +// Two independent signals feed this: every author's own listing failing +// (checked regardless of site size, since discovery.js now silently skips +// individual author failures — see the wave-2 comment in discovery.js) and, +// for larger sites, a near-zero loaded/total ratio at the article level. function renderCrawlHealthWarning(view) { - const { total, loaded } = getCrawlStats(); - if (total < 5 || loaded / total >= 0.1) return; + const { total, loaded, authorsTotal, authorsFailed } = getCrawlStats(); + const authorsBroken = authorsTotal > 0 && authorsFailed === authorsTotal; + const articlesBroken = total >= 5 && loaded / total < 0.1; + if (!authorsBroken && !articlesBroken) return; const warning = document.createElement('p'); warning.className = 'error'; - warning.textContent = `Pozor: načtena jen ${loaded} z ${total} položek — zkontrolujte nastavení serveru.`; + warning.textContent = authorsBroken + ? 'Pozor: nepodařilo se načíst žádného autora — zkontrolujte nastavení serveru.' + : `Pozor: načtena jen ${loaded} z ${total} položek — zkontrolujte nastavení serveru.`; view.appendChild(warning); }