From 96cefba24dd28555966042a4a12dec1101d0add1 Mon Sep 17 00:00:00 2001 From: biosias Date: Tue, 22 Sep 2026 16:18:03 +0200 Subject: [PATCH] Cache per-query search matches so repeat searches skip unchanged articles (AI) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a searchQueries IndexedDB store (js/cache.js) keyed by the exact search text, holding a { articlePath: { mtime, matched } } map per query. loadSearch() loads it once per new query, trusts a cached result whenever an article's mtime still matches instead of running matchesQuery() again, and writes back whatever's newly checked each scan round. Repeat searches for the same term — even after a page reload — now only need to actually check articles that are new or have changed since, not the whole site again. Capped at MAX_CACHED_SEARCHES (30) distinct queries with least-recently-used eviction so trying many different terms over time doesn't grow the cache unbounded. Co-Authored-By: Claude Sonnet 5 --- README.md | 21 +++++++++--- js/cache.js | 92 ++++++++++++++++++++++++++++++++++++++++++++++------- js/views.js | 50 +++++++++++++++++++++++++++-- 3 files changed, 144 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index d68f8c3..39f4066 100644 --- a/README.md +++ b/README.md @@ -507,11 +507,14 @@ of content is ever fetched per page view. content, so a code change to how articles are derived (in `js/discovery.js`) can make old cached entries stale in *structure*, not just content — and a file's `mtime` won't catch that. Bump `CACHE_SCHEMA_VERSION` in `js/cache.js` whenever that shape changes; it's baked into - the IndexedDB database name, so a bump invalidates every cached entry on the next deploy. -- **Concurrency** — `MAX_CONCURRENT_REQUESTS` in `js/discovery.js` (default `10`) caps how - many hydration (content) fetches run at once — the only per-article requests left, since - indexing no longer needs one; raise it for a fast/local nginx, lower it for a slow or shared/ - production instance. + the IndexedDB database name, so a bump invalidates every cached entry on the next deploy — + articles and cached searches alike. A change to the search-match record shape instead, or a new + store entirely, only needs `DB_VERSION` bumped (IndexedDB's own upgrade counter): `onupgradeneeded` + creates whatever store is missing without touching the other one's existing data. +- **Concurrency** — `MAX_CONCURRENT_REQUESTS` in `js/discovery.js` (default `10`) caps both how + many hydration (content) fetches run at once and how many authors' directory listings are + fetched at once during indexing; raise it for a fast/local nginx, lower it for a slow or + shared/production instance. - **Prefetching** — once the current page has fully rendered, the next page's articles are hydrated in the background (fire-and-forget), so clicking "next" is usually an instant cache hit rather than a fresh round-trip. @@ -525,6 +528,14 @@ of content is ever fetched per page view. a large site to a handful of requests instead of hydrating the entire article set up front. The page count is exact once the whole set's been scanned, and a lower bound (at least one more page) until then. +- **Persistent search-match cache** — alongside the resumable in-session scan above, each query's + per-article match results (`{ mtime, matched }`, keyed by article path) are cached in IndexedDB + too (`searchQueries` store in `js/cache.js`, keyed by the query text itself) and reused across + page reloads and browser sessions, not just while paging through one visit. Searching the same + term again only re-fetches and re-checks articles that are new or whose `mtime` has changed + since — everything else is trusted from the cache without a content re-fetch at all. Capped at + `MAX_CACHED_SEARCHES` (default `30`) distinct queries, evicting the least-recently-searched one + first, so trying many different search terms over time doesn't grow the cache unbounded. - **Crawl health** — if the root author-listing fetch fails outright, or if every individual author's own directory listing fails (wrong origin, CORS misconfigured), a small warning banner is shown above the article list instead of silently rendering an empty page (see diff --git a/js/cache.js b/js/cache.js index 2994674..6d52d38 100644 --- a/js/cache.js +++ b/js/cache.js @@ -1,10 +1,11 @@ -// Persistent cache for discovered articles, keyed by each article file's own -// path. Raw IndexedDB (no wrapper library — see +// Persistent cache for discovered articles (keyed by each article file's own +// path) and for per-search-query match results (keyed by the query text — +// see setSearchCacheEntry() below). Raw IndexedDB (no wrapper library — see // .claude/nginx-sitemap-crawler-plan.md L4/L5 on the zero-dependency, // long-stable-API requirement). If IndexedDB is unavailable (private // browsing, old browser, blocked), every operation below silently becomes a -// no-op so discovery.js gets live-fetch behaviour for free without needing -// its own availability branching (plan doc R6). +// no-op so discovery.js/views.js get live-fetch/live-scan behaviour for free +// without needing their own availability branching (plan doc R6). // Bump this whenever discovery.js's derived article record shape changes. // The version is baked into the DB name so a code deploy invalidates every @@ -18,7 +19,19 @@ const CACHE_SCHEMA_VERSION = 3; // this module degrades to live-fetch behaviour whenever the cache is empty // or unavailable). Do this deliberately, not casually. const DB_NAME = `cmkiss-cache-v${CACHE_SCHEMA_VERSION}`; -const STORE_NAME = 'articleFiles'; +const ARTICLE_STORE = 'articleFiles'; +const SEARCH_STORE = 'searchQueries'; +// IndexedDB's own upgrade counter — separate from CACHE_SCHEMA_VERSION, +// which only governs the *article* record shape (and busts the cache by +// renaming the whole database). Bump this instead whenever a store or index +// is added/changed; onupgradeneeded below creates only what's missing, so +// existing article-cache data survives the upgrade. +const DB_VERSION = 2; +// Caps how many distinct queries' match results stay cached — otherwise +// every unique term ever typed into search would keep its own per-article +// map forever, growing unbounded on a long-lived site. Evicted +// least-recently-searched-first — see setSearchCacheEntry(). +const MAX_CACHED_SEARCHES = 30; let cacheAvailable = true; let dbPromise = null; @@ -27,9 +40,13 @@ function openDb() { if (!dbPromise) { dbPromise = new Promise((resolve, reject) => { if (typeof indexedDB === 'undefined') { reject(new Error('IndexedDB unavailable')); return; } - const request = indexedDB.open(DB_NAME, 1); + const request = indexedDB.open(DB_NAME, DB_VERSION); request.onupgradeneeded = () => { - request.result.createObjectStore(STORE_NAME, { keyPath: 'path' }); + const db = request.result; + if (!db.objectStoreNames.contains(ARTICLE_STORE)) db.createObjectStore(ARTICLE_STORE, { keyPath: 'path' }); + if (!db.objectStoreNames.contains(SEARCH_STORE)) { + db.createObjectStore(SEARCH_STORE, { keyPath: 'query' }).createIndex('lastUsed', 'lastUsed'); + } }; request.onsuccess = () => resolve(request.result); request.onerror = () => reject(request.error); @@ -42,7 +59,7 @@ function openDb() { // Opens a store and resolves/rejects with the given IDBRequest's own outcome // — the standard pattern, so result timing never depends on transaction // lifecycle details. -async function runRequest(mode, makeRequest) { +async function runRequest(storeName, mode, makeRequest) { if (!cacheAvailable) return undefined; let db; try { @@ -51,7 +68,7 @@ async function runRequest(mode, makeRequest) { return undefined; // openDb() itself already flipped cacheAvailable } try { - const store = db.transaction(STORE_NAME, mode).objectStore(STORE_NAME); + const store = db.transaction(storeName, mode).objectStore(storeName); return await new Promise((resolve, reject) => { const request = makeRequest(store); request.onsuccess = () => resolve(request.result); @@ -72,12 +89,12 @@ async function runRequest(mode, makeRequest) { // reads this once up front rather than doing a point lookup per file during // indexing. Empty Map on a miss or if the cache is unavailable. export async function getAllCacheEntries() { - const records = await runRequest('readonly', store => store.getAll()); + const records = await runRequest(ARTICLE_STORE, 'readonly', store => store.getAll()); return new Map((records || []).map(record => [record.path, record])); } export async function setCacheEntry(path, entry) { - await runRequest('readwrite', store => store.put({ ...entry, path })); + await runRequest(ARTICLE_STORE, 'readwrite', store => store.put({ ...entry, path })); } // Removes any cached entry whose path is no longer in the live directory @@ -92,7 +109,7 @@ export async function pruneCache(validPaths) { return; // openDb() itself already flipped cacheAvailable } try { - const store = db.transaction(STORE_NAME, 'readwrite').objectStore(STORE_NAME); + const store = db.transaction(ARTICLE_STORE, 'readwrite').objectStore(ARTICLE_STORE); await new Promise((resolve, reject) => { const cursorRequest = store.openCursor(); cursorRequest.onsuccess = event => { @@ -108,3 +125,54 @@ export async function pruneCache(validPaths) { // mean IndexedDB itself is unavailable. } } + +// Per-query search-match cache: `entries` is a { [articlePath]: { mtime, +// matched } } map — see loadSearch() in views.js, which trusts an entry +// (skipping a content re-fetch entirely) whenever its mtime still matches +// the live article, and only re-checks what's new or changed. Keyed by the +// exact (locale-lowercased) text matchesQuery() itself compares against, so +// "Linux" and "linux" share one cached entry. +export async function getSearchCacheEntry(query) { + return runRequest(SEARCH_STORE, 'readonly', store => store.get(query)); +} + +export async function setSearchCacheEntry(query, entries) { + await runRequest(SEARCH_STORE, 'readwrite', store => store.put({ query, entries, lastUsed: Date.now() })); + await pruneSearchCache(); +} + +// Keeps the search-query cache bounded to MAX_CACHED_SEARCHES entries, +// dropping the least-recently-searched ones first — the same "don't grow +// unbounded" concern pruneCache() addresses for the article cache, just +// against searches instead of renamed/deleted files. +async function pruneSearchCache() { + if (!cacheAvailable) return; + let db; + try { + db = await openDb(); + } catch { + return; + } + try { + const store = db.transaction(SEARCH_STORE, 'readwrite').objectStore(SEARCH_STORE); + let excess = await new Promise((resolve, reject) => { + const request = store.count(); + request.onsuccess = () => resolve(request.result - MAX_CACHED_SEARCHES); + request.onerror = () => reject(request.error); + }); + if (excess <= 0) return; + await new Promise((resolve, reject) => { + const cursorRequest = store.index('lastUsed').openCursor(); + cursorRequest.onsuccess = event => { + const cursor = event.target.result; + if (!cursor || excess <= 0) { resolve(); return; } + cursor.delete(); + excess -= 1; + cursor.continue(); + }; + cursorRequest.onerror = () => reject(cursorRequest.error); + }); + } catch { + // Same reasoning as runRequest() above. + } +} diff --git a/js/views.js b/js/views.js index 5300b8c..2bedb1d 100644 --- a/js/views.js +++ b/js/views.js @@ -1,5 +1,6 @@ import { templates } from './templates.js'; import { deriveContentFields, discoverArticles, fetchAuthorProfile, findArticle, getCrawlStats, hydrateArticles, onArticlesProgress, resolveMediaUrl } from './discovery.js'; +import { getSearchCacheEntry, setSearchCacheEntry } from './cache.js'; import { parseFrontmatter } from './frontmatter.js'; import { t, getLocale } from './i18n.js'; @@ -374,6 +375,16 @@ const SEARCH_SCAN_BATCH = PAGE_SIZE; // same page) skip scanning entirely, since `matches` only ever grows and // stub order never changes. Reset whenever the query text changes; holds // only the most recent query, so a new search simply replaces it. +// +// `entries` is this same session's in-memory mirror of the persistent +// per-query cache (see cache.js's getSearchCacheEntry()/ +// setSearchCacheEntry()): a { [articlePath]: { mtime, matched } } map, +// loaded once when the query first changes and written back whenever a scan +// round adds something new to it (`dirty`). Reusing it — instead of only +// `cursor`/`matches` — means even a *first* search after a page reload can +// skip re-fetching and re-checking every article whose content hasn't +// changed since it was last searched for, no matter how this query's own +// scan happened to leave off last time. let searchState = null; export async function loadSearch(query, page = 1) { @@ -399,7 +410,16 @@ export async function loadSearch(query, page = 1) { renderFilterNavigation(stubs); if (!searchState || searchState.query !== query || searchState.stubs !== stubs) { - searchState = { query, stubs, cursor: 0, matches: [] }; + // Loaded once per new query. Pruned to the current stub set's + // paths so a renamed/deleted article's old cached result can't + // linger — cheap, since `stubs` (unlike hydrating them) is + // already in hand with no further fetch. + const cached = await getSearchCacheEntry(needle); + if (generation !== currentGeneration) return; + const validPaths = new Set(stubs.map(article => article.path)); + const entries = {}; + for (const [path, entry] of Object.entries(cached?.entries || {})) if (validPaths.has(path)) entries[path] = entry; + searchState = { query, stubs, cursor: 0, matches: [], entries, dirty: false }; } const state = searchState; @@ -451,12 +471,38 @@ export async function loadSearch(query, page = 1) { while (state.cursor < state.stubs.length && state.matches.length < target) { if (generation !== currentGeneration) return; const batch = state.stubs.slice(state.cursor, state.cursor + SEARCH_SCAN_BATCH); + // Still hydrates the whole batch — a no-op per already-hydrated + // stub (see hydrateArticle() in discovery.js), which a cache hit + // below always is: it can only exist for a path this exact + // query already matched content against, at this same mtime, + // which means it's already sitting in the article-level cache + // too. Rendering pageArticles further down never re-hydrates, + // so this stays unconditional rather than trusting that chain + // of reasoning to never drift out of sync. await hydrateArticles(batch); if (generation !== currentGeneration) return; state.cursor += batch.length; - batch.forEach(article => { if (matchesQuery(article)) state.matches.push(article); }); + batch.forEach(article => { + // A cached result only counts as known if its mtime still + // matches the live stub — an edited-since-last-search + // article is re-checked, same as one with no entry at all. + const known = state.entries[article.path]; + const isKnown = known?.mtime === article.mtime; + const matched = isKnown ? known.matched : matchesQuery(article); + if (!isKnown) { state.entries[article.path] = { mtime: article.mtime, matched }; state.dirty = true; } + if (matched) state.matches.push(article); + }); renderResults(); } + // Persist whatever's new this round — fire-and-forget, since a + // failed/slow write should never hold up rendering (cache.js + // already swallows its own storage errors). Next time this exact + // query is searched, anything covered here needs no re-fetch at + // all, only what's new or changed since. + if (state.dirty) { + state.dirty = false; + setSearchCacheEntry(needle, state.entries); + } } catch (error) { if (generation === currentGeneration) view.innerHTML = `

${t('searchError', { message: error.message })}

`; }