cmkiss/js/discovery.js

import { parseFrontmatter } from './frontmatter.js';
import { BASE_URL, ARTICLES_URL } from './config-default.js';
import { mapWithConcurrency } from './concurrency.js';
import { getAllCacheEntries, setCacheEntry, pruneCache } from './cache.js';
import { t } from './i18n.js';

// Caps simultaneous requests in the two places this module still fires one
// per item: markdown content fetches during hydration (see
// hydrateArticle()/hydrateArticles()) and each author's own directory
// listing during wave 2 of indexing (see runCrawl()) — indexing no longer
// needs a per-*article* fetch at all (every stub field is encoded in its
// file's own name — see buildStub()), but still fetches one listing per
// author. Raise it for a fast/local nginx, lower it for a slow or shared/
// production instance — see .claude/nginx-sitemap-crawler-plan.md §2a for
// the original rationale (written when this also governed per-article
// indexing requests).
const MAX_CONCURRENT_REQUESTS = 10;

const MONTH_ABBREVIATIONS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];

function parseHtmlDirectoryListing(raw) {
    const entries = [];
    const seen = new Set();
    const anchors = /<a\b[^>]*\bhref\s*=\s*["']([^"']+)["'][^>]*>[\s\S]*?<\/a>/gi;
    // nginx's default autoindex puts "DD-Mon-YYYY HH:MM" right after the </a>
    // on the same line — used as a best-effort mtime when JSON autoindex
    // isn't available (see the "mtime" note in the README).
    const datePattern = /(\d{2})-([A-Za-z]{3})-(\d{4})\s+(\d{2}):(\d{2})/;
    let match;
    while ((match = anchors.exec(raw))) {
        const href = match[1].replace(/&amp;/g, '&').replace(/&quot;/g, '"').replace(/&#39;/g, "'");
        if (!href || href.startsWith('../') || href.startsWith('/') || href.startsWith('#') || /^[a-z][a-z\d+.-]*:/i.test(href)) continue;
        const clean = href.split(/[?#]/, 1)[0];
        const name = decodeURIComponent(clean.replace(/\/$/, '')).split('/').pop();
        if (!name || name === '.' || name === '..' || seen.has(name)) continue;
        seen.add(name);

        const lineEnd = raw.indexOf('\n', anchors.lastIndex);
        const tail = raw.slice(anchors.lastIndex, lineEnd === -1 ? raw.length : lineEnd);
        const dateMatch = datePattern.exec(tail);
        const monthIndex = dateMatch ? MONTH_ABBREVIATIONS.indexOf(dateMatch[2]) : -1;
        const mtime = monthIndex >= 0
            ? new Date(Date.UTC(+dateMatch[3], monthIndex, +dateMatch[1], +dateMatch[4], +dateMatch[5])).toISOString()
            : null;

        entries.push({ name, type: clean.endsWith('/') ? 'directory' : 'file', mtime });
    }
    return entries;
}

export async function fetchJson(url) {
    const response = await fetch(url);
    if (!response.ok) throw new Error(`HTTP ${response.status} fetching ${url}`);
    const raw = await response.text();
    try {
        const parsed = JSON.parse(raw);
        if (Array.isArray(parsed)) return parsed;
    } catch { /* HTML autoindex fallback */ }
    const entries = parseHtmlDirectoryListing(raw);
    if (entries.length || /<(?:title|h1)[^>]*>\s*(?:index of|directory listing)/i.test(raw)) return entries;
    throw new Error(`Expected a directory listing at ${url}`);
}

// Images live alongside whatever references them: every article file sits
// directly in its author's own directory (see buildStub()), next to that
// author's shared `media/` folder — so `articlePath` (the referencing .md
// file's own URL) is the resolution base. A relative path in frontmatter
// like `media/cover.jpg` resolves against the article file's own directory,
// which is the author directory itself, landing it in that shared folder —
// letting multiple articles reference the same image without duplicating it
// per article.
export function resolveMediaUrl(value, articlePath = '') {
    if (!value) return '';
    let path = String(value).trim();
    if (/^(data:|blob:)/i.test(path)) return path;
    if (/^https?:\/\//i.test(path)) {
        try {
            const url = new URL(path);
            // Legacy compatibility, not a general-purpose feature: content
            // migrated from a WordPress export sometimes still carries an
            // absolute upload URL. Reduce it to a bare filename so it
            // resolves against the article's own directory like any other
            // relative image path, assuming the file itself was migrated
            // alongside the article. A no-op for anything that isn't
            // WordPress-derived content — most deployments never hit this.
            if (url.pathname.includes('/wp-content/uploads/')) path = url.pathname.split('/wp-content/uploads/')[1];
            else return path;
        } catch { return path; }
    }
    return new URL(path.replace(/^\//, ''), articlePath || BASE_URL).href;
}

let articleCache = null;
let crawlPromise = null;
let crawlStats = { total: 0, loaded: 0, authorsTotal: 0, authorsFailed: 0, rootFailed: false };

// Raw (unfiltered/unsorted) article records resolved so far by the crawl
// currently or most recently in flight — lets a late subscriber (a view
// that mounts mid-crawl) immediately see what's already available instead
// of waiting for the next progress tick.
let partialResolved = [];
const progressListeners = new Set();

function sortArticles(rawArticles) {
    return [...rawArticles].sort((a, b) => String(b.date).localeCompare(String(a.date)) || a.hash.localeCompare(b.hash));
}

// Subscribes to incremental crawl progress so a large site's article list
// can render as soon as the first articles are indexed instead of blocking
// on the whole tree. `listener(articles, done)` is called immediately with
// whatever has resolved so far, again as more of the crawl resolves, and a
// final time with `done: true`. Returns an unsubscribe function — always
// call it once the caller stops rendering (e.g. the user navigated away),
// otherwise a background crawl keeps calling a listener for a view that's
// no longer showing.
export function onArticlesProgress(listener) {
    progressListeners.add(listener);
    if (partialResolved.length) listener(sortArticles(partialResolved), false);
    return () => progressListeners.delete(listener);
}

function notifyProgress(done) {
    const snapshot = sortArticles(partialResolved);
    progressListeners.forEach(listener => listener(snapshot, done));
}

// How many of the last crawl's candidate files actually produced an article
// vs. how many were listed, plus how many authors' listings failed
// outright — lets the UI show a "something's badly wrong" indicator instead
// of silently rendering an empty/near-empty list forever.
export function getCrawlStats() {
    return { ...crawlStats };
}

function excerpt(content) {
    const text = String(content || '')
        .replace(/!\[[^\]]*\]\([^)]*\)/g, '')
        .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
        .replace(/`{1,3}[^`]*`{1,3}/g, '')
        .replace(/[>#*_~\\]/g, '')
        .replace(/\s+/g, ' ')
        .trim();
    if (text.length <= 280) return text;
    return `${text.slice(0, 277).replace(/\s+\S*$/, '')}…`;
}

// An article file's own name is the sole authoritative source of its slug,
// publish date, and tags — see the README's "Adding an article" section.
// Format: `<slug>~<DDMMYYYY>[~tag1+tag2+…].md`. `~` (rather than e.g. "--")
// is deliberate: a slug is expected to be plain kebab-case, and "~" isn't a
// character anyone would type by accident while writing one — unlike a
// double-hyphen separator, which collides with an easy slug typo (an extra
// "-") and silently drops the whole article instead of erroring. Returns
// null for any file name that doesn't match — callers treat that as a
// silent skip, e.g. the author's own bio file or a stray unrelated file.
// Drafts aren't a naming convention at all — they live in an author's own
// `draft/` subdirectory (see the README's "Adding an article" section),
// which the crawler never descends into (wave 2 below only reads `type ===
// 'file'` entries from an author's own listing) and which nginx should deny
// outright, so a draft is never indexed or fetchable in the first place.
function parseArticleFileName(fileName) {
    if (!fileName.endsWith('.md')) return null;
    const base = fileName.slice(0, -3);
    const [slug, dateSegment, tagsSegment] = base.split('~');
    if (!slug || !dateSegment || !/^\d{8}$/.test(dateSegment)) return null;
    const day = dateSegment.slice(0, 2);
    const month = dateSegment.slice(2, 4);
    const year = dateSegment.slice(4, 8);
    return {
        slug,
        date: `${year}-${month}-${day}`,
        tags: tagsSegment ? tagsSegment.split('+').filter(Boolean) : [],
    };
}

// Everything an article card/filter/sort needs, parsed entirely from its
// file's own name and its listing entry's mtime — no fetch of any kind.
// This is what runs for every article at index time; the (expensive)
// content fetch is deferred to hydrateArticle(), which only ever runs for
// articles actually being rendered. Returns null if the file name doesn't
// match the naming convention (see parseArticleFileName) — the caller
// treats that as a silent skip, matching the rest of this module's
// approach. `path` (the article file's own URL) doubles as its cache key —
// see cache.js — since every article now lives directly in its author's
// directory, alongside that author's bio file and shared `media/` folder,
// rather than in a directory of its own.
function buildStub(author, fileName, mtime, authorRoot) {
    const parsed = parseArticleFileName(fileName);
    if (!parsed) return null;
    const { slug, date, tags } = parsed;
    const path = `${authorRoot}${encodeURIComponent(fileName)}`;

    return {
        author,
        slug,
        hash: `${author}/${slug}`,
        path,
        mtime,
        date,
        tags,
        hydrated: false,
    };
}

// nginx's JSON autoindex reports mtimes with second (or finer) precision,
// but the HTML-autoindex fallback (see parseHtmlDirectoryListing) only ever
// has minute precision. Truncating both to the same minute-level grain here
// — the one place a raw listing mtime enters the system — means the cache
// comparison in resolveStub() below still matches consistently even if a
// server ever flips between the two formats, instead of every file looking
// "changed" the moment JSON autoindex is lost.
function normalizeMtime(mtime) {
    return mtime ? mtime.slice(0, 16) : null; // "YYYY-MM-DDTHH:MM" prefix
}

// Cache-aware: a file whose mtime matches its cached entry's mtime reuses
// that entry directly — worthwhile only when it's already hydrated
// (skipping a real content re-fetch); a merely-parsed stub is free to
// rebuild, so there's no point caching one just to skip re-parsing a
// string. `cacheSnapshot` is the whole cache read once up front (see
// runCrawl) — an in-memory lookup here instead of a separate IndexedDB read
// per file, which mattered when this also had to guard a listing fetch;
// kept anyway since it's still the same one cheap lookup. `mtime` is a
// change-detection signal, not a required key: `cached.mtime === mtime`
// already matches when both are `null` (a server whose listing never
// yields a parseable mtime — see parseHtmlDirectoryListing), which is the
// right call — there's no evidence the file changed, as opposed to a
// mismatch, which is evidence that it did. Treating "unknown" as "assume
// changed" would defeat the cache on every single crawl instead of just
// when a file actually needs re-hydrating.
function resolveStub(author, fileName, mtime, authorRoot, cacheSnapshot) {
    const path = `${authorRoot}${encodeURIComponent(fileName)}`;
    const cached = cacheSnapshot.get(path);
    if (cached && cached.mtime === mtime && cached.article?.hydrated) return cached.article;
    return buildStub(author, fileName, mtime, authorRoot);
}

// Fields only derivable from an article's actual markdown content (not its
// directory listing) — shared by hydrateArticle() (page-scoped list cards)
// and loadArticle() in views.js (which always fetches its own content
// directly, regardless of what's been hydrated for list rendering).
export function deriveContentFields(meta, content, path) {
    const words = content.trim().split(/\s+/).filter(Boolean).length;
    return {
        readingTime: Math.max(1, Math.round(words / 200)),
        description: meta.description || excerpt(content),
        coverUrl: resolveMediaUrl(meta.cover || meta.image, path),
    };
}

// Fetches and parses a stub's markdown content (title, description, cover
// image, reading time) — the one fetch this module defers until an article
// is actually about to be rendered (a visible list card, or the article
// itself), rather than for every article at index time. Mutates `stub` in
// place (rather than returning a new object) so every other reference to
// this same article — the shared discoverArticles() list, other pending
// hydrateArticle() calls for it — sees the update too, and a later
// re-render of the same page doesn't re-fetch it. No-ops if already
// hydrated. Leaves the stub as-is (title-less but still linkable/sortable)
// on any failure, matching this module's silent-skip approach elsewhere.
//
// De-duplicates concurrent calls for the *same* stub: without this, two
// callers racing on a not-yet-hydrated stub (e.g. a page's own hydration
// racing the next page's background prefetch) would each fire their own
// fetch for the same article file. `inFlightHydrations` tracks the shared
// promise for as long as one is in flight, keyed by the stub object itself
// (each article has exactly one stub instance for its whole lifetime — see
// buildStub()/resolveStub()) — removed again once settled, success or
// failure, so a failed hydration is still retried on the next call rather
// than permanently returning the same rejected attempt.
const inFlightHydrations = new WeakMap();

export function hydrateArticle(stub) {
    if (stub.hydrated) return Promise.resolve(stub);
    const existing = inFlightHydrations.get(stub);
    if (existing) return existing;
    const promise = hydrateArticleNow(stub).finally(() => inFlightHydrations.delete(stub));
    inFlightHydrations.set(stub, promise);
    return promise;
}

async function hydrateArticleNow(stub) {
    try {
        const response = await fetch(stub.path);
        if (!response.ok) throw new Error(`HTTP ${response.status}`);
        const raw = await response.text();
        const { meta, content } = parseFrontmatter(raw);
        // Snapshot stub's own fields *before* mutating, then reapply them
        // after merging in meta — so a stray same-named frontmatter field
        // (a leftover `tags:`/`date:`) can never override the authoritative
        // marker-derived values. (stub can't be used as its own "source of
        // truth" mid-Object.assign — by the time that argument is read it'd
        // already reflect this same mutation.)
        const authoritative = { ...stub };
        Object.assign(stub, meta, authoritative, deriveContentFields(meta, content, stub.path), { hydrated: true });
        await setCacheEntry(stub.path, { mtime: stub.mtime, article: stub });
    } catch (error) {
        // Leave stub unhydrated — retried next time it's rendered. Logged
        // (not surfaced in the UI, which just shows the title-less card)
        // so a persistent failure — a real bug in parseFrontmatter() or
        // deriveContentFields(), not just a transient fetch blip — is still
        // visible somewhere instead of silently retrying forever.
        console.error(`Failed to hydrate article ${stub.path}:`, error);
    }
    return stub;
}

// Concurrency-limited batch hydration. Processing order follows `stubs`'
// own array order (each worker below just claims the next unclaimed index)
// — since discoverArticles() already returns stubs sorted newest-first by
// the authoritative `date` field, that means hydration (and thus which
// search results fill in first) is already newest-first too, for free.
export function hydrateArticles(stubs) {
    return mapWithConcurrency(stubs, MAX_CONCURRENT_REQUESTS, hydrateArticle);
}

async function runCrawl() {
    partialResolved = [];

    // Independent of waves 1/2 below — kicked off in parallel with them and
    // only awaited once it's actually needed, so its latency is hidden
    // behind the (already-required) directory fetches rather than adding
    // to the total.
    const cacheSnapshotPromise = getAllCacheEntries();

    // Wave 1 (root) — always fetched live: the author directories. Unlike
    // wave 2 below, a failure here means discovery can't proceed at all
    // (wrong origin, CORS, autoindex disabled) — rather than letting that
    // reject all the way out to a generic "failed to load" message, treat it
    // the same as "every author's listing failed": resolve to an empty,
    // fully-indexed (done) result so the list view renders normally and
    // renderCrawlHealthWarning() (via rootFailed below) shows the specific
    // crawl-health message instead.
    let authorEntries;
    try {
        authorEntries = await fetchJson(ARTICLES_URL);
    } catch {
        crawlStats = { total: 0, loaded: 0, authorsTotal: 0, authorsFailed: 0, rootFailed: true };
        notifyProgress(true);
        return sortArticles(partialResolved);
    }
    const authors = authorEntries.filter(entry => entry.type === 'directory' && !entry.name.startsWith('.'));

    // Wave 2 — always fetched live, in full, for every author (not just the
    // current page): each author's own directory listing, with per-file
    // mtimes. Capped by MAX_CONCURRENT_REQUESTS like hydrateArticles() below
    // — a site with many authors would otherwise fire one listing request
    // per author simultaneously, with no limit. This is what builds the
    // flat, cross-author list that drives global sorting and pagination —
    // see the plan doc's "keep the flat feed" decision. It's also the *only*
    // per-author network request indexing ever needs: every stub field
    // (slug, date, tags) is parsed straight out of each article file's own
    // name (see buildStub/parseArticleFileName) — there's no per-article
    // fetch at index time. The `type === 'file'` filter below also means any
    // subdirectory (the author's shared `media/` folder, a `draft/` folder —
    // see "Adding an article" in the README) is skipped without ever being
    // descended into. The author's own bio file (`<author>.md`) is filtered
    // out by name in the same listing — only `.md` files other than the bio
    // file are candidate articles. One author's listing failing (network
    // blip, deleted mid-crawl) is silently skipped rather than failing every other
    // author's list too, but is still counted, so a *total* breakage (wrong
    // origin, CORS) remains visible via getCrawlStats() instead of just
    // silently producing an empty article list.
    let authorsFailed = 0;
    const authorFileLists = await mapWithConcurrency(authors, MAX_CONCURRENT_REQUESTS, async author => {
        const authorRoot = `${ARTICLES_URL}${encodeURIComponent(author.name)}/`;
        const bioFileName = `${author.name}.md`;
        try {
            const entries = await fetchJson(authorRoot);
            return entries
                .filter(entry => entry.type === 'file' && entry.name.endsWith('.md') && entry.name !== bioFileName)
                .map(entry => ({ author: author.name, fileName: entry.name, mtime: normalizeMtime(entry.mtime), authorRoot }));
        } catch {
            authorsFailed += 1;
            return [];
        }
    });
    const allFiles = authorFileLists.flat();

    // Every stub is parsed synchronously from data already in hand — no
    // per-file fetch, so no need for mapWithConcurrency here (that's still
    // used by hydrateArticles(), which does real network fetches).
    const cacheSnapshot = await cacheSnapshotPromise;
    let loaded = 0;
    for (const file of allFiles) {
        const stub = resolveStub(file.author, file.fileName, file.mtime, file.authorRoot, cacheSnapshot);
        if (stub) {
            loaded += 1;
            partialResolved.push(stub);
        }
    }

    await pruneCache(allFiles.map(file => `${file.authorRoot}${encodeURIComponent(file.fileName)}`));
    crawlStats = { total: allFiles.length, loaded, authorsTotal: authors.length, authorsFailed, rootFailed: false };

    const finalArticles = sortArticles(partialResolved);
    notifyProgress(true);
    return finalArticles;
}

// Cached after the first successful crawl; concurrent callers during the
// crawl share the same in-flight promise rather than each starting their
// own duplicate crawl. See onArticlesProgress() for incremental results.
export function discoverArticles() {
    if (articleCache) return Promise.resolve(articleCache);
    if (!crawlPromise) {
        crawlPromise = runCrawl().then(
            result => { articleCache = result; crawlPromise = null; return result; },
            error => { crawlPromise = null; throw error; },
        );
    }
    return crawlPromise;
}

export async function findArticle(hash) {
    const article = (await discoverArticles()).find(candidate => candidate.hash === hash);
    if (!article) throw new Error(t('articleNotFound'));
    return article;
}

export async function fetchAuthorProfile(author) {
    const path = `${ARTICLES_URL}${encodeURIComponent(author)}/${encodeURIComponent(author)}.md`;
    try {
        const response = await fetch(path);
        if (!response.ok) return null;
        const { meta, content } = parseFrontmatter(await response.text());
        return { author, ...meta, content, path };
    } catch { return null; }
}