cmkiss/js/discovery.js

import { parseFrontmatter } from './frontmatter.js';
import { BASE_URL, ARTICLES_URL } from './config.js';
import { mapWithConcurrency } from './concurrency.js';
import { getAllCacheEntries, setCacheEntry, pruneCache } from './cache.js';
import { t } from './i18n.js';

// Caps simultaneous markdown content fetches during hydration (see
// hydrateArticle()/hydrateArticles()) — the only per-article requests left,
// since indexing no longer needs a per-article listing fetch at all (every
// stub field is encoded in its file's own name — see buildStub()). Raise it
// for a fast/local nginx, lower it for a slow or shared/production instance
// — see .claude/nginx-sitemap-crawler-plan.md §2a for the original rationale
// (written when this also governed indexing requests).
const MAX_CONCURRENT_REQUESTS = 10;

const MONTH_ABBREVIATIONS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];

function parseHtmlDirectoryListing(raw) {
    const entries = [];
    const seen = new Set();
    const anchors = /<a\b[^>]*\bhref\s*=\s*["']([^"']+)["'][^>]*>[\s\S]*?<\/a>/gi;
    // nginx's default autoindex puts "DD-Mon-YYYY HH:MM" right after the </a>
    // on the same line — used as a best-effort mtime when JSON autoindex
    // isn't available (see the "mtime" note in the README).
    const datePattern = /(\d{2})-([A-Za-z]{3})-(\d{4})\s+(\d{2}):(\d{2})/;
    let match;
    while ((match = anchors.exec(raw))) {
        const href = match[1].replace(/&amp;/g, '&').replace(/&quot;/g, '"').replace(/&#39;/g, "'");
        if (!href || href.startsWith('../') || href.startsWith('/') || href.startsWith('#') || /^[a-z][a-z\d+.-]*:/i.test(href)) continue;
        const clean = href.split(/[?#]/, 1)[0];
        const name = decodeURIComponent(clean.replace(/\/$/, '')).split('/').pop();
        if (!name || name === '.' || name === '..' || seen.has(name)) continue;
        seen.add(name);

        const lineEnd = raw.indexOf('\n', anchors.lastIndex);
        const tail = raw.slice(anchors.lastIndex, lineEnd === -1 ? raw.length : lineEnd);
        const dateMatch = datePattern.exec(tail);
        const monthIndex = dateMatch ? MONTH_ABBREVIATIONS.indexOf(dateMatch[2]) : -1;
        const mtime = monthIndex >= 0
            ? new Date(Date.UTC(+dateMatch[3], monthIndex, +dateMatch[1], +dateMatch[4], +dateMatch[5])).toISOString()
            : null;

        entries.push({ name, type: clean.endsWith('/') ? 'directory' : 'file', mtime });
    }
    return entries;
}

export async function fetchJson(url) {
    const response = await fetch(url);
    if (!response.ok) throw new Error(`HTTP ${response.status} fetching ${url}`);
    const raw = await response.text();
    try {
        const parsed = JSON.parse(raw);
        if (Array.isArray(parsed)) return parsed;
    } catch { /* HTML autoindex fallback */ }
    const entries = parseHtmlDirectoryListing(raw);
    if (entries.length || /<(?:title|h1)[^>]*>\s*(?:index of|directory listing)/i.test(raw)) return entries;
    throw new Error(`Expected a directory listing at ${url}`);
}

// Images live alongside whatever references them: every article file sits
// directly in its author's own directory (see buildStub()), next to that
// author's shared `media/` folder — so `articlePath` (the referencing .md
// file's own URL) is the resolution base. A relative path in frontmatter
// like `media/cover.jpg` resolves against the article file's own directory,
// which is the author directory itself, landing it in that shared folder —
// letting multiple articles reference the same image without duplicating it
// per article.
export function resolveMediaUrl(value, articlePath = '') {
    if (!value) return '';
    let path = String(value).trim();
    if (/^(data:|blob:)/i.test(path)) return path;
    if (/^https?:\/\//i.test(path)) {
        try {
            const url = new URL(path);
            // Legacy compatibility, not a general-purpose feature: content
            // migrated from a WordPress export sometimes still carries an
            // absolute upload URL. Reduce it to a bare filename so it
            // resolves against the article's own directory like any other
            // relative image path, assuming the file itself was migrated
            // alongside the article. A no-op for anything that isn't
            // WordPress-derived content — most deployments never hit this.
            if (url.pathname.includes('/wp-content/uploads/')) path = url.pathname.split('/wp-content/uploads/')[1];
            else return path;
        } catch { return path; }
    }
    return new URL(path.replace(/^\//, ''), articlePath || BASE_URL).href;
}

let articleCache = null;
let crawlPromise = null;
let crawlStats = { total: 0, loaded: 0, authorsTotal: 0, authorsFailed: 0, rootFailed: false };

// Raw (unfiltered/unsorted) article records resolved so far by the crawl
// currently or most recently in flight — lets a late subscriber (a view
// that mounts mid-crawl) immediately see what's already available instead
// of waiting for the next progress tick.
let partialResolved = [];
const progressListeners = new Set();

function sortAndFilterPublished(rawArticles) {
    return rawArticles
        .filter(article => article?.published && article.date)
        .sort((a, b) => String(b.date).localeCompare(String(a.date)) || a.hash.localeCompare(b.hash));
}

// Subscribes to incremental crawl progress so a large site's article list
// can render as soon as the first articles are indexed instead of blocking
// on the whole tree. `listener(articles, done)` is called immediately with
// whatever has resolved so far, again as more of the crawl resolves, and a
// final time with `done: true`. Returns an unsubscribe function — always
// call it once the caller stops rendering (e.g. the user navigated away),
// otherwise a background crawl keeps calling a listener for a view that's
// no longer showing.
export function onArticlesProgress(listener) {
    progressListeners.add(listener);
    if (partialResolved.length) listener(sortAndFilterPublished(partialResolved), false);
    return () => progressListeners.delete(listener);
}

function notifyProgress(done) {
    const snapshot = sortAndFilterPublished(partialResolved);
    progressListeners.forEach(listener => listener(snapshot, done));
}

// How many of the last crawl's candidate files actually produced an article
// vs. how many were listed, plus how many authors' listings failed
// outright — lets the UI show a "something's badly wrong" indicator instead
// of silently rendering an empty/near-empty list forever.
export function getCrawlStats() {
    return { ...crawlStats };
}

function excerpt(content) {
    const text = String(content || '')
        .replace(/!\[[^\]]*\]\([^)]*\)/g, '')
        .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
        .replace(/`{1,3}[^`]*`{1,3}/g, '')
        .replace(/[>#*_~\\]/g, '')
        .replace(/\s+/g, ' ')
        .trim();
    if (text.length <= 280) return text;
    return `${text.slice(0, 277).replace(/\s+\S*$/, '')}…`;
}

// An article file's own name is the sole authoritative source of its slug,
// publish date, tags/category, and draft status — see the README's "Adding
// an article" section. Format: `<slug>~<DDMMYYYY>[~tag1+tag2+…].md`,
// optionally prefixed with `_` for draft. `~` (rather than e.g. "--") is
// deliberate: a slug is expected to be plain kebab-case, and "~" isn't a
// character anyone would type by accident while writing one — unlike a
// double-hyphen separator, which collides with an easy slug typo (an extra
// "-") and silently drops the whole article instead of erroring. Returns
// null for any file name that doesn't match — callers treat that as a
// silent skip, e.g. the author's own bio file or a stray unrelated file.
function parseArticleFileName(fileName) {
    if (!fileName.endsWith('.md')) return null;
    const base = fileName.slice(0, -3);
    const draft = base.startsWith('_');
    const core = draft ? base.slice(1) : base;
    const [slug, dateSegment, tagsSegment] = core.split('~');
    if (!slug || !dateSegment || !/^\d{8}$/.test(dateSegment)) return null;
    const day = dateSegment.slice(0, 2);
    const month = dateSegment.slice(2, 4);
    const year = dateSegment.slice(4, 8);
    return {
        slug,
        date: `${year}-${month}-${day}`,
        tags: tagsSegment ? tagsSegment.split('+').filter(Boolean) : [],
        draft,
    };
}

// Everything an article card/filter/sort needs, parsed entirely from its
// file's own name and its listing entry's mtime — no fetch of any kind.
// This is what runs for every article at index time; the (expensive)
// content fetch is deferred to hydrateArticle(), which only ever runs for
// articles actually being rendered. Returns null if the file name doesn't
// match the naming convention (see parseArticleFileName) — the caller
// treats that as a silent skip, matching the rest of this module's
// approach. `path` (the article file's own URL) doubles as its cache key —
// see cache.js — since every article now lives directly in its author's
// directory, alongside that author's bio file and shared `media/` folder,
// rather than in a directory of its own.
function buildStub(author, fileName, mtime, authorRoot) {
    const parsed = parseArticleFileName(fileName);
    if (!parsed) return null;
    const { slug, date, tags, draft } = parsed;
    const path = `${authorRoot}${encodeURIComponent(fileName)}`;
    // "Category" isn't a separate concept anymore — it's just whichever tag
    // id (if any) looks like a `year-YYYY` id.
    const category = tags.find(id => /^year-20\d{2}$/i.test(id)) || null;

    return {
        author,
        slug,
        hash: `${author}/${slug}`,
        path,
        mtime,
        category,
        categoryYear: category ? Number(category.replace(/^year-/i, '')) : null,
        date,
        tags,
        published: !draft,
        hydrated: false,
    };
}

// nginx's JSON autoindex reports mtimes with second (or finer) precision,
// but the HTML-autoindex fallback (see parseHtmlDirectoryListing) only ever
// has minute precision. Truncating both to the same minute-level grain here
// — the one place a raw listing mtime enters the system — means the cache
// comparison in resolveStub() below still matches consistently even if a
// server ever flips between the two formats, instead of every file looking
// "changed" the moment JSON autoindex is lost.
function normalizeMtime(mtime) {
    return mtime ? mtime.slice(0, 16) : null; // "YYYY-MM-DDTHH:MM" prefix
}

// Cache-aware: a file whose mtime matches its cached entry's mtime reuses
// that entry directly — worthwhile only when it's already hydrated
// (skipping a real content re-fetch); a merely-parsed stub is free to
// rebuild, so there's no point caching one just to skip re-parsing a
// string. `cacheSnapshot` is the whole cache read once up front (see
// runCrawl) — an in-memory lookup here instead of a separate IndexedDB read
// per file, which mattered when this also had to guard a listing fetch;
// kept anyway since it's still the same one cheap lookup.
function resolveStub(author, fileName, mtime, authorRoot, cacheSnapshot) {
    const path = `${authorRoot}${encodeURIComponent(fileName)}`;
    const cached = cacheSnapshot.get(path);
    if (cached && mtime != null && cached.mtime === mtime && cached.article?.hydrated) return cached.article;
    return buildStub(author, fileName, mtime, authorRoot);
}

// Fields only derivable from an article's actual markdown content (not its
// directory listing) — shared by hydrateArticle() (page-scoped list cards)
// and loadArticle() in views.js (which always fetches its own content
// directly, regardless of what's been hydrated for list rendering).
export function deriveContentFields(meta, content, path) {
    const words = content.trim().split(/\s+/).filter(Boolean).length;
    return {
        readingTime: Math.max(1, Math.round(words / 200)),
        description: meta.description || excerpt(content),
        coverUrl: resolveMediaUrl(meta.cover || meta.image, path),
    };
}

// Fetches and parses a stub's markdown content (title, description, cover
// image, reading time) — the one fetch this module defers until an article
// is actually about to be rendered (a visible list card, or the article
// itself), rather than for every article at index time. Mutates `stub` in
// place (rather than returning a new object) so every other reference to
// this same article — the shared discoverArticles() list, other pending
// hydrateArticle() calls for it — sees the update too, and a later
// re-render of the same page doesn't re-fetch it. No-ops if already
// hydrated. Leaves the stub as-is (title-less but still linkable/sortable)
// on any failure, matching this module's silent-skip approach elsewhere.
//
// De-duplicates concurrent calls for the *same* stub: without this, two
// callers racing on a not-yet-hydrated stub (e.g. a page's own hydration
// racing the next page's background prefetch) would each fire their own
// fetch for the same article file. `inFlightHydrations` tracks the shared
// promise for as long as one is in flight, keyed by the stub object itself
// (each article has exactly one stub instance for its whole lifetime — see
// buildStub()/resolveStub()) — removed again once settled, success or
// failure, so a failed hydration is still retried on the next call rather
// than permanently returning the same rejected attempt.
const inFlightHydrations = new WeakMap();

export function hydrateArticle(stub) {
    if (stub.hydrated) return Promise.resolve(stub);
    const existing = inFlightHydrations.get(stub);
    if (existing) return existing;
    const promise = hydrateArticleNow(stub).finally(() => inFlightHydrations.delete(stub));
    inFlightHydrations.set(stub, promise);
    return promise;
}

async function hydrateArticleNow(stub) {
    try {
        const response = await fetch(stub.path);
        if (!response.ok) throw new Error(`HTTP ${response.status}`);
        const raw = await response.text();
        const { meta, content } = parseFrontmatter(raw);
        // Snapshot stub's own fields *before* mutating, then reapply them
        // after merging in meta — so a stray same-named frontmatter field
        // (a leftover `tags:`/`date:`) can never override the authoritative
        // marker-derived values. (stub can't be used as its own "source of
        // truth" mid-Object.assign — by the time that argument is read it'd
        // already reflect this same mutation.)
        const authoritative = { ...stub };
        Object.assign(stub, meta, authoritative, deriveContentFields(meta, content, stub.path), { hydrated: true });
        await setCacheEntry(stub.path, { mtime: stub.mtime, article: stub });
    } catch { /* leave stub unhydrated — retried next time it's rendered */ }
    return stub;
}

// Concurrency-limited batch hydration. Processing order follows `stubs`'
// own array order (each worker below just claims the next unclaimed index)
// — since discoverArticles() already returns stubs sorted newest-first by
// the authoritative `date` field, that means hydration (and thus which
// search results fill in first) is already newest-first too, for free.
export function hydrateArticles(stubs) {
    return mapWithConcurrency(stubs, MAX_CONCURRENT_REQUESTS, hydrateArticle);
}

async function runCrawl() {
    partialResolved = [];

    // Independent of waves 1/2 below — kicked off in parallel with them and
    // only awaited once it's actually needed, so its latency is hidden
    // behind the (already-required) directory fetches rather than adding
    // to the total.
    const cacheSnapshotPromise = getAllCacheEntries();

    // Wave 1 (root) — always fetched live: the author directories. Unlike
    // wave 2 below, a failure here means discovery can't proceed at all
    // (wrong origin, CORS, autoindex disabled) — rather than letting that
    // reject all the way out to a generic "failed to load" message, treat it
    // the same as "every author's listing failed": resolve to an empty,
    // fully-indexed (done) result so the list view renders normally and
    // renderCrawlHealthWarning() (via rootFailed below) shows the specific
    // crawl-health message instead.
    let authorEntries;
    try {
        authorEntries = await fetchJson(ARTICLES_URL);
    } catch {
        crawlStats = { total: 0, loaded: 0, authorsTotal: 0, authorsFailed: 0, rootFailed: true };
        notifyProgress(true);
        return sortAndFilterPublished(partialResolved);
    }
    const authors = authorEntries.filter(entry => entry.type === 'directory' && !entry.name.startsWith('.'));

    // Wave 2 — always fetched live, in full, for every author (not just the
    // current page): each author's own directory listing, with per-file
    // mtimes. This is what builds the flat, cross-author list that drives
    // global sorting and pagination — see the plan doc's "keep the flat
    // feed" decision. It's also the *only* per-author network request
    // indexing ever needs: every stub field (slug, date, tags, category,
    // draft status) is parsed straight out of each article file's own name
    // (see buildStub/parseArticleFileName) — there's no per-article fetch
    // at index time. The author's own bio file (`<author>.md`) and shared
    // `media/` folder live in the same listing and are simply filtered out
    // here — only `.md` files other than the bio file are candidate
    // articles. One author's listing failing (network blip, deleted
    // mid-crawl) is silently skipped rather than failing every other
    // author's list too, but is still counted, so a *total* breakage (wrong
    // origin, CORS) remains visible via getCrawlStats() instead of just
    // silently producing an empty article list.
    let authorsFailed = 0;
    const authorFileLists = await Promise.all(authors.map(async author => {
        const authorRoot = `${ARTICLES_URL}${encodeURIComponent(author.name)}/`;
        const bioFileName = `${author.name}.md`;
        try {
            const entries = await fetchJson(authorRoot);
            return entries
                .filter(entry => entry.type === 'file' && entry.name.endsWith('.md') && entry.name !== bioFileName)
                .map(entry => ({ author: author.name, fileName: entry.name, mtime: normalizeMtime(entry.mtime), authorRoot }));
        } catch {
            authorsFailed += 1;
            return [];
        }
    }));
    const allFiles = authorFileLists.flat();

    // Every stub is parsed synchronously from data already in hand — no
    // per-file fetch, so no need for mapWithConcurrency here (that's still
    // used by hydrateArticles(), which does real network fetches).
    const cacheSnapshot = await cacheSnapshotPromise;
    let loaded = 0;
    for (const file of allFiles) {
        const stub = resolveStub(file.author, file.fileName, file.mtime, file.authorRoot, cacheSnapshot);
        if (stub) {
            loaded += 1;
            partialResolved.push(stub);
        }
    }

    await pruneCache(allFiles.map(file => `${file.authorRoot}${encodeURIComponent(file.fileName)}`));
    crawlStats = { total: allFiles.length, loaded, authorsTotal: authors.length, authorsFailed, rootFailed: false };

    const finalArticles = sortAndFilterPublished(partialResolved);
    notifyProgress(true);
    return finalArticles;
}

// Cached after the first successful crawl; concurrent callers during the
// crawl share the same in-flight promise rather than each starting their
// own duplicate crawl. See onArticlesProgress() for incremental results.
export function discoverArticles() {
    if (articleCache) return Promise.resolve(articleCache);
    if (!crawlPromise) {
        crawlPromise = runCrawl().then(
            result => { articleCache = result; crawlPromise = null; return result; },
            error => { crawlPromise = null; throw error; },
        );
    }
    return crawlPromise;
}

export async function findArticle(hash) {
    const article = (await discoverArticles()).find(candidate => candidate.hash === hash);
    if (!article) throw new Error(t('articleNotFound'));
    return article;
}

export async function fetchAuthorProfile(author) {
    const path = `${ARTICLES_URL}${encodeURIComponent(author)}/${encodeURIComponent(author)}.md`;
    try {
        const response = await fetch(path);
        if (!response.ok) return null;
        const { meta, content } = parseFrontmatter(await response.text());
        return { author, ...meta, content, path };
    } catch { return null; }
}