From 057f6673cbc6f0920a8ad28a48766719452a3ce2 Mon Sep 17 00:00:00 2001
From: biosias <jakub@gdos.sk>
Date: Tue, 22 Sep 2026 16:05:54 +0200
Subject: [PATCH] Cap concurrent per-author listing fetches during indexing
 (AI)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Wave 2 of the crawl fetched every author's directory listing via a
plain Promise.all, with no limit — a site with hundreds of authors
fired one simultaneous request per author on every cold-cache load.
Routed through mapWithConcurrency/MAX_CONCURRENT_REQUESTS, the same
cap hydrateArticles() already respects for content fetches.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 js/discovery.js | 37 +++++++++++++++++++++----------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/js/discovery.js b/js/discovery.js
index e879fbe..a7ebc2f 100644
--- a/js/discovery.js
+++ b/js/discovery.js
@@ -4,13 +4,16 @@ import { mapWithConcurrency } from './concurrency.js';
 import { getAllCacheEntries, setCacheEntry, pruneCache } from './cache.js';
 import { t } from './i18n.js';
 
-// Caps simultaneous markdown content fetches during hydration (see
-// hydrateArticle()/hydrateArticles()) — the only per-article requests left,
-// since indexing no longer needs a per-article listing fetch at all (every
-// stub field is encoded in its file's own name — see buildStub()). Raise it
-// for a fast/local nginx, lower it for a slow or shared/production instance
-// — see .claude/nginx-sitemap-crawler-plan.md §2a for the original rationale
-// (written when this also governed indexing requests).
+// Caps simultaneous requests in the two places this module still fires one
+// per item: markdown content fetches during hydration (see
+// hydrateArticle()/hydrateArticles()) and each author's own directory
+// listing during wave 2 of indexing (see runCrawl()) — indexing no longer
+// needs a per-*article* fetch at all (every stub field is encoded in its
+// file's own name — see buildStub()), but still fetches one listing per
+// author. Raise it for a fast/local nginx, lower it for a slow or shared/
+// production instance — see .claude/nginx-sitemap-crawler-plan.md §2a for
+// the original rationale (written when this also governed per-article
+// indexing requests).
 const MAX_CONCURRENT_REQUESTS = 10;
 
 const MONTH_ABBREVIATIONS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
@@ -333,13 +336,15 @@ async function runCrawl() {
 
     // Wave 2 — always fetched live, in full, for every author (not just the
     // current page): each author's own directory listing, with per-file
-    // mtimes. This is what builds the flat, cross-author list that drives
-    // global sorting and pagination — see the plan doc's "keep the flat
-    // feed" decision. It's also the *only* per-author network request
-    // indexing ever needs: every stub field (slug, date, tags) is parsed
-    // straight out of each article file's own name (see
-    // buildStub/parseArticleFileName) — there's no per-article fetch at
-    // index time. The `type === 'file'` filter below also means any
+    // mtimes. Capped by MAX_CONCURRENT_REQUESTS like hydrateArticles() below
+    // — a site with many authors would otherwise fire one listing request
+    // per author simultaneously, with no limit. This is what builds the
+    // flat, cross-author list that drives global sorting and pagination —
+    // see the plan doc's "keep the flat feed" decision. It's also the *only*
+    // per-author network request indexing ever needs: every stub field
+    // (slug, date, tags) is parsed straight out of each article file's own
+    // name (see buildStub/parseArticleFileName) — there's no per-article
+    // fetch at index time. The `type === 'file'` filter below also means any
     // subdirectory (the author's shared `media/` folder, a `draft/` folder —
     // see "Adding an article" in the README) is skipped without ever being
     // descended into. The author's own bio file (`<author>.md`) is filtered
@@ -350,7 +355,7 @@ async function runCrawl() {
     // origin, CORS) remains visible via getCrawlStats() instead of just
     // silently producing an empty article list.
     let authorsFailed = 0;
-    const authorFileLists = await Promise.all(authors.map(async author => {
+    const authorFileLists = await mapWithConcurrency(authors, MAX_CONCURRENT_REQUESTS, async author => {
         const authorRoot = `${ARTICLES_URL}${encodeURIComponent(author.name)}/`;
         const bioFileName = `${author.name}.md`;
         try {
@@ -362,7 +367,7 @@ async function runCrawl() {
             authorsFailed += 1;
             return [];
         }
-    }));
+    });
     const allFiles = authorFileLists.flat();
 
     // Every stub is parsed synchronously from data already in hand — no