From 538f7a55bc7bdeada6dee8b8fec306e1c6007532 Mon Sep 17 00:00:00 2001 From: King Omar Date: Wed, 22 Jul 2026 13:22:36 +1000 Subject: [PATCH] Fix crawl stall: host-diverse claim (ORDER BY RANDOM) Per-host politeness serialized entire same-host batches (contiguous internal links at depth), collapsing throughput to ~1 request in flight. Randomize the frontier claim so each batch spans many hosts. Co-Authored-By: Claude Opus 4.8 (1M context) --- crawler/src/frontier.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crawler/src/frontier.js b/crawler/src/frontier.js index 05709b3..901ca9d 100644 --- a/crawler/src/frontier.js +++ b/crawler/src/frontier.js @@ -22,7 +22,9 @@ db.exec(` const q = { add: db.prepare('INSERT OR IGNORE INTO frontier(url,domain,depth,rank) VALUES(?,?,?,?)'), - next: db.prepare('SELECT url,domain,depth,rank FROM frontier WHERE status=0 LIMIT ?'), + // Randomize the claim so a batch spans many hosts — otherwise contiguous + // same-host internal links get serialized by per-host politeness (stall). + next: db.prepare('SELECT url,domain,depth,rank FROM frontier WHERE status=0 ORDER BY RANDOM() LIMIT ?'), claim: db.prepare('UPDATE frontier SET status=3 WHERE url=?'), done: db.prepare('UPDATE frontier SET status=1 WHERE url=?'), fail: db.prepare('UPDATE frontier SET status=2 WHERE url=?'),