From 7fa8114a7fc7238efebd0cd078a71d45623060a7 Mon Sep 17 00:00:00 2001 From: King Omar Date: Wed, 22 Jul 2026 23:17:35 +1000 Subject: [PATCH] Crawler: shard-aware seeding for distributed crawling Add SHARD_ID/SHARD_COUNT env: each worker seeds only domains where hash(domain) % SHARD_COUNT === SHARD_ID (FNV-1a). Defaults to single-node (all domains). Crawler only follows same-domain links, so shards never overlap and lose no coverage vs a single crawler. Co-Authored-By: Claude Opus 4.8 (1M context) --- crawler/src/index.js | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/crawler/src/index.js b/crawler/src/index.js index 9c1c8f7..7412878 100644 --- a/crawler/src/index.js +++ b/crawler/src/index.js @@ -14,6 +14,18 @@ const MAX_DEPTH = parseInt(process.env.MAX_DEPTH || '2') // link const MAX_PAGES_PER_DOMAIN = parseInt(process.env.MAX_PAGES_PER_DOMAIN || '30') const HOST_DELAY_MS = parseInt(process.env.HOST_DELAY_MS || '1000') // politeness per host const FRESH = process.env.FRESH === '1' +// Distributed crawling: each worker owns a disjoint slice of the seed domains +// (hash(domain) % SHARD_COUNT === SHARD_ID). Defaults = single-node (all domains). +const SHARD_ID = parseInt(process.env.SHARD_ID || '0') +const SHARD_COUNT = parseInt(process.env.SHARD_COUNT || '1') + +// FNV-1a — cheap, stable hash so every worker agrees on who owns a domain. +function hashDomain(s) { + let h = 0x811c9dc5 + for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 0x01000193) } + return h >>> 0 +} +const ownsDomain = domain => SHARD_COUNT <= 1 || hashDomain(domain) % SHARD_COUNT === SHARD_ID const sleep = ms => new Promise(r => setTimeout(r, ms)) const lastHit = new Map() // host -> last fetch ts (politeness) @@ -24,12 +36,13 @@ function stats() { } function seed() { - let n = 0 + let n = 0, skipped = 0 for (const { rank, domain } of readDomains(SEED_DOMAINS)) { + if (!ownsDomain(regDomain(domain))) { skipped++; continue } F.enqueue(`https://${domain}`, regDomain(domain), 0, rank) n++ } - console.log(`Seeded ${n} domains`) + console.log(`Seeded ${n} domains (shard ${SHARD_ID}/${SHARD_COUNT}, skipped ${skipped} out-of-shard)`) } async function politeWait(host) {