From b0496a5ac121ba3aba005d97a67c80106d7349c4 Mon Sep 17 00:00:00 2001
From: King Omar <omar@bored.investments>
Date: Wed, 22 Jul 2026 02:51:03 +1000
Subject: [PATCH] Run crawler as guarded systemd service; fix stall

The crawl stalled when two crawler processes briefly overlapped on the same
WAL SQLite frontier (node:sqlite is synchronous, so a blocked write lock froze
the event loop). Fixes:
- run the crawler as a single-instance systemd service (search-crawler.service)
  with CPUQuota=150% + MemoryMax=1200M so it can't starve a shared VM
- add timeouts to all Meilisearch calls + resilient flush (an untimed hang was
  the amplifier that wedged every worker)
- crawl-diskguard.sh + cron: auto-stop the crawl and ntfy-alert if disk < 800MB

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 crawler/src/fetcher.js         |  2 +-
 crawler/src/indexer.js         |  9 ++++++++-
 scripts/crawl-diskguard.sh     | 10 ++++++++++
 scripts/search-crawler.service | 33 +++++++++++++++++++++++++++++++++
 4 files changed, 52 insertions(+), 2 deletions(-)
 create mode 100644 scripts/crawl-diskguard.sh
 create mode 100644 scripts/search-crawler.service

diff --git a/crawler/src/fetcher.js b/crawler/src/fetcher.js
index a9e4c9b..5a04c38 100644
--- a/crawler/src/fetcher.js
+++ b/crawler/src/fetcher.js
@@ -1,6 +1,6 @@
 import { fetch } from 'undici'
 
-const TIMEOUT_MS = 8000
+const TIMEOUT_MS = parseInt(process.env.FETCH_TIMEOUT_MS || '6000')
 const MAX_BODY_BYTES = 500_000
 
 const HEADERS = {
diff --git a/crawler/src/indexer.js b/crawler/src/indexer.js
index 0837b50..ce0daca 100644
--- a/crawler/src/indexer.js
+++ b/crawler/src/indexer.js
@@ -6,10 +6,12 @@ const BATCH_SIZE = 200
 let buffer = []
 
 async function meili(method, path, body) {
+  // Always time-bound: an untimed hang here would freeze every crawl worker.
   const res = await fetch(`${MEILI_URL}${path}`, {
     method,
     headers: { 'Authorization': `Bearer ${MEILI_KEY}`, 'Content-Type': 'application/json' },
     body: body ? JSON.stringify(body) : undefined,
+    signal: AbortSignal.timeout(20000),
   })
   return res.json()
 }
@@ -48,7 +50,12 @@ export async function indexDoc(doc) {
 export async function flush() {
   if (buffer.length === 0) return
   const docs = buffer.splice(0)
-  await meili('POST', `/indexes/${INDEX}/documents`, docs)
+  try {
+    await meili('POST', `/indexes/${INDEX}/documents`, docs)
+  } catch (e) {
+    // Never let an indexing hiccup wedge the crawl; drop the batch and continue.
+    console.error(`\nflush failed (${docs.length} docs dropped): ${e.message}`)
+  }
 }
 
 export { meili, baseScore }
diff --git a/scripts/crawl-diskguard.sh b/scripts/crawl-diskguard.sh
new file mode 100644
index 0000000..7890bda
--- /dev/null
+++ b/scripts/crawl-diskguard.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+# Autonomous safety: stop the crawler if disk gets low, so it can never fill
+# the shared VM and break other services. Runs from cron every 5 min.
+THRESHOLD_KB=800000   # 800 MB
+FREE=$(df --output=avail / | tail -1 | tr -d ' ')
+if [ "$FREE" -lt "$THRESHOLD_KB" ]; then
+  systemctl stop search-crawler
+  curl -s -d "RADICAL_SEARCH crawler STOPPED on adrian-bingo: disk free ${FREE}KB < ${THRESHOLD_KB}KB" \
+    ntfy.sh/radicalparty-vm-alerts-x7k2q9 >/dev/null 2>&1
+fi
diff --git a/scripts/search-crawler.service b/scripts/search-crawler.service
new file mode 100644
index 0000000..1a1e738
--- /dev/null
+++ b/scripts/search-crawler.service
@@ -0,0 +1,33 @@
+[Unit]
+Description=RADICAL_SEARCH crawler
+After=network.target meilisearch.service
+Requires=meilisearch.service
+
+[Service]
+Type=simple
+User=root
+WorkingDirectory=/opt/search-crawler
+ExecStart=/usr/bin/node --experimental-sqlite src/index.js
+# Batch job: exits 0 when the crawl finishes; only restart on crash.
+Restart=on-failure
+RestartSec=15
+
+# Guardrails so a runaway crawl can't starve other services on a shared box.
+CPUQuota=150%
+MemoryMax=1200M
+MemoryHigh=1000M
+
+Environment=LIMIT=150000
+Environment=SEED_DOMAINS=15000
+Environment=CONCURRENCY=20
+Environment=MAX_DEPTH=2
+Environment=MAX_PAGES_PER_DOMAIN=25
+Environment=HOST_DELAY_MS=600
+Environment=FETCH_TIMEOUT_MS=5000
+Environment=BODY_CHARS=2500
+Environment=FRONTIER_DB=/opt/search-crawler/frontier.db
+Environment=MEILI_URL=http://localhost:7700
+Environment=MEILI_KEY=masterKey
+
+[Install]
+WantedBy=multi-user.target