b0496a5ac1
The crawl stalled when two crawler processes briefly overlapped on the same WAL SQLite frontier (node:sqlite is synchronous, so a blocked write lock froze the event loop). Fixes: - run the crawler as a single-instance systemd service (search-crawler.service) with CPUQuota=150% + MemoryMax=1200M so it can't starve a shared VM - add timeouts to all Meilisearch calls + resilient flush (an untimed hang was the amplifier that wedged every worker) - crawl-diskguard.sh + cron: auto-stop the crawl and ntfy-alert if disk < 800MB Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
33 lines
874 B
Desktop File
33 lines
874 B
Desktop File
[Unit]
|
|
Description=RADICAL_SEARCH crawler
|
|
After=network.target meilisearch.service
|
|
Requires=meilisearch.service
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=root
|
|
WorkingDirectory=/opt/search-crawler
|
|
ExecStart=/usr/bin/node --experimental-sqlite src/index.js
|
|
# Batch job: exits 0 when the crawl finishes; only restart on crash.
|
|
Restart=on-failure
|
|
RestartSec=15
|
|
|
|
# Guardrails so a runaway crawl can't starve other services on a shared box.
|
|
CPUQuota=150%
|
|
MemoryMax=1200M
|
|
MemoryHigh=1000M
|
|
|
|
Environment=LIMIT=150000
|
|
Environment=SEED_DOMAINS=15000
|
|
Environment=CONCURRENCY=20
|
|
Environment=MAX_DEPTH=2
|
|
Environment=MAX_PAGES_PER_DOMAIN=25
|
|
Environment=HOST_DELAY_MS=600
|
|
Environment=FETCH_TIMEOUT_MS=5000
|
|
Environment=BODY_CHARS=2500
|
|
Environment=FRONTIER_DB=/opt/search-crawler/frontier.db
|
|
Environment=MEILI_URL=http://localhost:7700
|
|
Environment=MEILI_KEY=masterKey
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|