maverick/paywall
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Reject HTTP 4xx/5xx bodies in fetchBest; route dead/blocked pages to Wayback

Browse source 
A live 404/paywall stub with >80 words (e.g. a site's '404 - Page Not Found'
page) was being rendered as the article. fetchBest now ignores error-status
bodies and returns null when every crawler errors, so /reader correctly falls
through to the Wayback snapshot.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
King Omar 2026-07-25 16:02:56 +10:00
parent 88038a7c0e
commit 85c6cb7b61
2 changed files with 14 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/index.js
View file

@ -43,15 +43,17 @@ app.get('/reader', async (c) => {
try { try {
const ref = c.req.query('ref') || 'google' const ref = c.req.query('ref') || 'google'
const best = await fetchBest(url, { ref }) const best = await fetchBest(url, { ref })
if (!best) return c.html(errorPage(url, 'could not reach the page'), 200) // Non-HTML (pdf etc.) that we could reach — hand it to the proxy.
if (!best.html || !best.contentType.includes('html')) { if (best && !best.html && best.contentType && !best.contentType.includes('html')) {
return c.redirect(`/proxy?url=${encodeURIComponent(url)}`) return c.redirect(`/proxy?url=${encodeURIComponent(url)}`)
} }
const art = extractArticle(best.html, url) if (best && best.html) {
if (art.bodyHtml && art.wordCount >= 80) { const art = extractArticle(best.html, url)
return c.html(readerPage(art, url, HOST(c))) if (art.bodyHtml && art.wordCount >= 80) {
return c.html(readerPage(art, url, HOST(c)))
}
} }
// Live site blocked us — fall back to a Wayback snapshot and read from that. // Live site blocked us (or returned only an error/paywall stub) — fall back to a Wayback snapshot.
const wb = await fetchFromWayback(url) const wb = await fetchFromWayback(url)
if (wb) { if (wb) {
const art2 = extractArticle(wb.html, url) const art2 = extractArticle(wb.html, url)

11
src/proxy.js
View file

@ -44,24 +44,25 @@ export async function fetchBest(url, { ref = 'google', bots = ['google', 'facebo
const settled = await Promise.allSettled(bots.map((bot) => fetchAsCrawler(url, { ref, bot }))) const settled = await Promise.allSettled(bots.map((bot) => fetchAsCrawler(url, { ref, bot })))
let best = null let best = null
let bestScore = -1 let bestScore = -1
let fallback = null // first usable response even if low-score (e.g. non-html) let nonHtml = null // first usable non-html (pdf etc.) to hand through
for (let i = 0; i < settled.length; i++) { for (let i = 0; i < settled.length; i++) {
if (settled[i].status !== 'fulfilled') continue if (settled[i].status !== 'fulfilled') continue
const res = settled[i].value const res = settled[i].value
const status = res.status
const ct = res.headers.get('content-type') || '' const ct = res.headers.get('content-type') || ''
if (!ct.includes('html')) { if (!ct.includes('html')) {
if (!fallback) fallback = { html: null, headers: res.headers, contentType: ct, bot: bots[i], response: res } if (status < 400 && !nonHtml) nonHtml = { html: null, headers: res.headers, contentType: ct, bot: bots[i], status, ok: true }
continue continue
} }
let html let html
try { html = await res.text() } catch { continue } try { html = await res.text() } catch { continue }
if (status >= 400) continue // error page (404/401/403 body) — never treat as the article
// Score: paragraph density dominates; length is a mild tie-breaker. // Score: paragraph density dominates; length is a mild tie-breaker.
const paras = (html.match(/<p[ >]/gi) || []).length const paras = (html.match(/<p[ >]/gi) || []).length
const score = paras * 100 + Math.min(html.length / 1000, 400) const score = paras * 100 + Math.min(html.length / 1000, 400)
if (!fallback) fallback = { html, headers: res.headers, contentType: ct, bot: bots[i] } if (score > bestScore) { bestScore = score; best = { html, headers: res.headers, contentType: ct, bot: bots[i], status, ok: true } }
if (score > bestScore) { bestScore = score; best = { html, headers: res.headers, contentType: ct, bot: bots[i] } }
} }
return best || fallback return best || nonHtml // null if every crawler returned an error status
} }
// Fall back to the Internet Archive when the live site blocks every crawler // Fall back to the Internet Archive when the live site blocks every crawler