From 85c6cb7b61173c95d384b9f43c1f11a3385d2d95 Mon Sep 17 00:00:00 2001
From: King Omar <omar@bored.investments>
Date: Sat, 25 Jul 2026 16:02:56 +1000
Subject: [PATCH] Reject HTTP 4xx/5xx bodies in fetchBest; route dead/blocked
 pages to Wayback

A live 404/paywall stub with >80 words (e.g. a site's '404 - Page Not Found'
page) was being rendered as the article. fetchBest now ignores error-status
bodies and returns null when every crawler errors, so /reader correctly falls
through to the Wayback snapshot.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 src/index.js | 14 ++++++++------
 src/proxy.js | 11 ++++++-----
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/index.js b/src/index.js
index 507353a..db551db 100644
--- a/src/index.js
+++ b/src/index.js
@@ -43,15 +43,17 @@ app.get('/reader', async (c) => {
   try {
     const ref = c.req.query('ref') || 'google'
     const best = await fetchBest(url, { ref })
-    if (!best) return c.html(errorPage(url, 'could not reach the page'), 200)
-    if (!best.html || !best.contentType.includes('html')) {
+    // Non-HTML (pdf etc.) that we could reach — hand it to the proxy.
+    if (best && !best.html && best.contentType && !best.contentType.includes('html')) {
       return c.redirect(`/proxy?url=${encodeURIComponent(url)}`)
     }
-    const art = extractArticle(best.html, url)
-    if (art.bodyHtml && art.wordCount >= 80) {
-      return c.html(readerPage(art, url, HOST(c)))
+    if (best && best.html) {
+      const art = extractArticle(best.html, url)
+      if (art.bodyHtml && art.wordCount >= 80) {
+        return c.html(readerPage(art, url, HOST(c)))
+      }
     }
-    // Live site blocked us — fall back to a Wayback snapshot and read from that.
+    // Live site blocked us (or returned only an error/paywall stub) — fall back to a Wayback snapshot.
     const wb = await fetchFromWayback(url)
     if (wb) {
       const art2 = extractArticle(wb.html, url)
diff --git a/src/proxy.js b/src/proxy.js
index 05a3a83..5e0627b 100644
--- a/src/proxy.js
+++ b/src/proxy.js
@@ -44,24 +44,25 @@ export async function fetchBest(url, { ref = 'google', bots = ['google', 'facebo
   const settled = await Promise.allSettled(bots.map((bot) => fetchAsCrawler(url, { ref, bot })))
   let best = null
   let bestScore = -1
-  let fallback = null // first usable response even if low-score (e.g. non-html)
+  let nonHtml = null // first usable non-html (pdf etc.) to hand through
   for (let i = 0; i < settled.length; i++) {
     if (settled[i].status !== 'fulfilled') continue
     const res = settled[i].value
+    const status = res.status
     const ct = res.headers.get('content-type') || ''
     if (!ct.includes('html')) {
-      if (!fallback) fallback = { html: null, headers: res.headers, contentType: ct, bot: bots[i], response: res }
+      if (status < 400 && !nonHtml) nonHtml = { html: null, headers: res.headers, contentType: ct, bot: bots[i], status, ok: true }
       continue
     }
     let html
     try { html = await res.text() } catch { continue }
+    if (status >= 400) continue // error page (404/401/403 body) — never treat as the article
     // Score: paragraph density dominates; length is a mild tie-breaker.
     const paras = (html.match(/<p[ >]/gi) || []).length
     const score = paras * 100 + Math.min(html.length / 1000, 400)
-    if (!fallback) fallback = { html, headers: res.headers, contentType: ct, bot: bots[i] }
-    if (score > bestScore) { bestScore = score; best = { html, headers: res.headers, contentType: ct, bot: bots[i] } }
+    if (score > bestScore) { bestScore = score; best = { html, headers: res.headers, contentType: ct, bot: bots[i], status, ok: true } }
   }
-  return best || fallback
+  return best || nonHtml // null if every crawler returned an error status
 }
 
 // Fall back to the Internet Archive when the live site blocks every crawler