Wayback fallback: when live site blocks all crawlers, read from a Wayback snapshot

When the multi-crawler fetch can't get article text (FT, WSJ, etc. return
403/JS-only shells to every crawler), fall back to the Internet Archive:
resolve the closest snapshot via the availability API, fetch the raw id_
version, and extract THAT into the Radical Reader with a 'via Wayback Machine,
archived DATE' source note. archive.today can't be used server-side (429 +
captcha on datacenter IPs) so it stays as the user-side button.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/index.js

@@ -1,5 +1,5 @@
 import { Hono } from 'hono'
-import { fetchAsCrawler, fetchBest, stripPaywall, extractArticle } from './proxy.js'
+import { fetchAsCrawler, fetchBest, fetchFromWayback, stripPaywall, extractArticle } from './proxy.js'
 import { homePage, readPage, readerPage, errorPage, howPage, aboutPage, bookmarklet } from './views.js'
 
 const app = new Hono()
@@ -48,10 +48,19 @@ app.get('/reader', async (c) => {
       return c.redirect(`/proxy?url=${encodeURIComponent(url)}`)
     }
     const art = extractArticle(best.html, url)
-    if (!art.bodyHtml || art.wordCount < 80) {
-      return c.html(errorPage(url, 'not enough readable text'), 200)
-    }
+    if (art.bodyHtml && art.wordCount >= 80) {
       return c.html(readerPage(art, url, HOST(c)))
+    }
+    // Live site blocked us — fall back to a Wayback snapshot and read from that.
+    const wb = await fetchFromWayback(url)
+    if (wb) {
+      const art2 = extractArticle(wb.html, url)
+      if (art2.bodyHtml && art2.wordCount >= 80) {
+        art2.source = { name: 'Wayback Machine', timestamp: wb.timestamp, url: wb.snapshotUrl }
+        return c.html(readerPage(art2, url, HOST(c)))
+      }
+    }
+    return c.html(errorPage(url, 'not enough readable text'), 200)
   } catch (e) {
     return c.html(errorPage(url, e.message), 200)
   }

src/proxy.js

@@ -64,6 +64,30 @@ export async function fetchBest(url, { ref = 'google', bots = ['google', 'facebo
   return best || fallback
 }
 
+// Fall back to the Internet Archive when the live site blocks every crawler
+// (FT, WSJ, etc.). Wayback is Worker-friendly, unlike archive.today which
+// challenges datacenter IPs. Returns { html, timestamp, snapshotUrl } or null.
+export async function fetchFromWayback(url) {
+  try {
+    const api = `https://archive.org/wayback/available?url=${encodeURIComponent(url)}`
+    const r = await fetch(api, { headers: { 'User-Agent': 'PaywallFree/1.0 (+https://paywall.theradicalparty.com)' }, cf: { cacheTtl: 600 } })
+    if (!r.ok) return null
+    const data = await r.json()
+    const snap = data && data.archived_snapshots && data.archived_snapshots.closest
+    if (!snap || !snap.available || !snap.url) return null
+    // `id_` gives the raw archived page without Wayback's toolbar/URL-rewriting — cleaner to extract.
+    const raw = snap.url.replace(/^http:/, 'https:').replace(/\/web\/(\d+)\//, '/web/$1id_/')
+    const res = await fetch(raw, {
+      headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17 Safari/605.1.15' },
+      redirect: 'follow',
+    })
+    if (!res.ok) return null
+    if (!(res.headers.get('content-type') || '').includes('html')) return null
+    const html = await res.text()
+    return { html, timestamp: snap.timestamp, snapshotUrl: snap.url.replace(/^http:/, 'https:') }
+  } catch { return null }
+}
+
 // ---- Full-page proxy: keep the layout, drop the paywall ----
 
 const UNLOCK_CSS = `

src/views.js

@@ -58,7 +58,9 @@ footer{border-top:1px solid var(--line);margin-top:60px;padding:30px 0;color:var
 .legal{background:var(--raised);border:1px solid var(--line);border-left:3px solid var(--gold);border-radius:10px;padding:16px 20px;color:var(--muted);font-size:13px;margin:30px 0}
 .reader{max-width:720px}
 .reader h1.title{font-family:'Syne',sans-serif;font-size:clamp(28px,4vw,40px);line-height:1.1;letter-spacing:-1px;margin:26px 0 10px}
-.reader .meta{color:var(--muted);font-size:14px;font-family:'IBM Plex Mono',monospace;border-bottom:1px solid var(--line);padding-bottom:16px;margin-bottom:8px}
+.reader .meta{color:var(--muted);font-size:14px;font-family:'IBM Plex Mono',monospace;padding-bottom:12px}
+.reader .srcnote{font-size:13px;color:var(--gold);background:rgba(201,168,76,.08);border:1px solid rgba(201,168,76,.25);border-radius:9px;padding:9px 14px;margin:0 0 8px;border-bottom:1px solid var(--line)}
+.reader .srcnote a{color:var(--gold);font-weight:600}
 .reader .lead{font-size:19px;color:#d8d2c8;margin:18px 0}
 .reader .content{font-size:18px;line-height:1.75}
 .reader .content p{margin:0 0 22px}
@@ -171,6 +173,7 @@ export function readerPage(art, url, host) {
     ${art.image ? `<img class="heroimg" src="${esc(art.image)}" alt="" loading="lazy">` : ''}
     <h1 class="title">${esc(art.title || 'Untitled')}</h1>
     <div class="meta">${art.site ? esc(art.site) : safeHost(url)}${art.author ? ' · ' + esc(art.author) : ''}${art.published ? ' · ' + esc(art.published.slice(0, 10)) : ''}${art.wordCount ? ' · ' + art.wordCount + ' words' : ''}</div>
+    ${art.source ? `<div class="srcnote">🗄️ Read from a <a href="${esc(art.source.url)}" target="_blank" rel="noopener noreferrer">${esc(art.source.name)} snapshot</a>${art.source.timestamp ? ' archived ' + fmtTs(art.source.timestamp) : ''} — the live site blocked a direct fetch.</div>` : ''}
     ${art.description ? `<p class="lead">${esc(art.description)}</p>` : ''}
     <div class="content">${art.bodyHtml || ''}</div>
     <div class="toolbar" style="margin-top:36px">
@@ -256,6 +259,14 @@ export function aboutPage() {
 
 export function safeHost(url) { try { return new URL(url).host.replace(/^www\./, '') } catch { return 'this page' } }
 
+// Wayback timestamp YYYYMMDDHHMMSS -> "5 Nov 2024"
+export function fmtTs(ts) {
+  const m = /^(\d{4})(\d{2})(\d{2})/.exec(ts || '')
+  if (!m) return ''
+  const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
+  return `${+m[3]} ${months[+m[2] - 1] || ''} ${m[1]}`
+}
+
 export function bookmarklet(host) {
   const code = `(function(){var u=encodeURIComponent(location.href);location.href='https://${host}/read?url='+u;})();`
   return 'javascript:' + code