diff --git a/src/index.js b/src/index.js
index 37ec55b..507353a 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,5 +1,5 @@
import { Hono } from 'hono'
-import { fetchAsCrawler, fetchBest, stripPaywall, extractArticle } from './proxy.js'
+import { fetchAsCrawler, fetchBest, fetchFromWayback, stripPaywall, extractArticle } from './proxy.js'
import { homePage, readPage, readerPage, errorPage, howPage, aboutPage, bookmarklet } from './views.js'
const app = new Hono()
@@ -48,10 +48,19 @@ app.get('/reader', async (c) => {
return c.redirect(`/proxy?url=${encodeURIComponent(url)}`)
}
const art = extractArticle(best.html, url)
- if (!art.bodyHtml || art.wordCount < 80) {
- return c.html(errorPage(url, 'not enough readable text'), 200)
+ if (art.bodyHtml && art.wordCount >= 80) {
+ return c.html(readerPage(art, url, HOST(c)))
}
- return c.html(readerPage(art, url, HOST(c)))
+ // Live site blocked us — fall back to a Wayback snapshot and read from that.
+ const wb = await fetchFromWayback(url)
+ if (wb) {
+ const art2 = extractArticle(wb.html, url)
+ if (art2.bodyHtml && art2.wordCount >= 80) {
+ art2.source = { name: 'Wayback Machine', timestamp: wb.timestamp, url: wb.snapshotUrl }
+ return c.html(readerPage(art2, url, HOST(c)))
+ }
+ }
+ return c.html(errorPage(url, 'not enough readable text'), 200)
} catch (e) {
return c.html(errorPage(url, e.message), 200)
}
diff --git a/src/proxy.js b/src/proxy.js
index 7b098a4..05a3a83 100644
--- a/src/proxy.js
+++ b/src/proxy.js
@@ -64,6 +64,30 @@ export async function fetchBest(url, { ref = 'google', bots = ['google', 'facebo
return best || fallback
}
+// Fall back to the Internet Archive when the live site blocks every crawler
+// (FT, WSJ, etc.). Wayback is Worker-friendly, unlike archive.today which
+// challenges datacenter IPs. Returns { html, timestamp, snapshotUrl } or null.
+export async function fetchFromWayback(url) {
+ try {
+ const api = `https://archive.org/wayback/available?url=${encodeURIComponent(url)}`
+ const r = await fetch(api, { headers: { 'User-Agent': 'PaywallFree/1.0 (+https://paywall.theradicalparty.com)' }, cf: { cacheTtl: 600 } })
+ if (!r.ok) return null
+ const data = await r.json()
+ const snap = data && data.archived_snapshots && data.archived_snapshots.closest
+ if (!snap || !snap.available || !snap.url) return null
+ // `id_` gives the raw archived page without Wayback's toolbar/URL-rewriting — cleaner to extract.
+ const raw = snap.url.replace(/^http:/, 'https:').replace(/\/web\/(\d+)\//, '/web/$1id_/')
+ const res = await fetch(raw, {
+ headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17 Safari/605.1.15' },
+ redirect: 'follow',
+ })
+ if (!res.ok) return null
+ if (!(res.headers.get('content-type') || '').includes('html')) return null
+ const html = await res.text()
+ return { html, timestamp: snap.timestamp, snapshotUrl: snap.url.replace(/^http:/, 'https:') }
+ } catch { return null }
+}
+
// ---- Full-page proxy: keep the layout, drop the paywall ----
const UNLOCK_CSS = `
diff --git a/src/views.js b/src/views.js
index 2b81657..c2c551b 100644
--- a/src/views.js
+++ b/src/views.js
@@ -58,7 +58,9 @@ footer{border-top:1px solid var(--line);margin-top:60px;padding:30px 0;color:var
.legal{background:var(--raised);border:1px solid var(--line);border-left:3px solid var(--gold);border-radius:10px;padding:16px 20px;color:var(--muted);font-size:13px;margin:30px 0}
.reader{max-width:720px}
.reader h1.title{font-family:'Syne',sans-serif;font-size:clamp(28px,4vw,40px);line-height:1.1;letter-spacing:-1px;margin:26px 0 10px}
-.reader .meta{color:var(--muted);font-size:14px;font-family:'IBM Plex Mono',monospace;border-bottom:1px solid var(--line);padding-bottom:16px;margin-bottom:8px}
+.reader .meta{color:var(--muted);font-size:14px;font-family:'IBM Plex Mono',monospace;padding-bottom:12px}
+.reader .srcnote{font-size:13px;color:var(--gold);background:rgba(201,168,76,.08);border:1px solid rgba(201,168,76,.25);border-radius:9px;padding:9px 14px;margin:0 0 8px;border-bottom:1px solid var(--line)}
+.reader .srcnote a{color:var(--gold);font-weight:600}
.reader .lead{font-size:19px;color:#d8d2c8;margin:18px 0}
.reader .content{font-size:18px;line-height:1.75}
.reader .content p{margin:0 0 22px}
@@ -171,6 +173,7 @@ export function readerPage(art, url, host) {
${art.image ? `` : ''}
${esc(art.description)}
` : ''}