diff --git a/src/index.js b/src/index.js index 37ec55b..507353a 100644 --- a/src/index.js +++ b/src/index.js @@ -1,5 +1,5 @@ import { Hono } from 'hono' -import { fetchAsCrawler, fetchBest, stripPaywall, extractArticle } from './proxy.js' +import { fetchAsCrawler, fetchBest, fetchFromWayback, stripPaywall, extractArticle } from './proxy.js' import { homePage, readPage, readerPage, errorPage, howPage, aboutPage, bookmarklet } from './views.js' const app = new Hono() @@ -48,10 +48,19 @@ app.get('/reader', async (c) => { return c.redirect(`/proxy?url=${encodeURIComponent(url)}`) } const art = extractArticle(best.html, url) - if (!art.bodyHtml || art.wordCount < 80) { - return c.html(errorPage(url, 'not enough readable text'), 200) + if (art.bodyHtml && art.wordCount >= 80) { + return c.html(readerPage(art, url, HOST(c))) } - return c.html(readerPage(art, url, HOST(c))) + // Live site blocked us — fall back to a Wayback snapshot and read from that. + const wb = await fetchFromWayback(url) + if (wb) { + const art2 = extractArticle(wb.html, url) + if (art2.bodyHtml && art2.wordCount >= 80) { + art2.source = { name: 'Wayback Machine', timestamp: wb.timestamp, url: wb.snapshotUrl } + return c.html(readerPage(art2, url, HOST(c))) + } + } + return c.html(errorPage(url, 'not enough readable text'), 200) } catch (e) { return c.html(errorPage(url, e.message), 200) } diff --git a/src/proxy.js b/src/proxy.js index 7b098a4..05a3a83 100644 --- a/src/proxy.js +++ b/src/proxy.js @@ -64,6 +64,30 @@ export async function fetchBest(url, { ref = 'google', bots = ['google', 'facebo return best || fallback } +// Fall back to the Internet Archive when the live site blocks every crawler +// (FT, WSJ, etc.). Wayback is Worker-friendly, unlike archive.today which +// challenges datacenter IPs. Returns { html, timestamp, snapshotUrl } or null. +export async function fetchFromWayback(url) { + try { + const api = `https://archive.org/wayback/available?url=${encodeURIComponent(url)}` + const r = await fetch(api, { headers: { 'User-Agent': 'PaywallFree/1.0 (+https://paywall.theradicalparty.com)' }, cf: { cacheTtl: 600 } }) + if (!r.ok) return null + const data = await r.json() + const snap = data && data.archived_snapshots && data.archived_snapshots.closest + if (!snap || !snap.available || !snap.url) return null + // `id_` gives the raw archived page without Wayback's toolbar/URL-rewriting — cleaner to extract. + const raw = snap.url.replace(/^http:/, 'https:').replace(/\/web\/(\d+)\//, '/web/$1id_/') + const res = await fetch(raw, { + headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17 Safari/605.1.15' }, + redirect: 'follow', + }) + if (!res.ok) return null + if (!(res.headers.get('content-type') || '').includes('html')) return null + const html = await res.text() + return { html, timestamp: snap.timestamp, snapshotUrl: snap.url.replace(/^http:/, 'https:') } + } catch { return null } +} + // ---- Full-page proxy: keep the layout, drop the paywall ---- const UNLOCK_CSS = ` diff --git a/src/views.js b/src/views.js index 2b81657..c2c551b 100644 --- a/src/views.js +++ b/src/views.js @@ -58,7 +58,9 @@ footer{border-top:1px solid var(--line);margin-top:60px;padding:30px 0;color:var .legal{background:var(--raised);border:1px solid var(--line);border-left:3px solid var(--gold);border-radius:10px;padding:16px 20px;color:var(--muted);font-size:13px;margin:30px 0} .reader{max-width:720px} .reader h1.title{font-family:'Syne',sans-serif;font-size:clamp(28px,4vw,40px);line-height:1.1;letter-spacing:-1px;margin:26px 0 10px} -.reader .meta{color:var(--muted);font-size:14px;font-family:'IBM Plex Mono',monospace;border-bottom:1px solid var(--line);padding-bottom:16px;margin-bottom:8px} +.reader .meta{color:var(--muted);font-size:14px;font-family:'IBM Plex Mono',monospace;padding-bottom:12px} +.reader .srcnote{font-size:13px;color:var(--gold);background:rgba(201,168,76,.08);border:1px solid rgba(201,168,76,.25);border-radius:9px;padding:9px 14px;margin:0 0 8px;border-bottom:1px solid var(--line)} +.reader .srcnote a{color:var(--gold);font-weight:600} .reader .lead{font-size:19px;color:#d8d2c8;margin:18px 0} .reader .content{font-size:18px;line-height:1.75} .reader .content p{margin:0 0 22px} @@ -171,6 +173,7 @@ export function readerPage(art, url, host) { ${art.image ? `` : ''}

${esc(art.title || 'Untitled')}

${art.site ? esc(art.site) : safeHost(url)}${art.author ? ' · ' + esc(art.author) : ''}${art.published ? ' · ' + esc(art.published.slice(0, 10)) : ''}${art.wordCount ? ' · ' + art.wordCount + ' words' : ''}
+ ${art.source ? `
🗄️ Read from a ${esc(art.source.name)} snapshot${art.source.timestamp ? ' archived ' + fmtTs(art.source.timestamp) : ''} — the live site blocked a direct fetch.
` : ''} ${art.description ? `

${esc(art.description)}

` : ''}
${art.bodyHtml || ''}
@@ -256,6 +259,14 @@ export function aboutPage() { export function safeHost(url) { try { return new URL(url).host.replace(/^www\./, '') } catch { return 'this page' } } +// Wayback timestamp YYYYMMDDHHMMSS -> "5 Nov 2024" +export function fmtTs(ts) { + const m = /^(\d{4})(\d{2})(\d{2})/.exec(ts || '') + if (!m) return '' + const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + return `${+m[3]} ${months[+m[2] - 1] || ''} ${m[1]}` +} + export function bookmarklet(host) { const code = `(function(){var u=encodeURIComponent(location.href);location.href='https://${host}/read?url='+u;})();` return 'javascript:' + code