maverick/paywall
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Multi-crawler fetch: try Googlebot + social crawlers, keep richest copy

Browse source 
Hard paywalls (AFR etc.) block Googlebot by IP but serve full articles to
facebookexternalhit/Twitterbot for link previews. fetchBest() now fetches
several crawler identities concurrently and keeps the copy with the most text,
so the Reader works on AFR (0 -> 2256 words). Also rank archives first for
known hard-paywall domains.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
King Omar 2026-07-25 15:21:36 +10:00
parent c3103e6ecb
commit 665cd264cc
3 changed files with 72 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
src/index.js
View file

@ -1,5 +1,5 @@
import { Hono } from 'hono'
import { fetchAsCrawler, stripPaywall, extractArticle } from './proxy.js'
import { fetchAsCrawler, fetchBest, stripPaywall, extractArticle } from './proxy.js'
import { homePage, readPage, readerPage, errorPage, howPage, aboutPage, bookmarklet } from './views.js'
const app = new Hono()
@ -36,22 +36,18 @@ app.get('/read', (c) => {
return c.html(readPage(url, HOST(c)))
})
// Clean reader: fetch as crawler, extract article text.
// Clean reader: fetch as several crawlers, keep the richest copy, extract article text.
app.get('/reader', async (c) => {
const url = normalizeUrl(c.req.query('url'))
if (!url) return c.redirect('/')
try {
const res = await fetchAsCrawler(url, { ref: c.req.query('ref') || 'google' })
const ct = res.headers.get('content-type') || ''
if (!res.ok && res.status !== 403 && res.status !== 401) {
return c.html(errorPage(url, `origin returned ${res.status}`), 200)
}
if (!ct.includes('html')) {
// non-HTML (pdf, etc.) — just hand it through
const ref = c.req.query('ref') || 'google'
const best = await fetchBest(url, { ref })
if (!best) return c.html(errorPage(url, 'could not reach the page'), 200)
if (!best.html || !best.contentType.includes('html')) {
return c.redirect(`/proxy?url=${encodeURIComponent(url)}`)
}
const html = await res.text()
const art = extractArticle(html, url)
const art = extractArticle(best.html, url)
if (!art.bodyHtml || art.wordCount < 80) {
return c.html(errorPage(url, 'not enough readable text'), 200)
}
@ -61,20 +57,24 @@ app.get('/reader', async (c) => {
}
})
// Full-page proxy: fetch as crawler, strip paywall scripts/overlays, keep layout.
// Full-page proxy: fetch as several crawlers, keep the richest copy, strip the paywall.
app.get('/proxy', async (c) => {
const url = normalizeUrl(c.req.query('url'))
if (!url) return c.redirect('/')
try {
const res = await fetchAsCrawler(url, { ref: c.req.query('ref') || 'google' })
const ct = res.headers.get('content-type') || ''
if (!ct.includes('html')) {
// pass binary/other content straight through
const h = new Headers(res.headers)
const ref = c.req.query('ref') || 'google'
// ref=google means "arrive from Google" — honour it with a single Googlebot fetch.
const best = c.req.query('ref') === 'google' && c.req.query('single')
? await fetchBest(url, { ref, bots: ['google'] })
: await fetchBest(url, { ref })
if (!best) return c.html(errorPage(url, 'could not reach the page'), 200)
if (!best.html || !best.contentType.includes('html')) {
const single = await fetchAsCrawler(url, { ref, bot: best.bot || 'google' })
const h = new Headers(single.headers)
h.delete('content-security-policy'); h.delete('x-frame-options')
return new Response(res.body, { status: res.status, headers: h })
return new Response(single.body, { status: single.status, headers: h })
}
return stripPaywall(res, url)
return stripPaywall(best.html, best.headers, url)
} catch (e) {
return c.html(errorPage(url, e.message), 200)
}

15
src/methods.js
View file

@ -100,14 +100,27 @@ export const METHODS = [
},
]
// Hard paywalls that block Googlebot / don't do the SEO soft-paywall trick — for
// these, archive snapshots are the most reliable route, so bubble them up.
const HARD_PAYWALLS = [
'afr.com', 'wsj.com', 'ft.com', 'bloomberg.com', 'nytimes.com', 'economist.com',
'theaustralian.com.au', 'thetimes.co.uk', 'telegraph.co.uk', 'washingtonpost.com',
'newyorker.com', 'wired.com', 'businessinsider.com', 'seekingalpha.com',
'barrons.com', 'foreignpolicy.com', 'thetimes.com', 'nzherald.co.nz',
]
// Order methods so domain-specific + likely-to-work ones bubble to the top.
export function rankMethods(url) {
let host = ''
try { host = new URL(url).host.replace(/^www\./, '') } catch {}
const isHard = HARD_PAYWALLS.some((d) => host === d || host.endsWith('.' + d))
const scored = METHODS.map((m) => {
let score = 0
if (m.domains && m.domains.some((d) => host === d || host.endsWith('.' + d))) score += 100
if (m.kind === 'internal') score += 10
if (m.kind === 'internal') score += 60
// For hard paywalls, snapshots beat our crawler fetch — lead with them.
if (isHard && m.id === 'archive-ph') score += 70
if (isHard && m.id === 'wayback') score += 65
return { m, score }
})
// stable sort by score desc, preserving original order for ties

46
src/proxy.js
View file

@ -7,6 +7,8 @@
const CRAWLER_UAS = {
google: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
bing: 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
// Many hard paywalls (AFR, some News Corp titles) block Googlebot by IP but still
// serve the FULL article to social link-preview crawlers. Try these too.
facebook: 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
twitter: 'Twitterbot/1.0',
}
@ -34,6 +36,34 @@ export async function fetchAsCrawler(url, { ref = 'google', bot = 'google' } = {
return res
}
// Fetch the URL as several crawler identities at once and keep the richest HTML.
// Returns { html, headers, contentType, bot } or null on total failure.
// `bots` defaults to a search-crawler + two social crawlers — the combination that
// covers both SEO soft-paywalls (Googlebot) and social-only gates (AFR etc.).
export async function fetchBest(url, { ref = 'google', bots = ['google', 'facebook', 'bing'] } = {}) {
const settled = await Promise.allSettled(bots.map((bot) => fetchAsCrawler(url, { ref, bot })))
let best = null
let bestScore = -1
let fallback = null // first usable response even if low-score (e.g. non-html)
for (let i = 0; i < settled.length; i++) {
if (settled[i].status !== 'fulfilled') continue
const res = settled[i].value
const ct = res.headers.get('content-type') || ''
if (!ct.includes('html')) {
if (!fallback) fallback = { html: null, headers: res.headers, contentType: ct, bot: bots[i], response: res }
continue
}
let html
try { html = await res.text() } catch { continue }
// Score: paragraph density dominates; length is a mild tie-breaker.
const paras = (html.match(/<p[ >]/gi) || []).length
const score = paras * 100 + Math.min(html.length / 1000, 400)
if (!fallback) fallback = { html, headers: res.headers, contentType: ct, bot: bots[i] }
if (score > bestScore) { bestScore = score; best = { html, headers: res.headers, contentType: ct, bot: bots[i] } }
}
return best || fallback
}
// ---- Full-page proxy: keep the layout, drop the paywall ----
const UNLOCK_CSS = `
@ -72,7 +102,8 @@ const UNLOCK_JS = `(function(){
} catch(e){}
})();`
export function stripPaywall(res, url) {
export function stripPaywall(html, headers, url) {
const res = new Response(html, { headers })
const banner = `<div style="position:sticky;top:0;z-index:2147483647;background:#0B0B09;color:#F0EDE6;font:600 13px/1.4 system-ui,sans-serif;padding:8px 14px;border-bottom:2px solid #FF2D55;display:flex;gap:12px;align-items:center;justify-content:center">
<span style="color:#FF2D55">RADICAL</span> paywall bypass · <a href="/read?url=${encodeURIComponent(url)}" style="color:#FF6B82">try another method</a> · <a href="/reader?url=${encodeURIComponent(url)}" style="color:#FF6B82">reader view</a>
</div>`
@ -96,12 +127,13 @@ export function stripPaywall(res, url) {
})
const out = rewriter.transform(res)
const headers = new Headers(out.headers)
headers.delete('content-security-policy')
headers.delete('content-security-policy-report-only')
headers.delete('x-frame-options')
headers.set('cache-control', 'public, max-age=300')
return new Response(out.body, { status: 200, headers })
const outHeaders = new Headers(out.headers)
outHeaders.delete('content-security-policy')
outHeaders.delete('content-security-policy-report-only')
outHeaders.delete('x-frame-options')
outHeaders.set('content-type', 'text/html; charset=utf-8')
outHeaders.set('cache-control', 'public, max-age=300')
return new Response(out.body, { status: 200, headers: outHeaders })
}
// ---- Reader mode: extract the article into clean text ----