From 665cd264cc17b861d2912295fbe4bb3b14ee600f Mon Sep 17 00:00:00 2001 From: King Omar Date: Sat, 25 Jul 2026 15:21:36 +1000 Subject: [PATCH] Multi-crawler fetch: try Googlebot + social crawlers, keep richest copy Hard paywalls (AFR etc.) block Googlebot by IP but serve full articles to facebookexternalhit/Twitterbot for link previews. fetchBest() now fetches several crawler identities concurrently and keeps the copy with the most text, so the Reader works on AFR (0 -> 2256 words). Also rank archives first for known hard-paywall domains. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/index.js | 38 +++++++++++++++++++------------------- src/methods.js | 15 ++++++++++++++- src/proxy.js | 46 +++++++++++++++++++++++++++++++++++++++------- 3 files changed, 72 insertions(+), 27 deletions(-) diff --git a/src/index.js b/src/index.js index b146f69..37ec55b 100644 --- a/src/index.js +++ b/src/index.js @@ -1,5 +1,5 @@ import { Hono } from 'hono' -import { fetchAsCrawler, stripPaywall, extractArticle } from './proxy.js' +import { fetchAsCrawler, fetchBest, stripPaywall, extractArticle } from './proxy.js' import { homePage, readPage, readerPage, errorPage, howPage, aboutPage, bookmarklet } from './views.js' const app = new Hono() @@ -36,22 +36,18 @@ app.get('/read', (c) => { return c.html(readPage(url, HOST(c))) }) -// Clean reader: fetch as crawler, extract article text. +// Clean reader: fetch as several crawlers, keep the richest copy, extract article text. app.get('/reader', async (c) => { const url = normalizeUrl(c.req.query('url')) if (!url) return c.redirect('/') try { - const res = await fetchAsCrawler(url, { ref: c.req.query('ref') || 'google' }) - const ct = res.headers.get('content-type') || '' - if (!res.ok && res.status !== 403 && res.status !== 401) { - return c.html(errorPage(url, `origin returned ${res.status}`), 200) - } - if (!ct.includes('html')) { - // non-HTML (pdf, etc.) — just hand it through + const ref = c.req.query('ref') || 'google' + const best = await fetchBest(url, { ref }) + if (!best) return c.html(errorPage(url, 'could not reach the page'), 200) + if (!best.html || !best.contentType.includes('html')) { return c.redirect(`/proxy?url=${encodeURIComponent(url)}`) } - const html = await res.text() - const art = extractArticle(html, url) + const art = extractArticle(best.html, url) if (!art.bodyHtml || art.wordCount < 80) { return c.html(errorPage(url, 'not enough readable text'), 200) } @@ -61,20 +57,24 @@ app.get('/reader', async (c) => { } }) -// Full-page proxy: fetch as crawler, strip paywall scripts/overlays, keep layout. +// Full-page proxy: fetch as several crawlers, keep the richest copy, strip the paywall. app.get('/proxy', async (c) => { const url = normalizeUrl(c.req.query('url')) if (!url) return c.redirect('/') try { - const res = await fetchAsCrawler(url, { ref: c.req.query('ref') || 'google' }) - const ct = res.headers.get('content-type') || '' - if (!ct.includes('html')) { - // pass binary/other content straight through - const h = new Headers(res.headers) + const ref = c.req.query('ref') || 'google' + // ref=google means "arrive from Google" — honour it with a single Googlebot fetch. + const best = c.req.query('ref') === 'google' && c.req.query('single') + ? await fetchBest(url, { ref, bots: ['google'] }) + : await fetchBest(url, { ref }) + if (!best) return c.html(errorPage(url, 'could not reach the page'), 200) + if (!best.html || !best.contentType.includes('html')) { + const single = await fetchAsCrawler(url, { ref, bot: best.bot || 'google' }) + const h = new Headers(single.headers) h.delete('content-security-policy'); h.delete('x-frame-options') - return new Response(res.body, { status: res.status, headers: h }) + return new Response(single.body, { status: single.status, headers: h }) } - return stripPaywall(res, url) + return stripPaywall(best.html, best.headers, url) } catch (e) { return c.html(errorPage(url, e.message), 200) } diff --git a/src/methods.js b/src/methods.js index 35120a5..fdab047 100644 --- a/src/methods.js +++ b/src/methods.js @@ -100,14 +100,27 @@ export const METHODS = [ }, ] +// Hard paywalls that block Googlebot / don't do the SEO soft-paywall trick — for +// these, archive snapshots are the most reliable route, so bubble them up. +const HARD_PAYWALLS = [ + 'afr.com', 'wsj.com', 'ft.com', 'bloomberg.com', 'nytimes.com', 'economist.com', + 'theaustralian.com.au', 'thetimes.co.uk', 'telegraph.co.uk', 'washingtonpost.com', + 'newyorker.com', 'wired.com', 'businessinsider.com', 'seekingalpha.com', + 'barrons.com', 'foreignpolicy.com', 'thetimes.com', 'nzherald.co.nz', +] + // Order methods so domain-specific + likely-to-work ones bubble to the top. export function rankMethods(url) { let host = '' try { host = new URL(url).host.replace(/^www\./, '') } catch {} + const isHard = HARD_PAYWALLS.some((d) => host === d || host.endsWith('.' + d)) const scored = METHODS.map((m) => { let score = 0 if (m.domains && m.domains.some((d) => host === d || host.endsWith('.' + d))) score += 100 - if (m.kind === 'internal') score += 10 + if (m.kind === 'internal') score += 60 + // For hard paywalls, snapshots beat our crawler fetch — lead with them. + if (isHard && m.id === 'archive-ph') score += 70 + if (isHard && m.id === 'wayback') score += 65 return { m, score } }) // stable sort by score desc, preserving original order for ties diff --git a/src/proxy.js b/src/proxy.js index 46bf00c..7b098a4 100644 --- a/src/proxy.js +++ b/src/proxy.js @@ -7,6 +7,8 @@ const CRAWLER_UAS = { google: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', bing: 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)', + // Many hard paywalls (AFR, some News Corp titles) block Googlebot by IP but still + // serve the FULL article to social link-preview crawlers. Try these too. facebook: 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)', twitter: 'Twitterbot/1.0', } @@ -34,6 +36,34 @@ export async function fetchAsCrawler(url, { ref = 'google', bot = 'google' } = { return res } +// Fetch the URL as several crawler identities at once and keep the richest HTML. +// Returns { html, headers, contentType, bot } or null on total failure. +// `bots` defaults to a search-crawler + two social crawlers — the combination that +// covers both SEO soft-paywalls (Googlebot) and social-only gates (AFR etc.). +export async function fetchBest(url, { ref = 'google', bots = ['google', 'facebook', 'bing'] } = {}) { + const settled = await Promise.allSettled(bots.map((bot) => fetchAsCrawler(url, { ref, bot }))) + let best = null + let bestScore = -1 + let fallback = null // first usable response even if low-score (e.g. non-html) + for (let i = 0; i < settled.length; i++) { + if (settled[i].status !== 'fulfilled') continue + const res = settled[i].value + const ct = res.headers.get('content-type') || '' + if (!ct.includes('html')) { + if (!fallback) fallback = { html: null, headers: res.headers, contentType: ct, bot: bots[i], response: res } + continue + } + let html + try { html = await res.text() } catch { continue } + // Score: paragraph density dominates; length is a mild tie-breaker. + const paras = (html.match(/]/gi) || []).length + const score = paras * 100 + Math.min(html.length / 1000, 400) + if (!fallback) fallback = { html, headers: res.headers, contentType: ct, bot: bots[i] } + if (score > bestScore) { bestScore = score; best = { html, headers: res.headers, contentType: ct, bot: bots[i] } } + } + return best || fallback +} + // ---- Full-page proxy: keep the layout, drop the paywall ---- const UNLOCK_CSS = ` @@ -72,7 +102,8 @@ const UNLOCK_JS = `(function(){ } catch(e){} })();` -export function stripPaywall(res, url) { +export function stripPaywall(html, headers, url) { + const res = new Response(html, { headers }) const banner = `
RADICAL paywall bypass · try another method · reader view
` @@ -96,12 +127,13 @@ export function stripPaywall(res, url) { }) const out = rewriter.transform(res) - const headers = new Headers(out.headers) - headers.delete('content-security-policy') - headers.delete('content-security-policy-report-only') - headers.delete('x-frame-options') - headers.set('cache-control', 'public, max-age=300') - return new Response(out.body, { status: 200, headers }) + const outHeaders = new Headers(out.headers) + outHeaders.delete('content-security-policy') + outHeaders.delete('content-security-policy-report-only') + outHeaders.delete('x-frame-options') + outHeaders.set('content-type', 'text/html; charset=utf-8') + outHeaders.set('cache-control', 'public, max-age=300') + return new Response(out.body, { status: 200, headers: outHeaders }) } // ---- Reader mode: extract the article into clean text ----