Multi-crawler fetch: try Googlebot + social crawlers, keep richest copy

Hard paywalls (AFR etc.) block Googlebot by IP but serve full articles to
facebookexternalhit/Twitterbot for link previews. fetchBest() now fetches
several crawler identities concurrently and keeps the copy with the most text,
so the Reader works on AFR (0 -> 2256 words). Also rank archives first for
known hard-paywall domains.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/index.js

@@ -1,5 +1,5 @@
 import { Hono } from 'hono'
-import { fetchAsCrawler, stripPaywall, extractArticle } from './proxy.js'
+import { fetchAsCrawler, fetchBest, stripPaywall, extractArticle } from './proxy.js'
 import { homePage, readPage, readerPage, errorPage, howPage, aboutPage, bookmarklet } from './views.js'
 
 const app = new Hono()
@@ -36,22 +36,18 @@ app.get('/read', (c) => {
   return c.html(readPage(url, HOST(c)))
 })
 
-// Clean reader: fetch as crawler, extract article text.
+// Clean reader: fetch as several crawlers, keep the richest copy, extract article text.
 app.get('/reader', async (c) => {
   const url = normalizeUrl(c.req.query('url'))
   if (!url) return c.redirect('/')
   try {
-    const res = await fetchAsCrawler(url, { ref: c.req.query('ref') || 'google' })
+    const ref = c.req.query('ref') || 'google'
-    const ct = res.headers.get('content-type') || ''
+    const best = await fetchBest(url, { ref })
-    if (!res.ok && res.status !== 403 && res.status !== 401) {
+    if (!best) return c.html(errorPage(url, 'could not reach the page'), 200)
-      return c.html(errorPage(url, `origin returned ${res.status}`), 200)
+    if (!best.html || !best.contentType.includes('html')) {
-    }
-    if (!ct.includes('html')) {
-      // non-HTML (pdf, etc.) — just hand it through
       return c.redirect(`/proxy?url=${encodeURIComponent(url)}`)
     }
-    const html = await res.text()
+    const art = extractArticle(best.html, url)
-    const art = extractArticle(html, url)
     if (!art.bodyHtml || art.wordCount < 80) {
       return c.html(errorPage(url, 'not enough readable text'), 200)
     }
@@ -61,20 +57,24 @@ app.get('/reader', async (c) => {
   }
 })
 
-// Full-page proxy: fetch as crawler, strip paywall scripts/overlays, keep layout.
+// Full-page proxy: fetch as several crawlers, keep the richest copy, strip the paywall.
 app.get('/proxy', async (c) => {
   const url = normalizeUrl(c.req.query('url'))
   if (!url) return c.redirect('/')
   try {
-    const res = await fetchAsCrawler(url, { ref: c.req.query('ref') || 'google' })
+    const ref = c.req.query('ref') || 'google'
-    const ct = res.headers.get('content-type') || ''
+    // ref=google means "arrive from Google" — honour it with a single Googlebot fetch.
-    if (!ct.includes('html')) {
+    const best = c.req.query('ref') === 'google' && c.req.query('single')
-      // pass binary/other content straight through
+      ? await fetchBest(url, { ref, bots: ['google'] })
-      const h = new Headers(res.headers)
+      : await fetchBest(url, { ref })
+    if (!best) return c.html(errorPage(url, 'could not reach the page'), 200)
+    if (!best.html || !best.contentType.includes('html')) {
+      const single = await fetchAsCrawler(url, { ref, bot: best.bot || 'google' })
+      const h = new Headers(single.headers)
       h.delete('content-security-policy'); h.delete('x-frame-options')
-      return new Response(res.body, { status: res.status, headers: h })
+      return new Response(single.body, { status: single.status, headers: h })
     }
-    return stripPaywall(res, url)
+    return stripPaywall(best.html, best.headers, url)
   } catch (e) {
     return c.html(errorPage(url, e.message), 200)
   }

src/methods.js

@@ -100,14 +100,27 @@ export const METHODS = [
   },
 ]
 
+// Hard paywalls that block Googlebot / don't do the SEO soft-paywall trick — for
+// these, archive snapshots are the most reliable route, so bubble them up.
+const HARD_PAYWALLS = [
+  'afr.com', 'wsj.com', 'ft.com', 'bloomberg.com', 'nytimes.com', 'economist.com',
+  'theaustralian.com.au', 'thetimes.co.uk', 'telegraph.co.uk', 'washingtonpost.com',
+  'newyorker.com', 'wired.com', 'businessinsider.com', 'seekingalpha.com',
+  'barrons.com', 'foreignpolicy.com', 'thetimes.com', 'nzherald.co.nz',
+]
+
 // Order methods so domain-specific + likely-to-work ones bubble to the top.
 export function rankMethods(url) {
   let host = ''
   try { host = new URL(url).host.replace(/^www\./, '') } catch {}
+  const isHard = HARD_PAYWALLS.some((d) => host === d || host.endsWith('.' + d))
   const scored = METHODS.map((m) => {
     let score = 0
     if (m.domains && m.domains.some((d) => host === d || host.endsWith('.' + d))) score += 100
-    if (m.kind === 'internal') score += 10
+    if (m.kind === 'internal') score += 60
+    // For hard paywalls, snapshots beat our crawler fetch — lead with them.
+    if (isHard && m.id === 'archive-ph') score += 70
+    if (isHard && m.id === 'wayback') score += 65
     return { m, score }
   })
   // stable sort by score desc, preserving original order for ties

src/proxy.js

@@ -7,6 +7,8 @@
 const CRAWLER_UAS = {
   google: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
   bing: 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
+  // Many hard paywalls (AFR, some News Corp titles) block Googlebot by IP but still
+  // serve the FULL article to social link-preview crawlers. Try these too.
   facebook: 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
   twitter: 'Twitterbot/1.0',
 }
@@ -34,6 +36,34 @@ export async function fetchAsCrawler(url, { ref = 'google', bot = 'google' } = {
   return res
 }
 
+// Fetch the URL as several crawler identities at once and keep the richest HTML.
+// Returns { html, headers, contentType, bot } or null on total failure.
+// `bots` defaults to a search-crawler + two social crawlers — the combination that
+// covers both SEO soft-paywalls (Googlebot) and social-only gates (AFR etc.).
+export async function fetchBest(url, { ref = 'google', bots = ['google', 'facebook', 'bing'] } = {}) {
+  const settled = await Promise.allSettled(bots.map((bot) => fetchAsCrawler(url, { ref, bot })))
+  let best = null
+  let bestScore = -1
+  let fallback = null // first usable response even if low-score (e.g. non-html)
+  for (let i = 0; i < settled.length; i++) {
+    if (settled[i].status !== 'fulfilled') continue
+    const res = settled[i].value
+    const ct = res.headers.get('content-type') || ''
+    if (!ct.includes('html')) {
+      if (!fallback) fallback = { html: null, headers: res.headers, contentType: ct, bot: bots[i], response: res }
+      continue
+    }
+    let html
+    try { html = await res.text() } catch { continue }
+    // Score: paragraph density dominates; length is a mild tie-breaker.
+    const paras = (html.match(/<p[ >]/gi) || []).length
+    const score = paras * 100 + Math.min(html.length / 1000, 400)
+    if (!fallback) fallback = { html, headers: res.headers, contentType: ct, bot: bots[i] }
+    if (score > bestScore) { bestScore = score; best = { html, headers: res.headers, contentType: ct, bot: bots[i] } }
+  }
+  return best || fallback
+}
+
 // ---- Full-page proxy: keep the layout, drop the paywall ----
 
 const UNLOCK_CSS = `
@@ -72,7 +102,8 @@ const UNLOCK_JS = `(function(){
   } catch(e){}
 })();`
 
-export function stripPaywall(res, url) {
+export function stripPaywall(html, headers, url) {
+  const res = new Response(html, { headers })
   const banner = `<div style="position:sticky;top:0;z-index:2147483647;background:#0B0B09;color:#F0EDE6;font:600 13px/1.4 system-ui,sans-serif;padding:8px 14px;border-bottom:2px solid #FF2D55;display:flex;gap:12px;align-items:center;justify-content:center">
     <span style="color:#FF2D55">RADICAL</span> paywall bypass · <a href="/read?url=${encodeURIComponent(url)}" style="color:#FF6B82">try another method</a> · <a href="/reader?url=${encodeURIComponent(url)}" style="color:#FF6B82">reader view</a>
   </div>`
@@ -96,12 +127,13 @@ export function stripPaywall(res, url) {
     })
 
   const out = rewriter.transform(res)
-  const headers = new Headers(out.headers)
+  const outHeaders = new Headers(out.headers)
-  headers.delete('content-security-policy')
+  outHeaders.delete('content-security-policy')
-  headers.delete('content-security-policy-report-only')
+  outHeaders.delete('content-security-policy-report-only')
-  headers.delete('x-frame-options')
+  outHeaders.delete('x-frame-options')
-  headers.set('cache-control', 'public, max-age=300')
+  outHeaders.set('content-type', 'text/html; charset=utf-8')
-  return new Response(out.body, { status: 200, headers })
+  outHeaders.set('cache-control', 'public, max-age=300')
+  return new Response(out.body, { status: 200, headers: outHeaders })
 }
 
 // ---- Reader mode: extract the article into clean text ----