From 7e9eca6b0fbb605d7b68f173ab85342ea96e71b5 Mon Sep 17 00:00:00 2001 From: King Omar Date: Wed, 22 Jul 2026 01:56:38 +1000 Subject: [PATCH] Expose API via Cloudflare Tunnel over HTTPS/443 A deployed Worker's fetch() cannot reach the API on :3000, so publish it at https://search-api.theradicalparty.com through a Cloudflare Tunnel. Point the frontend API_URL at HTTPS/443, add cloudflared install to install-vm.sh (driven by CLOUDFLARED_TOKEN), thread the token through deploy.sh, and add docs/DEPLOY.md with the full tunnel + DNS setup. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/DEPLOY.md | 90 ++++++++++++++++++++++++++++++++++++++ frontend/wrangler.jsonc | 4 +- scripts/deploy.env.example | 4 ++ scripts/deploy.sh | 2 +- scripts/install-vm.sh | 21 ++++++++- 5 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 docs/DEPLOY.md diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md new file mode 100644 index 0000000..847f4bb --- /dev/null +++ b/docs/DEPLOY.md @@ -0,0 +1,90 @@ +# Deploying RADICAL_SEARCH + +Two moving parts: the **frontend** (Cloudflare Worker) and the **backend** +(Meilisearch + API + crawler on a VM, published through a Cloudflare Tunnel). + +## Why a tunnel? + +The frontend Worker calls the API server-side with `fetch(API_URL)`. A deployed +Cloudflare Worker's `fetch()` can only reliably reach **ports 80 and 443** — it +cannot talk to a non-standard port like `:3000`, even for a DNS-only (grey-cloud) +origin. So the API is published at **`https://search-api.theradicalparty.com`** +(port 443) via a **Cloudflare Tunnel**, which also gives TLS and requires no open +inbound ports on the VM. + +``` +Worker ──https:443──▶ Cloudflare ──tunnel──▶ cloudflared (VM) ──▶ 127.0.0.1:3000 (API) +``` + +## Backend (VM) + +### 1. One-time: create the tunnel + +In the Cloudflare **Zero Trust** dashboard → **Networks → Tunnels**: + +1. **Create a tunnel** (type: *Cloudflared*), name it e.g. `search-api`. +2. Copy the **connector token** (the long string in the `cloudflared service + install ` snippet). +3. Add a **Public Hostname**: + - Subdomain `search-api`, domain `theradicalparty.com` + - Service: **HTTP** `://localhost:3000` + + Cloudflare automatically creates the orange-cloud CNAME + `search-api.theradicalparty.com → .cfargotunnel.com`. + +### 2. Configure credentials + +Copy the template and fill it in (this file is git-ignored): + +```bash +cp scripts/deploy.env.example scripts/deploy.env +# set VM_HOST / VM_USER / VM_PASS and CLOUDFLARED_TOKEN +``` + +> ⚠️ The VM password previously committed in `deploy.sh` was rejected by the VM +> on 2026-07-22. Confirm working SSH credentials before deploying. + +### 3. Deploy + +```bash +cd scripts && ./deploy.sh +``` + +This syncs `api/` and `crawler/` to the VM and runs `install-vm.sh`, which: +- installs Meilisearch (systemd `meilisearch.service`, `:7700`, master key) +- installs the API (systemd `search-api.service`, `:3000`) +- installs the crawler under `/opt/search-crawler` +- if `CLOUDFLARED_TOKEN` is set: installs `cloudflared` as a systemd service + +### 4. Crawl + +```bash +ssh root@ 'cd /opt/search-crawler && LIMIT=50000 CONCURRENCY=8 npm start' +``` + +### 5. Verify + +```bash +curl https://search-api.theradicalparty.com/health +curl 'https://search-api.theradicalparty.com/search?q=news' +``` + +## Frontend (Cloudflare Worker) + +`frontend/wrangler.jsonc` already points `API_URL` at +`https://search-api.theradicalparty.com` and routes +`search.theradicalparty.com/*`. Deploy with: + +```bash +cd frontend && npm install && npm run deploy +``` + +> Deploy the frontend **after** the tunnel + API are live, otherwise +> `search.theradicalparty.com` will return API errors until the backend answers. + +## Hardening (optional) + +- Firewall the VM so `:3000` and `:7700` are not reachable from the public + internet — all external access should go through the tunnel. +- Rotate the Meilisearch master key (currently `masterKey`) and update the + `MEILI_KEY` env in `install-vm.sh`. diff --git a/frontend/wrangler.jsonc b/frontend/wrangler.jsonc index 81b1a40..981f09f 100644 --- a/frontend/wrangler.jsonc +++ b/frontend/wrangler.jsonc @@ -7,6 +7,8 @@ { "pattern": "search.theradicalparty.com/*", "zone_name": "theradicalparty.com" } ], "vars": { - "API_URL": "http://search-api.theradicalparty.com:3000" + // API is fronted by a Cloudflare Tunnel on HTTPS/443 — a deployed Worker's + // fetch() cannot reliably reach non-standard ports (e.g. :3000). See docs/DEPLOY.md. + "API_URL": "https://search-api.theradicalparty.com" } } diff --git a/scripts/deploy.env.example b/scripts/deploy.env.example index 0781787..33b5f61 100644 --- a/scripts/deploy.env.example +++ b/scripts/deploy.env.example @@ -2,3 +2,7 @@ VM_HOST=your.vm.ip.address VM_USER=root VM_PASS=your-vm-password + +# Cloudflare Tunnel connector token (Zero Trust > Networks > Tunnels). +# Route public hostname search-api.theradicalparty.com -> http://localhost:3000. +CLOUDFLARED_TOKEN= diff --git a/scripts/deploy.sh b/scripts/deploy.sh index fa3107e..5991c14 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -21,7 +21,7 @@ scp_cmd ../crawler/. "$VM_USER@$VM_HOST:/tmp/search-crawler/" scp_cmd install-vm.sh "$VM_USER@$VM_HOST:/tmp/install-vm.sh" echo "=== Running install script ===" -ssh_cmd "bash /tmp/install-vm.sh" +ssh_cmd "CLOUDFLARED_TOKEN='${CLOUDFLARED_TOKEN:-}' bash /tmp/install-vm.sh" echo "=== Done ===" ssh_cmd "systemctl status meilisearch --no-pager && systemctl status search-api --no-pager" diff --git a/scripts/install-vm.sh b/scripts/install-vm.sh index 321fe62..5ce3c38 100755 --- a/scripts/install-vm.sh +++ b/scripts/install-vm.sh @@ -65,10 +65,29 @@ cp -r /tmp/search-crawler/. /opt/search-crawler/ cd /opt/search-crawler npm install +echo "=== Installing cloudflared tunnel (exposes API over HTTPS/443) ===" +# A deployed Cloudflare Worker's fetch() can't reach :3000, so the API is +# published at https://search-api.theradicalparty.com via a Cloudflare Tunnel. +# Provide CLOUDFLARED_TOKEN (connector token from the Zero Trust dashboard); +# route the public hostname search-api.theradicalparty.com -> http://localhost:3000 there. +if [ -n "$CLOUDFLARED_TOKEN" ]; then + if ! command -v cloudflared >/dev/null 2>&1; then + curl -L --output /tmp/cloudflared.deb \ + https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb + dpkg -i /tmp/cloudflared.deb + fi + cloudflared service install "$CLOUDFLARED_TOKEN" + systemctl enable cloudflared + systemctl restart cloudflared + echo "cloudflared installed and running" +else + echo "CLOUDFLARED_TOKEN not set — skipping tunnel install (see docs/DEPLOY.md)" +fi + echo "" echo "Done! Services running:" echo " Meilisearch: http://localhost:7700" -echo " Search API: http://localhost:3000" +echo " Search API: http://localhost:3000 (published via tunnel at https://search-api.theradicalparty.com)" echo "" echo "To start crawling:" echo " cd /opt/search-crawler && LIMIT=50000 CONCURRENCY=8 npm start"