Switch CRICOS to official data.gov.au D1 dataset (26K courses, 47K locations)

Replace unreliable JS-rendered scraper with fast D1 queries against the official CRICOS government dataset, updated monthly from data.gov.au. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-07-10 12:52:25 +10:00 · 2026-07-10 12:52:25 +10:00 · 53bcbf7050
commit 53bcbf7050
parent 158023df1a
3 changed files with 127 additions and 99 deletions
--- a/migrations/002_cricos_data.sql
+++ b/migrations/002_cricos_data.sql
@ -0,0 +1,33 @@
+CREATE TABLE IF NOT EXISTS cricos_courses (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  provider_code TEXT NOT NULL,
+  institution_name TEXT NOT NULL,
+  course_code TEXT NOT NULL,
+  course_name TEXT NOT NULL,
+  vet_national_code TEXT,
+  course_level TEXT,
+  field_broad TEXT,
+  field_narrow TEXT,
+  duration_weeks INTEGER,
+  tuition_fee TEXT,
+  course_language TEXT,
+  work_component INTEGER DEFAULT 0,
+  foundation_studies INTEGER DEFAULT 0,
+  expired INTEGER DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS cricos_course_locations (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  provider_code TEXT NOT NULL,
+  course_code TEXT NOT NULL,
+  location_name TEXT,
+  location_city TEXT,
+  location_state TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_courses_code ON cricos_courses(course_code);
+CREATE INDEX IF NOT EXISTS idx_courses_institution ON cricos_courses(provider_code);
+CREATE INDEX IF NOT EXISTS idx_courses_level ON cricos_courses(course_level);
+CREATE INDEX IF NOT EXISTS idx_courses_expired ON cricos_courses(expired);
+CREATE INDEX IF NOT EXISTS idx_locations_code ON cricos_course_locations(course_code);
+CREATE INDEX IF NOT EXISTS idx_locations_state ON cricos_course_locations(location_state);
--- a/src/cricos.js
+++ b/src/cricos.js
@ -1,105 +1,95 @@
-const CRICOS_BASE = 'https://cricos.education.gov.au';
-const CACHE_TTL_HOURS = 24;
-
-async function fetchViewState() {
-  const res = await fetch(`${CRICOS_BASE}/Course/CourseSearch.aspx`, {
-    headers: {
-      'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-      'Accept': 'text/html,application/xhtml+xml',
-    }
-  });
-  const html = await res.text();
-  const cookies = res.headers.get('set-cookie') || '';
-
-  const vs = (html.match(/id="__VIEWSTATE"\s+value="([^"]+)"/) || [])[1] || '';
-  const vsgen = (html.match(/id="__VIEWSTATEGENERATOR"\s+value="([^"]+)"/) || [])[1] || '';
-  const ev = (html.match(/id="__EVENTVALIDATION"\s+value="([^"]+)"/) || [])[1] || '';
-  return { vs, vsgen, ev, cookies };
-}
-
-function parseCricosResults(html) {
-  const courses = [];
-  // Match table rows in the results table
-  const tableMatch = html.match(/<table[^>]*id="[^"]*grd[^"]*"[^>]*>([\s\S]*?)<\/table>/i)
-    || html.match(/<table[^>]*class="[^"]*result[^"]*"[^>]*>([\s\S]*?)<\/table>/i)
-    || html.match(/<table[^>]*>([\s\S]*?)<\/table>/gi);
-
-  if (!tableMatch) return courses;
-
-  const tableHtml = Array.isArray(tableMatch) ? tableMatch.find(t => t.includes('CRICOS') || t.includes('Course')) || tableMatch[0] : tableMatch[1] || tableMatch[0];
-
-  // Extract all <tr> rows with <td> cells
-  const rows = tableHtml.match(/<tr[^>]*>([\s\S]*?)<\/tr>/gi) || [];
-  for (const row of rows) {
-    const cells = (row.match(/<td[^>]*>([\s\S]*?)<\/td>/gi) || [])
-      .map(td => td.replace(/<[^>]+>/g, '').replace(/&amp;/g, '&').replace(/&nbsp;/g, ' ').trim());
-    if (cells.length >= 4 && cells.some(c => c.length > 2)) {
-      courses.push({
-        cricosCode: cells[0] || '',
-        courseName: cells[1] || '',
-        provider: cells[2] || '',
-        state: cells[3] || '',
-        duration: cells[4] || '',
-        fee: cells[5] || '',
-      });
-    }
-  }
-  return courses.filter(c => c.cricosCode || c.courseName);
-}
+// CRICOS search using the official data.gov.au dataset imported into D1.
+// Data is refreshed monthly from:
+// https://data.gov.au/data/dataset/e5ae7059-bfa8-4fa4-a5c0-c13cf3520193

 async function searchCricos(env, params) {
-  const cacheKey = JSON.stringify(params);
-  const cached = await env.DB.prepare('SELECT results_json, cached_at FROM cricos_cache WHERE cache_key = ?')
-    .bind(cacheKey).first();
+  const { courseName, cricosCode, state, courseLevel } = params;
+  const limit = 50;

-  if (cached) {
-    const ageHours = (Date.now() - new Date(cached.cached_at).getTime()) / 3600000;
-    if (ageHours < CACHE_TTL_HOURS) {
-      return { results: JSON.parse(cached.results_json), fromCache: true };
-    }
+  // Build WHERE clauses
+  const conditions = ['c.expired = 0'];
+  const bindings = [];
+
+  if (courseName && courseName.trim()) {
+    conditions.push("c.course_name LIKE ?");
+    bindings.push(`%${courseName.trim()}%`);
+  }
+  if (cricosCode && cricosCode.trim()) {
+    conditions.push("c.course_code LIKE ?");
+    bindings.push(`%${cricosCode.trim()}%`);
+  }
+  if (courseLevel && courseLevel.trim()) {
+    // Match against course_level field
+    const levelMap = {
+      '1': 'Bachelor',
+      '2': 'Master',
+      '3': 'Doctoral',
+      '4': 'Diploma',
+      '5': 'Certificate',
+      '6': 'Advanced Diploma',
+      '7': 'Graduate Certificate',
+      '8': 'Graduate Diploma',
+    };
+    const levelText = levelMap[courseLevel] || courseLevel;
+    conditions.push("c.course_level LIKE ?");
+    bindings.push(`%${levelText}%`);
  }

-  try {
-    const { vs, vsgen, ev, cookies } = await fetchViewState();
+  const whereClause = conditions.join(' AND ');

-    const formData = new URLSearchParams({
-      '__VIEWSTATE': vs,
-      '__VIEWSTATEGENERATOR': vsgen,
-      '__EVENTVALIDATION': ev,
-      'ctl00$ContentPlaceHolder1$txtCourseName': params.courseName || '',
-      'ctl00$ContentPlaceHolder1$txtCRICOSCourseCode': params.cricosCode || '',
-      'ctl00$ContentPlaceHolder1$ddlState': params.state || '',
-      'ctl00$ContentPlaceHolder1$ddlCourseLevel': params.courseLevel || '',
-      'ctl00$ContentPlaceHolder1$ddlBroadField': params.broadField || '',
-      'ctl00$ContentPlaceHolder1$ddlNarrowField': '',
-      'ctl00$ContentPlaceHolder1$ddlDetailedField': '',
-      'ctl00$ContentPlaceHolder1$ddlLanguage': '',
-      'ctl00$ContentPlaceHolder1$btnSearch': 'Search',
-    });
+  let query, results;

-    const res = await fetch(`${CRICOS_BASE}/Course/CourseSearch.aspx`, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/x-www-form-urlencoded',
-        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-        'Referer': `${CRICOS_BASE}/Course/CourseSearch.aspx`,
-        'Cookie': cookies,
-      },
-      body: formData.toString(),
-    });
-
-    const html = await res.text();
-    const results = parseCricosResults(html);
-
-    await env.DB.prepare(
-      'INSERT OR REPLACE INTO cricos_cache (cache_key, results_json, cached_at) VALUES (?, ?, datetime("now"))'
-    ).bind(cacheKey, JSON.stringify(results)).run();
-
-    return { results, fromCache: false };
-  } catch (e) {
-    if (cached) return { results: JSON.parse(cached.results_json), fromCache: true, stale: true };
-    return { results: [], error: e.message };
+  if (state && state.trim()) {
+    // Join with locations to filter by state
+    query = `
+      SELECT DISTINCT c.course_code, c.course_name, c.institution_name, c.course_level,
+             c.duration_weeks, c.tuition_fee, c.provider_code, c.vet_national_code,
+             l.location_state, l.location_city
+      FROM cricos_courses c
+      INNER JOIN cricos_course_locations l ON c.course_code = l.course_code
+      WHERE ${whereClause}
+        AND l.location_state = ?
+      ORDER BY c.institution_name, c.course_name
+      LIMIT ${limit}
+    `;
+    bindings.push(state.trim());
+  } else {
+    // No state filter — use course_locations for a representative state
+    query = `
+      SELECT c.course_code, c.course_name, c.institution_name, c.course_level,
+             c.duration_weeks, c.tuition_fee, c.provider_code, c.vet_national_code,
+             (SELECT l.location_state FROM cricos_course_locations l WHERE l.course_code = c.course_code LIMIT 1) as location_state,
+             (SELECT l.location_city FROM cricos_course_locations l WHERE l.course_code = c.course_code LIMIT 1) as location_city
+      FROM cricos_courses c
+      WHERE ${whereClause}
+      ORDER BY c.institution_name, c.course_name
+      LIMIT ${limit}
+    `;
  }
+
+  const stmt = env.DB.prepare(query).bind(...bindings);
+  const raw = await stmt.all();
+  results = (raw.results || []).map(row => ({
+    cricosCode: row.course_code,
+    courseName: row.course_name,
+    provider: row.institution_name,
+    providerCode: row.provider_code,
+    level: row.course_level,
+    state: row.location_state || '',
+    city: row.location_city || '',
+    durationWeeks: row.duration_weeks,
+    tuitionFee: row.tuition_fee,
+    vetCode: row.vet_national_code,
+  }));
+
+  return { results, fromCache: false };
 }

-export { searchCricos };
+async function getCricosStats(env) {
+  const r = await env.DB.prepare(
+    'SELECT COUNT(*) as total, COUNT(DISTINCT institution_name) as providers FROM cricos_courses WHERE expired=0'
+  ).first();
+  return r;
+}
+
+export { searchCricos, getCricosStats };
--- a/src/templates.js
+++ b/src/templates.js
@ -230,7 +230,7 @@ function homePage(user) {
      <div><div class="stat-num">5,000+</div><div class="stat-label">Clients Helped</div></div>
      <div><div class="stat-num">98%</div><div class="stat-label">Success Rate</div></div>
      <div><div class="stat-num">15+</div><div class="stat-label">Years Experience</div></div>
-      <div><div class="stat-num">50+</div><div class="stat-label">Partner Institutions</div></div>
+      <div><div class="stat-num">1,500+</div><div class="stat-label">CRICOS Institutions</div></div>
    </div>
  </div>

@ -288,6 +288,9 @@ function homePage(user) {
              <option value="3">Doctoral Degree</option>
              <option value="4">Diploma</option>
              <option value="5">Certificate</option>
+              <option value="6">Advanced Diploma</option>
+              <option value="7">Graduate Certificate</option>
+              <option value="8">Graduate Diploma</option>
            </select>
          </div>
          <div style="display:flex;align-items:flex-end">
@ -423,9 +426,11 @@ function coursesPage(user, results, params, fromCache, error) {
              <div style="color:#64748b;font-size:.9rem;margin-bottom:6px">${esc(c.provider || '')}</div>
              <div class="course-meta">
                ${c.cricosCode ? `<span class="badge badge-blue">CRICOS: ${esc(c.cricosCode)}</span>` : ''}
-                ${c.state ? `<span class="badge badge-gray">📍 ${esc(c.state)}</span>` : ''}
-                ${c.duration ? `<span class="badge badge-gray">⏱ ${esc(c.duration)}</span>` : ''}
-                ${c.fee ? `<span class="badge badge-green">💰 ${esc(c.fee)}</span>` : ''}
+                ${c.level ? `<span class="badge badge-gray">${esc(c.level)}</span>` : ''}
+                ${(c.city || c.state) ? `<span class="badge badge-gray">📍 ${esc([c.city,c.state].filter(Boolean).join(', '))}</span>` : ''}
+                ${c.durationWeeks ? `<span class="badge badge-gray">⏱ ${esc(String(c.durationWeeks))} weeks</span>` : ''}
+                ${c.tuitionFee ? `<span class="badge badge-green">💰 ${esc(c.tuitionFee)}</span>` : ''}
+                ${c.vetCode ? `<span class="badge badge-gray">VET: ${esc(c.vetCode)}</span>` : ''}
              </div>
            </div>
            <div class="course-actions">