Files
threshing-atlas/site/qa/crawl.js
ekcSchianoandClaude Opus 5.5 3b24a11f5a Collection as an ETL pipeline; runs_clean.csv from every source; Report a run
- collection/pipeline.yaml declares every source (extract kind, transform,
  privacy rules, load target, run source); `python -m collection update|build`
- extract keeps data/raw/<source>/current + previous only (git-ignored);
  data/manifest.json records the current raw checksum and change date
- transforms are pure; validate checks runs against data/dictionary.yaml
  before load; load writes byte-stable files only when they change
- data/public/runs_clean.csv: every run from every source with a new
  `source` column (reddit spreadsheet, reddit thread, reported, bot,
  reddit pdf); the sheet's own row label moves to `entry`; run_id is text
- bot importer ported to Python (same output as bot.js on the fixture);
  labels and crosswalk in collection/specs/bot.yaml, written into graph.js
- Reddit through the official API (result links to add by hand); the 12
  result pages move to collection/manual/reddit_results.yaml
- "Report a run": a GitHub issue form generated from
  collection/specs/report_form.yaml, collected as source "reported", plus a
  site tab pointing to it, the community spreadsheet and the megathread
- analysis and the route model read runs_clean.csv (sources in
  measures.yaml); stats.json and policy.json unchanged apart from the
  factor's new name
- daily workflow runs the pipeline; the raw store is kept between runs in
  the Actions cache, encrypted with RAW_ARCHIVE_KEY

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-04 02:46:48 +00:00

73 lines
4.6 KiB
JavaScript

// QA crawl: every tab, text dump, links, buttons, console errors. node crawl.js <outdir>
const { chromium } = require("playwright");
const fs = require("fs");
// screenshots and logs go to the folder given, or a temp folder (never a folder named "undefined" in the repo)
const OUT = process.argv[2] || require("path").join(require("os").tmpdir(), "atlas-qa");
require("fs").mkdirSync(OUT, { recursive: true });
const BASE = "http://localhost:8765/";
(async () => {
const b = await chromium.launch();
const log = { errors: [], links: [], buttons: [], tabs: {} };
for (const [vpName, vp] of [["desktop", { width: 1440, height: 1000 }], ["phone", { width: 390, height: 844 }]]) {
const p = await b.newPage({ viewport: vp });
p.on("pageerror", (e) => log.errors.push(`${vpName} pageerror: ${e.message}`));
p.on("console", (m) => { if (m.type() === "error" && !/net::ERR|Failed to load resource/.test(m.text())) log.errors.push(`${vpName} console: ${m.text()}`); });
for (const t of ["valley", "odds", "dragons", "deep", "data", "report"]) {
await p.goto(BASE + "#" + t); await p.waitForTimeout(t === "valley" ? 2500 : 1800);
// open every <details> so hidden text is reviewed too
await p.$$eval(`#panel-${t} details`, (ds) => ds.forEach((d) => (d.open = true)));
await p.waitForTimeout(300);
await p.screenshot({ path: `${OUT}/${vpName}-${t}.png`, fullPage: true });
if (vpName === "desktop") {
log.tabs[t] = await p.evaluate((t) => document.querySelector("#panel-" + t).innerText, t);
const links = await p.$$eval(`#panel-${t} a[href], header a[href], footer a[href]`, (as) => as.map((a) => ({ text: a.innerText.trim().slice(0, 60), href: a.getAttribute("href") })));
links.forEach((l) => log.links.push({ tab: t, ...l }));
const btns = await p.$$eval(`#panel-${t} button`, (bs) => bs.map((x) => x.innerText.trim().replace(/\s+/g, " ").slice(0, 50)));
log.buttons.push({ tab: t, n: btns.length, sample: [...new Set(btns)].slice(0, 40) });
}
// overflow check
const sw = await p.evaluate(() => document.documentElement.scrollWidth);
if (sw > vp.width + 1) log.errors.push(`${vpName} ${t}: horizontal overflow ${sw}px`);
}
await p.close();
}
// click every button in every tab (desktop), checking for errors after each
const p = await b.newPage({ viewport: { width: 1440, height: 1000 } });
p.on("pageerror", (e) => log.errors.push(`click pageerror: ${e.message}`));
for (const t of ["valley", "odds", "dragons", "deep", "data", "report"]) {
await p.goto(BASE + "#" + t); await p.waitForTimeout(2000);
const n = await p.$$eval(`#panel-${t} button`, (bs) => bs.length);
let clicked = 0;
for (let i = 0; i < Math.min(n, 120); i++) {
const bs = await p.$$(`#panel-${t} button`);
if (!bs[i]) continue;
const vis = await bs[i].isVisible();
if (!vis) continue;
const label = (await bs[i].innerText()).trim().slice(0, 40);
if (/download|copy/i.test(label)) continue;
try { await bs[i].click({ timeout: 1500 }); clicked++; await p.waitForTimeout(120); } catch (e) { log.errors.push(`${t}: could not click "${label}": ${e.message.split("\n")[0]}`); }
const cur = await p.evaluate(() => location.hash);
if (!cur.startsWith("#" + t)) { await p.goto(BASE + "#" + t); await p.waitForTimeout(1200); }
}
log.tabs[t + "_clicked"] = clicked;
}
// in-page links
await p.goto(BASE); await p.waitForTimeout(2000);
for (const l of log.links.filter((x) => x.href.startsWith("#"))) {
await p.goto(BASE + "#" + l.tab); await p.waitForTimeout(800);
const el = await p.$(`#panel-${l.tab} a[href="${l.href}"]`);
if (!el || !(await el.isVisible())) continue;
await el.click(); await p.waitForTimeout(800);
const sel = await p.evaluate(() => document.querySelector("[role=tab][aria-selected=true]").id);
const target = l.href.slice(1).split("/")[0];
const exists = await p.evaluate((id) => !!document.getElementById(id) || ["valley","odds","dragons","deep","data"].includes(id), target);
l.result = `${sel}${exists ? "" : " MISSING TARGET"}`;
}
fs.writeFileSync(`${OUT}/log.json`, JSON.stringify(log, null, 1));
for (const [t, txt] of Object.entries(log.tabs)) if (typeof txt === "string") fs.writeFileSync(`${OUT}/text-${t}.txt`, txt);
console.log("errors:", log.errors.length, log.errors.slice(0, 20));
console.log("clicked:", Object.entries(log.tabs).filter(([k]) => k.endsWith("_clicked")));
console.log("links:"); log.links.forEach((l) => console.log(" ", l.tab, "|", l.text, "|", l.href, "|", l.result || ""));
await b.close();
})();