Files
threshing-atlas/analysis/network/build-graph.js
ekcSchianoandClaude Opus 5.5 3b24a11f5a Collection as an ETL pipeline; runs_clean.csv from every source; Report a run
- collection/pipeline.yaml declares every source (extract kind, transform,
  privacy rules, load target, run source); `python -m collection update|build`
- extract keeps data/raw/<source>/current + previous only (git-ignored);
  data/manifest.json records the current raw checksum and change date
- transforms are pure; validate checks runs against data/dictionary.yaml
  before load; load writes byte-stable files only when they change
- data/public/runs_clean.csv: every run from every source with a new
  `source` column (reddit spreadsheet, reddit thread, reported, bot,
  reddit pdf); the sheet's own row label moves to `entry`; run_id is text
- bot importer ported to Python (same output as bot.js on the fixture);
  labels and crosswalk in collection/specs/bot.yaml, written into graph.js
- Reddit through the official API (result links to add by hand); the 12
  result pages move to collection/manual/reddit_results.yaml
- "Report a run": a GitHub issue form generated from
  collection/specs/report_form.yaml, collected as source "reported", plus a
  site tab pointing to it, the community spreadsheet and the megathread
- analysis and the route model read runs_clean.csv (sources in
  measures.yaml); stats.json and policy.json unchanged apart from the
  factor's new name
- daily workflow runs the pipeline; the raw store is kept between runs in
  the Actions cache, encrypted with RAW_ARCHIVE_KEY

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-04 02:46:48 +00:00

47 lines
2.3 KiB
JavaScript

// Build data/public/graph.json from every run in data/public/runs_clean.csv (every source), plus the bot's scene graph.
// node analysis/network/build-graph.js
const fs = require("fs");
const path = require("path");
const G = require("../../core/graph.js");
const root = path.resolve(__dirname, "../..");
const rel = (p) => path.join(root, p);
/** The coded runs of every source, written by `python -m collection build`. */
function readCSV(file) {
const text = fs.readFileSync(file, "utf8"), rows = [];
let row = [], cell = "", q = false;
for (let i = 0; i < text.length; i++) {
const ch = text[i];
if (q) {
if (ch === '"' && text[i + 1] === '"') { cell += '"'; i++; }
else if (ch === '"') q = false;
else cell += ch;
} else if (ch === '"') q = true;
else if (ch === ",") { row.push(cell); cell = ""; }
else if (ch === "\n") { row.push(cell); rows.push(row); row = []; cell = ""; }
else if (ch !== "\r") cell += ch;
}
if (cell || row.length) { row.push(cell); rows.push(row); }
const [head, ...body] = rows;
return body.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] === "" || r[i] === undefined ? null : r[i]])));
}
const runs = readCSV(rel("data/public/runs_clean.csv"));
// spreadsheet runs keep the split between form submissions and bulk logs
const sourceKey = (r) => (r.source === "reddit spreadsheet" ? (r.entry === "Form" ? "form" : "form-bulk") : r.source);
const records = runs.map((r) => ({ source: sourceKey(r), vars: G.formRunVariables(r) }));
const botDir = rel("data/bot");
const scenes = fs.existsSync(path.join(botDir, "scenes.json")) ? JSON.parse(fs.readFileSync(path.join(botDir, "scenes.json"), "utf8")) : {};
const trajs = [];
for (const f of fs.readdirSync(botDir).filter((f) => /^trajectories-.*\.jsonl$/.test(f)).sort()) {
for (const line of fs.readFileSync(path.join(botDir, f), "utf8").split("\n")) if (line.trim()) trajs.push(JSON.parse(line));
}
const graph = {
sources: records.reduce((m, r) => ((m[r.source] = (m[r.source] || 0) + 1), m), {}),
variables: G.buildVariableGraph(records),
scenes: G.buildSceneGraph(scenes, trajs)
};
fs.writeFileSync(rel("data/public/graph.json"), JSON.stringify(graph));
console.log(`graph.json: ${JSON.stringify(graph.sources)}, ${Object.keys(graph.variables.nodes).length} variable nodes, ${Object.keys(graph.scenes).length} scenes`);