- collection/pipeline.yaml declares every source (extract kind, transform, privacy rules, load target, run source); `python -m collection update|build` - extract keeps data/raw/<source>/current + previous only (git-ignored); data/manifest.json records the current raw checksum and change date - transforms are pure; validate checks runs against data/dictionary.yaml before load; load writes byte-stable files only when they change - data/public/runs_clean.csv: every run from every source with a new `source` column (reddit spreadsheet, reddit thread, reported, bot, reddit pdf); the sheet's own row label moves to `entry`; run_id is text - bot importer ported to Python (same output as bot.js on the fixture); labels and crosswalk in collection/specs/bot.yaml, written into graph.js - Reddit through the official API (result links to add by hand); the 12 result pages move to collection/manual/reddit_results.yaml - "Report a run": a GitHub issue form generated from collection/specs/report_form.yaml, collected as source "reported", plus a site tab pointing to it, the community spreadsheet and the megathread - analysis and the route model read runs_clean.csv (sources in measures.yaml); stats.json and policy.json unchanged apart from the factor's new name - daily workflow runs the pipeline; the raw store is kept between runs in the Actions cache, encrypted with RAW_ARCHIVE_KEY Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
47 lines
2.3 KiB
JavaScript
47 lines
2.3 KiB
JavaScript
// Build data/public/graph.json from every run in data/public/runs_clean.csv (every source), plus the bot's scene graph.
|
|
// node analysis/network/build-graph.js
|
|
const fs = require("fs");
|
|
const path = require("path");
|
|
const G = require("../../core/graph.js");
|
|
const root = path.resolve(__dirname, "../..");
|
|
const rel = (p) => path.join(root, p);
|
|
|
|
/** The coded runs of every source, written by `python -m collection build`. */
|
|
function readCSV(file) {
|
|
const text = fs.readFileSync(file, "utf8"), rows = [];
|
|
let row = [], cell = "", q = false;
|
|
for (let i = 0; i < text.length; i++) {
|
|
const ch = text[i];
|
|
if (q) {
|
|
if (ch === '"' && text[i + 1] === '"') { cell += '"'; i++; }
|
|
else if (ch === '"') q = false;
|
|
else cell += ch;
|
|
} else if (ch === '"') q = true;
|
|
else if (ch === ",") { row.push(cell); cell = ""; }
|
|
else if (ch === "\n") { row.push(cell); rows.push(row); row = []; cell = ""; }
|
|
else if (ch !== "\r") cell += ch;
|
|
}
|
|
if (cell || row.length) { row.push(cell); rows.push(row); }
|
|
const [head, ...body] = rows;
|
|
return body.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] === "" || r[i] === undefined ? null : r[i]])));
|
|
}
|
|
const runs = readCSV(rel("data/public/runs_clean.csv"));
|
|
// spreadsheet runs keep the split between form submissions and bulk logs
|
|
const sourceKey = (r) => (r.source === "reddit spreadsheet" ? (r.entry === "Form" ? "form" : "form-bulk") : r.source);
|
|
const records = runs.map((r) => ({ source: sourceKey(r), vars: G.formRunVariables(r) }));
|
|
|
|
const botDir = rel("data/bot");
|
|
const scenes = fs.existsSync(path.join(botDir, "scenes.json")) ? JSON.parse(fs.readFileSync(path.join(botDir, "scenes.json"), "utf8")) : {};
|
|
const trajs = [];
|
|
for (const f of fs.readdirSync(botDir).filter((f) => /^trajectories-.*\.jsonl$/.test(f)).sort()) {
|
|
for (const line of fs.readFileSync(path.join(botDir, f), "utf8").split("\n")) if (line.trim()) trajs.push(JSON.parse(line));
|
|
}
|
|
|
|
const graph = {
|
|
sources: records.reduce((m, r) => ((m[r.source] = (m[r.source] || 0) + 1), m), {}),
|
|
variables: G.buildVariableGraph(records),
|
|
scenes: G.buildSceneGraph(scenes, trajs)
|
|
};
|
|
fs.writeFileSync(rel("data/public/graph.json"), JSON.stringify(graph));
|
|
console.log(`graph.json: ${JSON.stringify(graph.sources)}, ${Object.keys(graph.variables.nodes).length} variable nodes, ${Object.keys(graph.scenes).length} scenes`);
|