Collection as an ETL pipeline; runs_clean.csv from every source; Report a run
- collection/pipeline.yaml declares every source (extract kind, transform, privacy rules, load target, run source); `python -m collection update|build` - extract keeps data/raw/<source>/current + previous only (git-ignored); data/manifest.json records the current raw checksum and change date - transforms are pure; validate checks runs against data/dictionary.yaml before load; load writes byte-stable files only when they change - data/public/runs_clean.csv: every run from every source with a new `source` column (reddit spreadsheet, reddit thread, reported, bot, reddit pdf); the sheet's own row label moves to `entry`; run_id is text - bot importer ported to Python (same output as bot.js on the fixture); labels and crosswalk in collection/specs/bot.yaml, written into graph.js - Reddit through the official API (result links to add by hand); the 12 result pages move to collection/manual/reddit_results.yaml - "Report a run": a GitHub issue form generated from collection/specs/report_form.yaml, collected as source "reported", plus a site tab pointing to it, the community spreadsheet and the megathread - analysis and the route model read runs_clean.csv (sources in measures.yaml); stats.json and policy.json unchanged apart from the factor's new name - daily workflow runs the pipeline; the raw store is kept between runs in the Actions cache, encrypted with RAW_ARCHIVE_KEY Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
61bc50583c
commit
3b24a11f5a
@@ -1,4 +1,4 @@
|
||||
// Build data/public/graph.json from every source: community form runs + DragonKind bot trajectories.
|
||||
// Build data/public/graph.json from every run in data/public/runs_clean.csv (every source), plus the bot's scene graph.
|
||||
// node analysis/network/build-graph.js
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
@@ -6,7 +6,7 @@ const G = require("../../core/graph.js");
|
||||
const root = path.resolve(__dirname, "../..");
|
||||
const rel = (p) => path.join(root, p);
|
||||
|
||||
/** The coded runs written by the Python coding rules (python -m atlas.coding -> data/derived/runs_clean.csv). */
|
||||
/** The coded runs of every source, written by `python -m collection build`. */
|
||||
function readCSV(file) {
|
||||
const text = fs.readFileSync(file, "utf8"), rows = [];
|
||||
let row = [], cell = "", q = false;
|
||||
@@ -25,8 +25,10 @@ function readCSV(file) {
|
||||
const [head, ...body] = rows;
|
||||
return body.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] === "" || r[i] === undefined ? null : r[i]])));
|
||||
}
|
||||
const runs = readCSV(rel("data/derived/runs_clean.csv"));
|
||||
const records = runs.map((r) => ({ source: r.source === "Form" ? "form" : "form-bulk", vars: G.formRunVariables(r) }));
|
||||
const runs = readCSV(rel("data/public/runs_clean.csv"));
|
||||
// spreadsheet runs keep the split between form submissions and bulk logs
|
||||
const sourceKey = (r) => (r.source === "reddit spreadsheet" ? (r.entry === "Form" ? "form" : "form-bulk") : r.source);
|
||||
const records = runs.map((r) => ({ source: sourceKey(r), vars: G.formRunVariables(r) }));
|
||||
|
||||
const botDir = rel("data/bot");
|
||||
const scenes = fs.existsSync(path.join(botDir, "scenes.json")) ? JSON.parse(fs.readFileSync(path.join(botDir, "scenes.json"), "utf8")) : {};
|
||||
@@ -34,12 +36,6 @@ const trajs = [];
|
||||
for (const f of fs.readdirSync(botDir).filter((f) => /^trajectories-.*\.jsonl$/.test(f)).sort()) {
|
||||
for (const line of fs.readFileSync(path.join(botDir, f), "utf8").split("\n")) if (line.trim()) trajs.push(JSON.parse(line));
|
||||
}
|
||||
const unmapped = {};
|
||||
for (const t of trajs) {
|
||||
const { vars, unmapped: u } = G.trajectoryVariables(t);
|
||||
records.push({ source: "bot", vars });
|
||||
for (const x of u) unmapped[x] = (unmapped[x] || 0) + 1;
|
||||
}
|
||||
|
||||
const graph = {
|
||||
sources: records.reduce((m, r) => ((m[r.source] = (m[r.source] || 0) + 1), m), {}),
|
||||
@@ -48,5 +44,3 @@ const graph = {
|
||||
};
|
||||
fs.writeFileSync(rel("data/public/graph.json"), JSON.stringify(graph));
|
||||
console.log(`graph.json: ${JSON.stringify(graph.sources)}, ${Object.keys(graph.variables.nodes).length} variable nodes, ${Object.keys(graph.scenes).length} scenes`);
|
||||
const um = Object.keys(unmapped).filter((t) => !/^(head deeper|investigate the scream|lower your|show your bravery|hide behind|run and hope|reds are too|jump|daggers or not|engage the smaller)/i.test(t));
|
||||
if (um.length) console.log("bot choices with no form equivalent:", um);
|
||||
|
||||
Reference in New Issue
Block a user