Collection as an ETL pipeline; runs_clean.csv from every source; Report a run

- collection/pipeline.yaml declares every source (extract kind, transform,
  privacy rules, load target, run source); `python -m collection update|build`
- extract keeps data/raw/<source>/current + previous only (git-ignored);
  data/manifest.json records the current raw checksum and change date
- transforms are pure; validate checks runs against data/dictionary.yaml
  before load; load writes byte-stable files only when they change
- data/public/runs_clean.csv: every run from every source with a new
  `source` column (reddit spreadsheet, reddit thread, reported, bot,
  reddit pdf); the sheet's own row label moves to `entry`; run_id is text
- bot importer ported to Python (same output as bot.js on the fixture);
  labels and crosswalk in collection/specs/bot.yaml, written into graph.js
- Reddit through the official API (result links to add by hand); the 12
  result pages move to collection/manual/reddit_results.yaml
- "Report a run": a GitHub issue form generated from
  collection/specs/report_form.yaml, collected as source "reported", plus a
  site tab pointing to it, the community spreadsheet and the megathread
- analysis and the route model read runs_clean.csv (sources in
  measures.yaml); stats.json and policy.json unchanged apart from the
  factor's new name
- daily workflow runs the pipeline; the raw store is kept between runs in
  the Actions cache, encrypted with RAW_ARCHIVE_KEY

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-04 02:46:48 +00:00
co-authored by Claude Opus 5.5
parent 61bc50583c
commit 3b24a11f5a
64 changed files with 8626 additions and 6909 deletions
+209
View File
@@ -0,0 +1,209 @@
"""The collection pipeline: extract's raw store, every transform, the coders, validation and the generated files."""
import csv
import json
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
from collection import run # noqa: E402
from collection.etl import codegen, extract, report_form, runs, validate # noqa: E402
from collection.etl.common import read_yaml # noqa: E402
from collection.etl.transform import bot, reddit, reports, sheet # noqa: E402
FX = ROOT / "tests" / "fixtures" / "bot"
TMP = Path(tempfile.mkdtemp(prefix="collection-test-"))
def fixture_import(salt="test-salt"):
kn = json.loads((FX / "data/knowledge.json").read_text(encoding="utf-8"))
enc = bot._jsonl((FX / "logs/encounters.jsonl").read_text(encoding="utf-8"))
rn = bot._jsonl((FX / "data/runs.jsonl").read_text(encoding="utf-8"))
return bot.import_bot(kn, enc, rn, salt)
class RawStore(unittest.TestCase):
def setUp(self):
self.old = extract.RAW_DIR
extract.RAW_DIR = TMP / "raw"
shutil.rmtree(extract.RAW_DIR, ignore_errors=True)
def tearDown(self):
extract.RAW_DIR = self.old
def test_current_and_one_previous_only(self):
m = {}
self.assertTrue(extract.store("s", "txt", b"one", m, today="2026-11-01"))
self.assertFalse(extract.store("s", "txt", b"one", m, today="2026-11-02"))
self.assertEqual(m["s"]["changed"], "2026-11-01")
self.assertTrue(extract.store("s", "txt", b"two", m, today="2026-11-03"))
self.assertTrue(extract.store("s", "txt", b"three", m, today="2026-11-04"))
cur, prev = extract.raw_paths("s", "txt")
self.assertEqual((cur.read_bytes(), prev.read_bytes()), (b"three", b"two"))
self.assertEqual(sorted(p.name for p in cur.parent.iterdir()), ["current.txt", "previous.txt"])
self.assertEqual(set(m["s"]), {"sha256", "bytes", "changed"}) # the current version only
def test_missing_secrets_skip_not_fail(self):
with self.assertRaises(extract.Skip):
extract.reddit({"secrets": ["A", "B"], "threads": []}, {})
with self.assertRaises(extract.Skip):
extract.local_dir({"env": "DRAGONKIND_BOT_DIR", "files": []}, {})
class Bot(unittest.TestCase):
def test_same_output_as_the_javascript_importer(self):
scenes, trajs, unlabeled = fixture_import()
expected = json.loads((FX / "expected.json").read_text(encoding="utf-8"))
self.assertEqual(json.dumps(scenes, ensure_ascii=False), json.dumps(expected["scenes"], ensure_ascii=False))
self.assertEqual(json.dumps(trajs, ensure_ascii=False), json.dumps(expected["trajectories"], ensure_ascii=False))
self.assertEqual(unlabeled, expected["unlabeled"])
def test_one_trajectory_per_attempt(self):
_, (t1, t2), _ = fixture_import()
self.assertEqual(len(t1["steps"]), 3)
self.assertEqual(t1["steps"][-1]["to"], "BURNED")
self.assertEqual(t1["steps"][0]["to"], t1["steps"][1]["scene"])
self.assertEqual(t2["steps"][-1]["to"], "BONDED:green")
self.assertTrue(t2["steps"][0]["manual"])
def test_accounts_hashed_and_no_prose(self):
scenes, trajs, _ = fixture_import()
self.assertRegex(trajs[0]["meta"]["accountKey"], r"^[0-9a-f]{10}$")
text = json.dumps({"s": scenes, "t": trajs})
self.assertNotIn("acct-1", text)
self.assertNotIn("Synthetic", text)
P = read_yaml("collection/pipeline.yaml")["sources"]["bot"]["privacy"]
bot.check_public(scenes, set(P["refuse_fields"]), P["max_string"])
with self.assertRaises(ValueError):
bot.check_public({"prose": "x"}, set(P["refuse_fields"]), 200)
with self.assertRaises(ValueError):
bot.check_public({"note": "y" * 201}, set(), 200)
def test_crosswalk_matches_graph_js(self):
"""The bot's advisor maps choices with core/graph.js; the importer with bot.yaml. Same answers."""
trajs = runs.bot_trajectories(read_yaml("collection/pipeline.yaml")["sources"]["bot"])
trajs += json.loads((FX / "expected.json").read_text(encoding="utf-8"))["trajectories"]
script = ("const G=require('./core/graph.js');let s='';process.stdin.on('data',d=>s+=d).on('end',()=>"
"console.log(JSON.stringify(JSON.parse(s).map(t=>G.trajectoryVariables(t).vars))))")
js = json.loads(subprocess.run(["node", "-e", script], input=json.dumps(trajs), capture_output=True,
text=True, cwd=ROOT, check=True).stdout)
import re
cw = [(re.compile(rx), c, v) for rx, c, v in read_yaml("collection/specs/bot.yaml")["crosswalk"]]
for t, j in zip(trajs, js):
py, _ = runs.bot_variables(t, cw)
self.assertEqual(py, {k: v for k, v in j.items() if k not in ("outcome", "outcome_color")})
def test_every_bot_choice_is_mapped(self):
"""Every choice the bot has seen maps onto a column, except answers inside a bonding encounter."""
import re
cw = [re.compile(rx) for rx, _, _ in read_yaml("collection/specs/bot.yaml")["crosswalk"]]
scenes = json.loads((ROOT / "data/bot/scenes.json").read_text(encoding="utf-8"))
unmapped = [f"{s['label']}: {o['text']}" for s in scenes.values()
if len(s["options"]) > 1 and "two Greens in the clearing" not in s["label"]
for o in s["options"] if not any(rx.search(bot.normalize(o["text"])) for rx in cw)]
self.assertEqual(unmapped, [])
def test_graph_js_crosswalk_is_generated(self):
self.assertEqual(codegen.graph_js(), (ROOT / codegen.GRAPH_JS).read_text(encoding="utf-8"))
class Sheets(unittest.TestCase):
def test_privacy_rules(self):
p = TMP / "sheet.csv"
p.write_text("Timestamp,Cadet name,Photo upload,Notes\n2026-10-01,Violet,http://x,mail me at a@b.co\n", encoding="utf-8")
spec = {"label": "t", "transform": {"name": "sheet"},
"privacy": {"redact_emails": True, "drop_columns_matching": ["photo", "cadet name"]}}
out = sheet.transform(p, spec, {"changed": "2026-11-01"})
self.assertEqual(out["headers"], ["Timestamp", "Notes"])
self.assertEqual(out["rows"], [{"Timestamp": "2026-10-01", "Notes": "mail me at [email removed]"}])
self.assertEqual(out["exported"], "2026-11-01")
class Reddit(unittest.TestCase):
def test_new_result_links_only_and_no_usernames(self):
known = read_yaml("collection/manual/reddit_results.yaml")["results"][0]["id"]
tree = [{"kind": "Listing", "data": {"children": [
{"kind": "t1", "data": {"author": "someone", "permalink": "/r/fourthwing/comments/x/y/c1/",
"body": f"got mine! https://dragonkind.com/results/{known} and https://dragonkind.com/results/THRESH-NEW123",
"replies": {"kind": "Listing", "data": {"children": [
{"kind": "t1", "data": {"author": "other", "permalink": "/c2/", "body": "dragonkind.com/results/thresh-new456"}}]}}}}]}}]
p = TMP / "reddit.json"
p.write_text(json.dumps({"x": tree}), encoding="utf-8")
out = reddit.transform(p, {"transform": {"manual": "collection/manual/reddit_results.yaml"}}, {})
self.assertEqual([r["id"] for r in out["new_results"]], ["THRESH-NEW123", "THRESH-NEW456"])
self.assertNotIn("someone", json.dumps(out))
class Reports(unittest.TestCase):
def issue(self, number, answers, **kw):
form = report_form.spec()
by_id = {f["id"]: f for f in form["fields"]}
body = "\n\n".join(f"### {f['label']}\n\n{answers.get(f['id'], '_No response_')}" for f in by_id.values())
return {"number": number, "body": body, "created_at": "2026-11-05T10:00:00Z", "user": {"login": "rider"},
"labels": [{"name": "run report"}], "state": "open", **kw}
def test_issue_form_is_generated(self):
self.assertEqual(report_form.render(), (ROOT / report_form.TEMPLATE).read_text(encoding="utf-8"))
def test_every_option_is_a_dictionary_value(self):
cols = read_yaml("data/dictionary.yaml")["columns"]
for f in report_form.spec()["fields"]:
self.assertIn(f["column"], cols)
for v in (f.get("options") or {}).values():
if v is not None and "values" in cols[f["column"]]:
self.assertIn(str(v), cols[f["column"]]["values"], f["id"])
def test_reports_become_valid_runs(self):
good = self.issue(7, {"company": "Alone", "direction": "Toward the forest centre", "first_sighting": "Red",
"scream": "Investigated, and helped my squadmate", "branch": "Valley: passed the Red, the meadow",
"outcome": "Bonded", "outcome_color": "Blue", "tail": "Clubtail", "attempt": "3rd"})
spam = self.issue(8, {"company": "Alone"}, state="closed", state_reason="not_planned")
dup = self.issue(9, {"company": "Alone"}, labels=[{"name": "run report"}, {"name": "duplicate"}])
raw = TMP / "issues.json"
raw.write_text(json.dumps([good, spam, dup]), encoding="utf-8")
spec = read_yaml("collection/pipeline.yaml")["sources"]["reported"]
out = reports.transform(raw, spec, {})
self.assertEqual([r["number"] for r in out["reports"]], [7])
self.assertNotIn("rider", json.dumps(out))
target = TMP / "reports.json"
target.write_text(json.dumps(out), encoding="utf-8")
rows = runs.reports("reported", {**spec, "load": str(target)})
r = rows[0]
self.assertEqual((r["run_id"], r["source"], r["date"]), ("reported-7", "reported", "2026-11-05"))
self.assertEqual((r["company"], r["direction"], r["hour"], r["attempt"]), ("Alone", "Center", "Valley", 3))
self.assertEqual((r["outcome"], r["outcome_color"], r["enc_Blue"], r["saw_blue_meadow"]), ("Bonded", "Blue", "Bonded!", True))
self.assertEqual(validate.problems(rows, runs.DICT), [])
class Build(unittest.TestCase):
def test_published_files_are_up_to_date(self):
self.assertEqual(run.build(check=True), [], "run `python -m collection build`")
def test_every_source_value_is_described(self):
pipe = read_yaml("collection/pipeline.yaml")
described = runs.DICT["columns"]["source"]["values"]
for name, spec in pipe["sources"].items():
if spec.get("run_source"):
self.assertIn(spec["run_source"], described, name)
def test_validation_catches_unknown_values(self):
row = runs.blank_row(run_id="x", source="reported", company="Five squadmates")
self.assertTrue(any("company" in p for p in validate.problems([row], runs.DICT)))
self.assertTrue(any("unique" in p for p in validate.problems([row, row], runs.DICT)))
def test_spreadsheet_rows_keep_their_entry_label(self):
with open(ROOT / "data/public/runs_clean.csv", encoding="utf-8", newline="") as f:
rows = list(csv.DictReader(f))
sheet_rows = [r for r in rows if r["source"] == "reddit spreadsheet"]
self.assertTrue(sheet_rows and all(r["entry"] and r["run_id"].startswith("sheet-") for r in sheet_rows))
self.assertTrue(all(not r["entry"] for r in rows if r["source"] != "reddit spreadsheet"))
if __name__ == "__main__":
unittest.main()
+3 -31
View File
@@ -2,39 +2,11 @@
const test = require("node:test");
const assert = require("node:assert");
const path = require("path");
const { importBot, assertPublicSafe } = require("../../collection/bot/bot");
const G = require("../../core/graph");
const fx = path.join(__dirname, "../fixtures/bot");
const { scenes, trajectories } = importBot({ dataDir: path.join(fx, "data"), logsDir: path.join(fx, "logs"), salt: "test-salt" });
test("one trajectory per attempt, duplicates and repeated burns collapsed", () => {
assert.strictEqual(trajectories.length, 2);
const [t1, t2] = trajectories;
assert.strictEqual(t1.steps.length, 3);
assert.deepStrictEqual(t1.steps.map((s) => s.to).slice(-1), ["BURNED"]);
assert.strictEqual(t1.steps[0].to, t1.steps[1].scene);
assert.strictEqual(t2.outcome.type, "BONDED");
assert.strictEqual(t2.steps.at(-1).to, "BONDED:green");
assert.strictEqual(t2.steps[0].manual, true, "manual picks are flagged");
});
test("run metadata attached, account id hashed with the salt", () => {
const t1 = trajectories[0];
assert.strictEqual(t1.meta.target, "black");
assert.strictEqual(t1.meta.decisionMode, "explore");
assert.strictEqual(t1.outcome.intentional, true);
assert.match(t1.meta.accountKey, /^[0-9a-f]{10}$/);
assert.ok(!JSON.stringify(trajectories).includes("acct-1"));
});
test("public export contains no prose", () => {
assertPublicSafe(scenes);
assertPublicSafe(trajectories);
assert.ok(!JSON.stringify({ scenes, trajectories }).includes("Synthetic"));
assert.throws(() => assertPublicSafe({ prose: "x" }));
assert.throws(() => assertPublicSafe({ note: "y".repeat(201) }));
});
// the bot importer's output on tests/fixtures/bot (the importer is Python now: tests/collection/test_bot.py checks
// that it still produces exactly this file)
const { scenes, trajectories } = require(path.join(__dirname, "../fixtures/bot/expected.json"));
test("crosswalk maps bot choices to form variables", () => {
const { vars } = G.trajectoryVariables(trajectories[0]);
+5 -6
View File
@@ -1,21 +1,20 @@
"""The data dictionary describes every published column and value, and data/croissant.json matches it."""
import csv
import importlib.util
import json
import re
import sys
import unittest
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[2]
spec = importlib.util.spec_from_file_location("publish_metadata", ROOT / "collection" / "publish_metadata.py")
publish_metadata = importlib.util.module_from_spec(spec)
spec.loader.exec_module(publish_metadata)
sys.path.insert(0, str(ROOT))
from collection.etl import metadata as publish_metadata # noqa: E402
with open(ROOT / "data" / "dictionary.yaml", encoding="utf-8") as f:
DICT = yaml.safe_load(f)
with open(ROOT / "data" / "derived" / "runs_clean.csv", encoding="utf-8", newline="") as f:
with open(ROOT / "data" / "public" / "runs_clean.csv", encoding="utf-8", newline="") as f:
reader = csv.DictReader(f)
HEADER = reader.fieldnames
ROWS = list(reader)
@@ -61,7 +60,7 @@ class Croissant(unittest.TestCase):
def test_croissant_is_up_to_date(self):
expected = publish_metadata.render(publish_metadata.build(DICT))
actual = (ROOT / "data" / "croissant.json").read_text(encoding="utf-8")
self.assertEqual(actual, expected, "run python3 collection/publish_metadata.py")
self.assertEqual(actual, expected, "run python -m collection build")
def test_fields_and_checksums(self):
doc = json.loads((ROOT / "data" / "croissant.json").read_text(encoding="utf-8"))
+178
View File
@@ -0,0 +1,178 @@
{
"scenes": {
"b0fa671066ec": {
"stage": "TEST",
"tint": null,
"backdrop": null,
"label": "TEST",
"options": [
{
"id": "c18f80dd1e",
"text": "Enter the valley alone"
},
{
"id": "3da794f136",
"text": "Enter the valley with one of your squadmates"
}
]
},
"e0b5530601be": {
"stage": "TEST",
"tint": null,
"backdrop": null,
"label": "TEST",
"options": [
{
"id": "5f7eefac34",
"text": "Head east toward the roar"
},
{
"id": "05d19abb5d",
"text": "Head toward the center of the forest"
}
]
},
"1c86a35c091f": {
"stage": "TEST",
"tint": null,
"backdrop": null,
"label": "TEST",
"options": [
{
"id": "a31cbcb659",
"text": "Lower your eyes in respect"
},
{
"id": "1859b3cd4a",
"text": "Hide behind a tree"
}
]
}
},
"trajectories": [
{
"id": "bot-9176f0ecbe",
"source": "bot",
"meta": {
"attempt": 1,
"startedAt": "2026-01-01T00:00:00.000Z",
"accountKey": "7bd9b15fe8",
"target": "black",
"decisionMode": "explore",
"policyId": null,
"experimentId": null,
"arm": null
},
"steps": [
{
"scene": "b0fa671066ec",
"stage": "TEST",
"tint": null,
"option": "c18f80dd1e",
"optionText": "Enter the valley alone",
"offered": [
"c18f80dd1e",
"3da794f136"
],
"rule": "R",
"decisionMode": "guide",
"manual": false,
"at": "2026-01-01T00:00:00.000Z",
"to": "e0b5530601be"
},
{
"scene": "e0b5530601be",
"stage": "TEST",
"tint": null,
"option": "05d19abb5d",
"optionText": "Head toward the center of the forest",
"offered": [
"5f7eefac34",
"05d19abb5d"
],
"rule": "R",
"decisionMode": "guide",
"manual": false,
"at": "2026-01-01T00:00:02.000Z",
"to": "1c86a35c091f"
},
{
"scene": "1c86a35c091f",
"stage": "TEST",
"tint": null,
"option": "1859b3cd4a",
"optionText": "Hide behind a tree",
"offered": [
"a31cbcb659",
"1859b3cd4a"
],
"rule": "R",
"decisionMode": "guide",
"manual": false,
"at": "2026-01-01T00:00:03.000Z",
"to": "BURNED"
}
],
"outcome": {
"type": "BURNED",
"colour": null,
"intentional": true
}
},
{
"id": "bot-b34ab7e2df",
"source": "bot",
"meta": {
"attempt": 2,
"startedAt": "2026-01-01T00:00:07.000Z",
"accountKey": null,
"target": null,
"decisionMode": null
},
"steps": [
{
"scene": "b0fa671066ec",
"stage": "TEST",
"tint": null,
"option": "3da794f136",
"optionText": "Enter the valley with one of your squadmates",
"offered": [
"c18f80dd1e",
"3da794f136"
],
"rule": null,
"decisionMode": "manual",
"manual": true,
"at": "2026-01-01T00:00:07.000Z",
"to": "e0b5530601be"
},
{
"scene": "e0b5530601be",
"stage": "TEST",
"tint": null,
"option": "5f7eefac34",
"optionText": "Head east toward the roar",
"offered": [
"5f7eefac34",
"05d19abb5d"
],
"rule": "R",
"decisionMode": "guide",
"manual": false,
"at": "2026-01-01T00:00:08.000Z",
"to": "BONDED:green"
}
],
"outcome": {
"type": "BONDED",
"colour": "green",
"intentional": false
}
}
],
"unlabeled": [
"b0fa671066ec (TEST)",
"e0b5530601be (TEST)",
"1c86a35c091f (TEST)"
]
}