Collection as an ETL pipeline; runs_clean.csv from every source; Report a run
- collection/pipeline.yaml declares every source (extract kind, transform, privacy rules, load target, run source); `python -m collection update|build` - extract keeps data/raw/<source>/current + previous only (git-ignored); data/manifest.json records the current raw checksum and change date - transforms are pure; validate checks runs against data/dictionary.yaml before load; load writes byte-stable files only when they change - data/public/runs_clean.csv: every run from every source with a new `source` column (reddit spreadsheet, reddit thread, reported, bot, reddit pdf); the sheet's own row label moves to `entry`; run_id is text - bot importer ported to Python (same output as bot.js on the fixture); labels and crosswalk in collection/specs/bot.yaml, written into graph.js - Reddit through the official API (result links to add by hand); the 12 result pages move to collection/manual/reddit_results.yaml - "Report a run": a GitHub issue form generated from collection/specs/report_form.yaml, collected as source "reported", plus a site tab pointing to it, the community spreadsheet and the megathread - analysis and the route model read runs_clean.csv (sources in measures.yaml); stats.json and policy.json unchanged apart from the factor's new name - daily workflow runs the pipeline; the raw store is kept between runs in the Actions cache, encrypted with RAW_ARCHIVE_KEY Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
61bc50583c
commit
3b24a11f5a
@@ -0,0 +1,209 @@
|
||||
"""The collection pipeline: extract's raw store, every transform, the coders, validation and the generated files."""
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from collection import run # noqa: E402
|
||||
from collection.etl import codegen, extract, report_form, runs, validate # noqa: E402
|
||||
from collection.etl.common import read_yaml # noqa: E402
|
||||
from collection.etl.transform import bot, reddit, reports, sheet # noqa: E402
|
||||
|
||||
FX = ROOT / "tests" / "fixtures" / "bot"
|
||||
TMP = Path(tempfile.mkdtemp(prefix="collection-test-"))
|
||||
|
||||
|
||||
def fixture_import(salt="test-salt"):
|
||||
kn = json.loads((FX / "data/knowledge.json").read_text(encoding="utf-8"))
|
||||
enc = bot._jsonl((FX / "logs/encounters.jsonl").read_text(encoding="utf-8"))
|
||||
rn = bot._jsonl((FX / "data/runs.jsonl").read_text(encoding="utf-8"))
|
||||
return bot.import_bot(kn, enc, rn, salt)
|
||||
|
||||
|
||||
class RawStore(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.old = extract.RAW_DIR
|
||||
extract.RAW_DIR = TMP / "raw"
|
||||
shutil.rmtree(extract.RAW_DIR, ignore_errors=True)
|
||||
|
||||
def tearDown(self):
|
||||
extract.RAW_DIR = self.old
|
||||
|
||||
def test_current_and_one_previous_only(self):
|
||||
m = {}
|
||||
self.assertTrue(extract.store("s", "txt", b"one", m, today="2026-11-01"))
|
||||
self.assertFalse(extract.store("s", "txt", b"one", m, today="2026-11-02"))
|
||||
self.assertEqual(m["s"]["changed"], "2026-11-01")
|
||||
self.assertTrue(extract.store("s", "txt", b"two", m, today="2026-11-03"))
|
||||
self.assertTrue(extract.store("s", "txt", b"three", m, today="2026-11-04"))
|
||||
cur, prev = extract.raw_paths("s", "txt")
|
||||
self.assertEqual((cur.read_bytes(), prev.read_bytes()), (b"three", b"two"))
|
||||
self.assertEqual(sorted(p.name for p in cur.parent.iterdir()), ["current.txt", "previous.txt"])
|
||||
self.assertEqual(set(m["s"]), {"sha256", "bytes", "changed"}) # the current version only
|
||||
|
||||
def test_missing_secrets_skip_not_fail(self):
|
||||
with self.assertRaises(extract.Skip):
|
||||
extract.reddit({"secrets": ["A", "B"], "threads": []}, {})
|
||||
with self.assertRaises(extract.Skip):
|
||||
extract.local_dir({"env": "DRAGONKIND_BOT_DIR", "files": []}, {})
|
||||
|
||||
|
||||
class Bot(unittest.TestCase):
|
||||
def test_same_output_as_the_javascript_importer(self):
|
||||
scenes, trajs, unlabeled = fixture_import()
|
||||
expected = json.loads((FX / "expected.json").read_text(encoding="utf-8"))
|
||||
self.assertEqual(json.dumps(scenes, ensure_ascii=False), json.dumps(expected["scenes"], ensure_ascii=False))
|
||||
self.assertEqual(json.dumps(trajs, ensure_ascii=False), json.dumps(expected["trajectories"], ensure_ascii=False))
|
||||
self.assertEqual(unlabeled, expected["unlabeled"])
|
||||
|
||||
def test_one_trajectory_per_attempt(self):
|
||||
_, (t1, t2), _ = fixture_import()
|
||||
self.assertEqual(len(t1["steps"]), 3)
|
||||
self.assertEqual(t1["steps"][-1]["to"], "BURNED")
|
||||
self.assertEqual(t1["steps"][0]["to"], t1["steps"][1]["scene"])
|
||||
self.assertEqual(t2["steps"][-1]["to"], "BONDED:green")
|
||||
self.assertTrue(t2["steps"][0]["manual"])
|
||||
|
||||
def test_accounts_hashed_and_no_prose(self):
|
||||
scenes, trajs, _ = fixture_import()
|
||||
self.assertRegex(trajs[0]["meta"]["accountKey"], r"^[0-9a-f]{10}$")
|
||||
text = json.dumps({"s": scenes, "t": trajs})
|
||||
self.assertNotIn("acct-1", text)
|
||||
self.assertNotIn("Synthetic", text)
|
||||
P = read_yaml("collection/pipeline.yaml")["sources"]["bot"]["privacy"]
|
||||
bot.check_public(scenes, set(P["refuse_fields"]), P["max_string"])
|
||||
with self.assertRaises(ValueError):
|
||||
bot.check_public({"prose": "x"}, set(P["refuse_fields"]), 200)
|
||||
with self.assertRaises(ValueError):
|
||||
bot.check_public({"note": "y" * 201}, set(), 200)
|
||||
|
||||
def test_crosswalk_matches_graph_js(self):
|
||||
"""The bot's advisor maps choices with core/graph.js; the importer with bot.yaml. Same answers."""
|
||||
trajs = runs.bot_trajectories(read_yaml("collection/pipeline.yaml")["sources"]["bot"])
|
||||
trajs += json.loads((FX / "expected.json").read_text(encoding="utf-8"))["trajectories"]
|
||||
script = ("const G=require('./core/graph.js');let s='';process.stdin.on('data',d=>s+=d).on('end',()=>"
|
||||
"console.log(JSON.stringify(JSON.parse(s).map(t=>G.trajectoryVariables(t).vars))))")
|
||||
js = json.loads(subprocess.run(["node", "-e", script], input=json.dumps(trajs), capture_output=True,
|
||||
text=True, cwd=ROOT, check=True).stdout)
|
||||
import re
|
||||
cw = [(re.compile(rx), c, v) for rx, c, v in read_yaml("collection/specs/bot.yaml")["crosswalk"]]
|
||||
for t, j in zip(trajs, js):
|
||||
py, _ = runs.bot_variables(t, cw)
|
||||
self.assertEqual(py, {k: v for k, v in j.items() if k not in ("outcome", "outcome_color")})
|
||||
|
||||
def test_every_bot_choice_is_mapped(self):
|
||||
"""Every choice the bot has seen maps onto a column, except answers inside a bonding encounter."""
|
||||
import re
|
||||
cw = [re.compile(rx) for rx, _, _ in read_yaml("collection/specs/bot.yaml")["crosswalk"]]
|
||||
scenes = json.loads((ROOT / "data/bot/scenes.json").read_text(encoding="utf-8"))
|
||||
unmapped = [f"{s['label']}: {o['text']}" for s in scenes.values()
|
||||
if len(s["options"]) > 1 and "two Greens in the clearing" not in s["label"]
|
||||
for o in s["options"] if not any(rx.search(bot.normalize(o["text"])) for rx in cw)]
|
||||
self.assertEqual(unmapped, [])
|
||||
|
||||
def test_graph_js_crosswalk_is_generated(self):
|
||||
self.assertEqual(codegen.graph_js(), (ROOT / codegen.GRAPH_JS).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
class Sheets(unittest.TestCase):
|
||||
def test_privacy_rules(self):
|
||||
p = TMP / "sheet.csv"
|
||||
p.write_text("Timestamp,Cadet name,Photo upload,Notes\n2026-10-01,Violet,http://x,mail me at a@b.co\n", encoding="utf-8")
|
||||
spec = {"label": "t", "transform": {"name": "sheet"},
|
||||
"privacy": {"redact_emails": True, "drop_columns_matching": ["photo", "cadet name"]}}
|
||||
out = sheet.transform(p, spec, {"changed": "2026-11-01"})
|
||||
self.assertEqual(out["headers"], ["Timestamp", "Notes"])
|
||||
self.assertEqual(out["rows"], [{"Timestamp": "2026-10-01", "Notes": "mail me at [email removed]"}])
|
||||
self.assertEqual(out["exported"], "2026-11-01")
|
||||
|
||||
|
||||
class Reddit(unittest.TestCase):
|
||||
def test_new_result_links_only_and_no_usernames(self):
|
||||
known = read_yaml("collection/manual/reddit_results.yaml")["results"][0]["id"]
|
||||
tree = [{"kind": "Listing", "data": {"children": [
|
||||
{"kind": "t1", "data": {"author": "someone", "permalink": "/r/fourthwing/comments/x/y/c1/",
|
||||
"body": f"got mine! https://dragonkind.com/results/{known} and https://dragonkind.com/results/THRESH-NEW123",
|
||||
"replies": {"kind": "Listing", "data": {"children": [
|
||||
{"kind": "t1", "data": {"author": "other", "permalink": "/c2/", "body": "dragonkind.com/results/thresh-new456"}}]}}}}]}}]
|
||||
p = TMP / "reddit.json"
|
||||
p.write_text(json.dumps({"x": tree}), encoding="utf-8")
|
||||
out = reddit.transform(p, {"transform": {"manual": "collection/manual/reddit_results.yaml"}}, {})
|
||||
self.assertEqual([r["id"] for r in out["new_results"]], ["THRESH-NEW123", "THRESH-NEW456"])
|
||||
self.assertNotIn("someone", json.dumps(out))
|
||||
|
||||
|
||||
class Reports(unittest.TestCase):
|
||||
def issue(self, number, answers, **kw):
|
||||
form = report_form.spec()
|
||||
by_id = {f["id"]: f for f in form["fields"]}
|
||||
body = "\n\n".join(f"### {f['label']}\n\n{answers.get(f['id'], '_No response_')}" for f in by_id.values())
|
||||
return {"number": number, "body": body, "created_at": "2026-11-05T10:00:00Z", "user": {"login": "rider"},
|
||||
"labels": [{"name": "run report"}], "state": "open", **kw}
|
||||
|
||||
def test_issue_form_is_generated(self):
|
||||
self.assertEqual(report_form.render(), (ROOT / report_form.TEMPLATE).read_text(encoding="utf-8"))
|
||||
|
||||
def test_every_option_is_a_dictionary_value(self):
|
||||
cols = read_yaml("data/dictionary.yaml")["columns"]
|
||||
for f in report_form.spec()["fields"]:
|
||||
self.assertIn(f["column"], cols)
|
||||
for v in (f.get("options") or {}).values():
|
||||
if v is not None and "values" in cols[f["column"]]:
|
||||
self.assertIn(str(v), cols[f["column"]]["values"], f["id"])
|
||||
|
||||
def test_reports_become_valid_runs(self):
|
||||
good = self.issue(7, {"company": "Alone", "direction": "Toward the forest centre", "first_sighting": "Red",
|
||||
"scream": "Investigated, and helped my squadmate", "branch": "Valley: passed the Red, the meadow",
|
||||
"outcome": "Bonded", "outcome_color": "Blue", "tail": "Clubtail", "attempt": "3rd"})
|
||||
spam = self.issue(8, {"company": "Alone"}, state="closed", state_reason="not_planned")
|
||||
dup = self.issue(9, {"company": "Alone"}, labels=[{"name": "run report"}, {"name": "duplicate"}])
|
||||
raw = TMP / "issues.json"
|
||||
raw.write_text(json.dumps([good, spam, dup]), encoding="utf-8")
|
||||
spec = read_yaml("collection/pipeline.yaml")["sources"]["reported"]
|
||||
out = reports.transform(raw, spec, {})
|
||||
self.assertEqual([r["number"] for r in out["reports"]], [7])
|
||||
self.assertNotIn("rider", json.dumps(out))
|
||||
target = TMP / "reports.json"
|
||||
target.write_text(json.dumps(out), encoding="utf-8")
|
||||
rows = runs.reports("reported", {**spec, "load": str(target)})
|
||||
r = rows[0]
|
||||
self.assertEqual((r["run_id"], r["source"], r["date"]), ("reported-7", "reported", "2026-11-05"))
|
||||
self.assertEqual((r["company"], r["direction"], r["hour"], r["attempt"]), ("Alone", "Center", "Valley", 3))
|
||||
self.assertEqual((r["outcome"], r["outcome_color"], r["enc_Blue"], r["saw_blue_meadow"]), ("Bonded", "Blue", "Bonded!", True))
|
||||
self.assertEqual(validate.problems(rows, runs.DICT), [])
|
||||
|
||||
|
||||
class Build(unittest.TestCase):
|
||||
def test_published_files_are_up_to_date(self):
|
||||
self.assertEqual(run.build(check=True), [], "run `python -m collection build`")
|
||||
|
||||
def test_every_source_value_is_described(self):
|
||||
pipe = read_yaml("collection/pipeline.yaml")
|
||||
described = runs.DICT["columns"]["source"]["values"]
|
||||
for name, spec in pipe["sources"].items():
|
||||
if spec.get("run_source"):
|
||||
self.assertIn(spec["run_source"], described, name)
|
||||
|
||||
def test_validation_catches_unknown_values(self):
|
||||
row = runs.blank_row(run_id="x", source="reported", company="Five squadmates")
|
||||
self.assertTrue(any("company" in p for p in validate.problems([row], runs.DICT)))
|
||||
self.assertTrue(any("unique" in p for p in validate.problems([row, row], runs.DICT)))
|
||||
|
||||
def test_spreadsheet_rows_keep_their_entry_label(self):
|
||||
with open(ROOT / "data/public/runs_clean.csv", encoding="utf-8", newline="") as f:
|
||||
rows = list(csv.DictReader(f))
|
||||
sheet_rows = [r for r in rows if r["source"] == "reddit spreadsheet"]
|
||||
self.assertTrue(sheet_rows and all(r["entry"] and r["run_id"].startswith("sheet-") for r in sheet_rows))
|
||||
self.assertTrue(all(not r["entry"] for r in rows if r["source"] != "reddit spreadsheet"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user