- forge.yaml: the repository's host (Gitea at git.kenschiano.net, limited visibility); the "Report a run" collector, the page's links (data/public/links.json, filled in by site/build.js) and the dataset metadata read it - collection/etl/forge.py: issues, pull requests (merge when checks succeed) and labels for Gitea and GitHub; `python -m collection forge ...` - .gitea/workflows: CI and the daily update (pushes a branch, opens a pull request that merges itself, opens an issue when a source breaks); .github/workflows kept for GitHub - docs: Gitea setup (REPO_SETTINGS), the bot reading atlas files from the repository Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
313 lines
17 KiB
Python
313 lines
17 KiB
Python
"""The collection pipeline: extract's raw store, every transform, the coders, validation and the generated files."""
|
|
import csv
|
|
import json
|
|
import re
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from collection import run # noqa: E402
|
|
from collection.etl import codegen, extract, report_form, runs, validate # noqa: E402
|
|
from collection.etl.common import read_yaml # noqa: E402
|
|
from collection.etl.transform import bot, reddit, reports, sheet # noqa: E402
|
|
|
|
FX = ROOT / "tests" / "fixtures" / "bot"
|
|
TMP = Path(tempfile.mkdtemp(prefix="collection-test-"))
|
|
|
|
|
|
def fixture_import(salt="test-salt"):
|
|
kn = json.loads((FX / "data/knowledge.json").read_text(encoding="utf-8"))
|
|
enc = bot._jsonl((FX / "logs/encounters.jsonl").read_text(encoding="utf-8"))
|
|
rn = bot._jsonl((FX / "data/runs.jsonl").read_text(encoding="utf-8"))
|
|
return bot.import_bot(kn, enc, rn, salt)
|
|
|
|
|
|
class RawStore(unittest.TestCase):
|
|
def setUp(self):
|
|
self.old = extract.RAW_DIR
|
|
extract.RAW_DIR = TMP / "raw"
|
|
shutil.rmtree(extract.RAW_DIR, ignore_errors=True)
|
|
|
|
def tearDown(self):
|
|
extract.RAW_DIR = self.old
|
|
|
|
def test_current_and_one_previous_only(self):
|
|
m = {}
|
|
self.assertTrue(extract.store("s", "txt", b"one", m, today="2026-11-01"))
|
|
self.assertFalse(extract.store("s", "txt", b"one", m, today="2026-11-02"))
|
|
self.assertEqual(m["s"]["changed"], "2026-11-01")
|
|
self.assertTrue(extract.store("s", "txt", b"two", m, today="2026-11-03"))
|
|
self.assertTrue(extract.store("s", "txt", b"three", m, today="2026-11-04"))
|
|
cur, prev = extract.raw_paths("s", "txt")
|
|
self.assertEqual((cur.read_bytes(), prev.read_bytes()), (b"three", b"two"))
|
|
self.assertEqual(sorted(p.name for p in cur.parent.iterdir()), ["current.txt", "previous.txt"])
|
|
self.assertEqual(set(m["s"]), {"sha256", "bytes", "changed"}) # the current version only
|
|
|
|
def test_missing_secrets_skip_not_fail(self):
|
|
with self.assertRaises(extract.Skip):
|
|
extract.reddit({"secrets": ["A", "B"], "threads": []}, {})
|
|
with self.assertRaises(extract.Skip):
|
|
extract.local_dir({"env": "DRAGONKIND_BOT_DIR", "files": []}, {})
|
|
|
|
|
|
class Bot(unittest.TestCase):
|
|
def test_same_output_as_the_javascript_importer(self):
|
|
scenes, trajs, unlabeled = fixture_import()
|
|
expected = json.loads((FX / "expected.json").read_text(encoding="utf-8"))
|
|
self.assertEqual(json.dumps(scenes, ensure_ascii=False), json.dumps(expected["scenes"], ensure_ascii=False))
|
|
self.assertEqual(json.dumps(trajs, ensure_ascii=False), json.dumps(expected["trajectories"], ensure_ascii=False))
|
|
self.assertEqual(unlabeled, expected["unlabeled"])
|
|
|
|
def test_one_trajectory_per_attempt(self):
|
|
_, (t1, t2), _ = fixture_import()
|
|
self.assertEqual(len(t1["steps"]), 3)
|
|
self.assertEqual(t1["steps"][-1]["to"], "BURNED")
|
|
self.assertEqual(t1["steps"][0]["to"], t1["steps"][1]["scene"])
|
|
self.assertEqual(t2["steps"][-1]["to"], "BONDED:green")
|
|
self.assertTrue(t2["steps"][0]["manual"])
|
|
|
|
def test_accounts_hashed_and_no_prose(self):
|
|
scenes, trajs, _ = fixture_import()
|
|
self.assertRegex(trajs[0]["meta"]["accountKey"], r"^[0-9a-f]{10}$")
|
|
text = json.dumps({"s": scenes, "t": trajs})
|
|
self.assertNotIn("acct-1", text)
|
|
self.assertNotIn("Synthetic", text)
|
|
P = read_yaml("collection/pipeline.yaml")["sources"]["bot"]["privacy"]
|
|
bot.check_public(scenes, set(P["refuse_fields"]), P["max_string"])
|
|
with self.assertRaises(ValueError):
|
|
bot.check_public({"prose": "x"}, set(P["refuse_fields"]), 200)
|
|
with self.assertRaises(ValueError):
|
|
bot.check_public({"note": "y" * 201}, set(), 200)
|
|
|
|
def test_crosswalk_matches_graph_js(self):
|
|
"""The bot's advisor maps choices with core/graph.js; the importer with bot.yaml. Same answers."""
|
|
trajs = runs.bot_trajectories(read_yaml("collection/pipeline.yaml")["sources"]["bot"])
|
|
trajs += json.loads((FX / "expected.json").read_text(encoding="utf-8"))["trajectories"]
|
|
script = ("const G=require('./core/graph.js');let s='';process.stdin.on('data',d=>s+=d).on('end',()=>"
|
|
"console.log(JSON.stringify(JSON.parse(s).map(t=>G.trajectoryVariables(t).vars))))")
|
|
js = json.loads(subprocess.run(["node", "-e", script], input=json.dumps(trajs), capture_output=True,
|
|
text=True, cwd=ROOT, check=True).stdout)
|
|
import re
|
|
cw = [(re.compile(rx), c, v) for rx, c, v in read_yaml("collection/specs/bot.yaml")["crosswalk"]]
|
|
for t, j in zip(trajs, js):
|
|
py, _ = runs.bot_variables(t, cw)
|
|
self.assertEqual(py, {k: v for k, v in j.items() if k not in ("outcome", "outcome_color")})
|
|
|
|
def test_every_bot_choice_is_mapped(self):
|
|
"""Every route choice the bot has seen before the hour maps onto a column (answers inside the bonding
|
|
encounters are not route choices), unless bot.yaml lists it as not coded."""
|
|
import re
|
|
spec = read_yaml("collection/specs/bot.yaml")
|
|
cw = [re.compile(rx) for rx, _, _ in spec["crosswalk"]] + [re.compile(rx) for rx in spec.get("not_coded", [])]
|
|
scenes = json.loads((ROOT / "data/bot/scenes.json").read_text(encoding="utf-8"))
|
|
route = {"ENTER_THRESHING", "FIRST_SIGHTING", "SCREAM_AND_CADETS"}
|
|
unmapped = [f"{s['label']}: {o['text']}" for s in scenes.values()
|
|
if len(s["options"]) > 1 and s["stage"] in route and "two Greens in the clearing" not in s["label"]
|
|
for o in s["options"] if not any(rx.search(bot.normalize(o["text"])) for rx in cw)]
|
|
self.assertEqual(unmapped, [])
|
|
|
|
def test_runs_with_steps_and_accounts(self):
|
|
"""The bot's newer run records (one folder per account) become trajectories without account names or
|
|
prose; the top-level files mirror the first account and are skipped."""
|
|
run = {"runId": "run-1", "attempt": 3, "accountAttempt": 3, "accountId": "someone-example-com",
|
|
"accountRole": "research", "startedAt": "2026-11-09T22:02:14.430Z", "targetDragon": "black",
|
|
"decisionMode": "atlas_policy", "policyId": "atlas:x", "outcome": "BURNED", "intentionalBurn": True,
|
|
"burn": {"reason": "TREE_NO_SHADOW_AUTO_BURN"}, "dragonColor": None,
|
|
"steps": [{"nodeId": "a68f681076c6", "stage": "ENTER_THRESHING", "choiceText": "Find the tallest tree and climb for a better view",
|
|
"offered": [{"text": "Find the tallest tree and climb for a better view"}, {"text": "Continue toward the center of the valley"}],
|
|
"observed": {"shadow": False, "dragons": ["Green"]}, "propensity": 0.5, "toKey": "TERMINAL:BURNED",
|
|
"label": "x" * 300, "prose": "game text"}]}
|
|
line = json.dumps(run) + "\n"
|
|
files = {"data/runs.jsonl": line, "data/accounts/someone-example-com/runs.jsonl": line,
|
|
"logs/accounts/someone-example-com/encounters.jsonl": ""}
|
|
accts = bot._account_files(files)
|
|
self.assertEqual(list(accts), ["someone-example-com"])
|
|
t = bot.trajectory_from_run(run, "abc123")
|
|
self.assertEqual(t["steps"][0]["seen"], {"dragons": ["green"], "shadow": False})
|
|
self.assertEqual((t["steps"][0]["to"], t["meta"]["accountKey"], t["outcome"]["reason"]), ("BURNED", "abc123", "TREE_NO_SHADOW_AUTO_BURN"))
|
|
text = json.dumps(t)
|
|
self.assertNotIn("someone", text)
|
|
self.assertNotIn("game text", text)
|
|
bot.check_public(t, {"prose", "accountId"}, 200)
|
|
|
|
def test_graph_js_crosswalk_is_generated(self):
|
|
self.assertEqual(codegen.graph_js(), (ROOT / codegen.GRAPH_JS).read_text(encoding="utf-8"))
|
|
|
|
|
|
class Handoff(unittest.TestCase):
|
|
def test_bot_zip_imports_like_the_folder(self):
|
|
import zipfile
|
|
spec = read_yaml("collection/pipeline.yaml")["sources"]["bot"]["extract"]
|
|
z = TMP / "ken.zip"
|
|
with zipfile.ZipFile(z, "w") as f: # files under a top folder, as a zipped repo would have them
|
|
for rel in spec["files"]:
|
|
if "*" not in rel:
|
|
f.write(FX / rel, f"DragonKind-main/{rel}")
|
|
self.assertEqual(extract.pack_local(spec, z), extract.pack_local(spec, FX))
|
|
empty = TMP / "empty.zip"
|
|
with zipfile.ZipFile(empty, "w") as f:
|
|
f.writestr("notes.txt", "hi")
|
|
with self.assertRaises(RuntimeError):
|
|
extract.pack_local(spec, empty)
|
|
|
|
def test_bot_kit(self):
|
|
import zipfile
|
|
from collection.etl import handoff
|
|
zpath = handoff.export_bot_kit(str(TMP / "exports"))
|
|
with zipfile.ZipFile(zpath) as z:
|
|
self.assertEqual(sorted(z.namelist()), ["README.txt", "graph.js", "policy.json", "solver.js"])
|
|
self.assertEqual(z.read("solver.js"), (ROOT / "core/solver.js").read_bytes())
|
|
self.assertEqual(json.loads(z.read("policy.json"))["id"],
|
|
json.loads((ROOT / "data/public/policy.json").read_text(encoding="utf-8"))["id"])
|
|
|
|
|
|
class Forge(unittest.TestCase):
|
|
GITEA = {"kind": "gitea", "url": "https://git.example.net/", "repo": "me/threshing-atlas", "branch": "main"}
|
|
GITHUB = {"kind": "github", "url": "https://github.com", "repo": "me/threshing-atlas", "branch": "main"}
|
|
|
|
def test_links(self):
|
|
from collection.etl import forge
|
|
self.assertEqual(forge.file_url("data/x.csv", self.GITEA), "https://git.example.net/me/threshing-atlas/raw/branch/main/data/x.csv")
|
|
self.assertEqual(forge.file_url("data/x.csv", self.GITHUB), "https://raw.githubusercontent.com/me/threshing-atlas/main/data/x.csv")
|
|
self.assertIn("template=.github/ISSUE_TEMPLATE/report-a-run.yml", forge.new_issue_url(".github/ISSUE_TEMPLATE/report-a-run.yml", self.GITEA))
|
|
self.assertTrue(forge.new_issue_url(".github/ISSUE_TEMPLATE/report-a-run.yml", self.GITHUB).endswith("template=report-a-run.yml"))
|
|
self.assertEqual(forge.api_base(self.GITEA), "https://git.example.net/api/v1/repos/me/threshing-atlas")
|
|
|
|
def test_gitea_calls(self):
|
|
from collection.etl import forge
|
|
calls = []
|
|
|
|
def fake(method, url, body=None, tok=None, f=None):
|
|
calls.append((method, url, body, tok))
|
|
if url.endswith("/pulls"):
|
|
return {"number": 7}
|
|
if "/issues?" in url:
|
|
page = int(url.rsplit("page=", 1)[1])
|
|
return [{"number": 50 * (page - 1) + i, "body": "", "labels": [], "state": "open"} for i in range(50 if page == 1 else 3)]
|
|
return None
|
|
old = forge._call
|
|
forge._call = fake
|
|
try:
|
|
issues = forge.list_issues("run report", {"ATLAS_BOT_TOKEN": "t"}, self.GITEA)
|
|
self.assertEqual(len(issues), 53)
|
|
self.assertIsNone(issues[0]["state_reason"])
|
|
self.assertIn("type=issues", calls[0][1])
|
|
self.assertEqual(forge.open_pr("data/x", "T", "B", True, {"ATLAS_BOT_TOKEN": "t"}, self.GITEA), 7)
|
|
merge = calls[-1]
|
|
self.assertTrue(merge[1].endswith("/pulls/7/merge") and merge[2]["merge_when_checks_succeed"])
|
|
self.assertEqual(merge[3], "t")
|
|
finally:
|
|
forge._call = old
|
|
|
|
def test_page_links_follow_forge_yaml(self):
|
|
from collection.etl import codegen, forge
|
|
links = codegen.links()
|
|
self.assertEqual(links["repo"], forge.repo_url())
|
|
self.assertTrue(links["report"].startswith(forge.repo_url()))
|
|
page = (ROOT / "site/atlas/index.html").read_text(encoding="utf-8")
|
|
for key in set(re.findall(r"\{\{(\w+)\}\}", page)):
|
|
self.assertIn(key, links)
|
|
|
|
|
|
class Sheets(unittest.TestCase):
|
|
def test_privacy_rules(self):
|
|
p = TMP / "sheet.csv"
|
|
p.write_text("Timestamp,Cadet name,Photo upload,Notes\n2026-10-01,Violet,http://x,mail me at a@b.co\n", encoding="utf-8")
|
|
spec = {"label": "t", "transform": {"name": "sheet"},
|
|
"privacy": {"redact_emails": True, "drop_columns_matching": ["photo", "cadet name"]}}
|
|
out = sheet.transform(p, spec, {"changed": "2026-11-01"})
|
|
self.assertEqual(out["headers"], ["Timestamp", "Notes"])
|
|
self.assertEqual(out["rows"], [{"Timestamp": "2026-10-01", "Notes": "mail me at [email removed]"}])
|
|
self.assertEqual(out["exported"], "2026-11-01")
|
|
|
|
|
|
class Reddit(unittest.TestCase):
|
|
def test_new_result_links_only_and_no_usernames(self):
|
|
known = read_yaml("collection/manual/reddit_results.yaml")["results"][0]["id"]
|
|
tree = [{"kind": "Listing", "data": {"children": [
|
|
{"kind": "t1", "data": {"author": "someone", "permalink": "/r/fourthwing/comments/x/y/c1/",
|
|
"body": f"got mine! https://dragonkind.com/results/{known} and https://dragonkind.com/results/THRESH-NEW123",
|
|
"replies": {"kind": "Listing", "data": {"children": [
|
|
{"kind": "t1", "data": {"author": "other", "permalink": "/c2/", "body": "dragonkind.com/results/thresh-new456"}}]}}}}]}}]
|
|
p = TMP / "reddit.json"
|
|
p.write_text(json.dumps({"x": tree}), encoding="utf-8")
|
|
out = reddit.transform(p, {"transform": {"manual": "collection/manual/reddit_results.yaml"}}, {})
|
|
self.assertEqual([r["id"] for r in out["new_results"]], ["THRESH-NEW123", "THRESH-NEW456"])
|
|
self.assertNotIn("someone", json.dumps(out))
|
|
|
|
|
|
class Reports(unittest.TestCase):
|
|
def issue(self, number, answers, **kw):
|
|
form = report_form.spec()
|
|
by_id = {f["id"]: f for f in form["fields"]}
|
|
body = "\n\n".join(f"### {f['label']}\n\n{answers.get(f['id'], '_No response_')}" for f in by_id.values())
|
|
return {"number": number, "body": body, "created_at": "2026-11-05T10:00:00Z", "user": {"login": "rider"},
|
|
"labels": [{"name": "run report"}], "state": "open", **kw}
|
|
|
|
def test_issue_form_is_generated(self):
|
|
self.assertEqual(report_form.render(), (ROOT / report_form.TEMPLATE).read_text(encoding="utf-8"))
|
|
|
|
def test_every_option_is_a_dictionary_value(self):
|
|
cols = read_yaml("data/dictionary.yaml")["columns"]
|
|
for f in report_form.spec()["fields"]:
|
|
self.assertIn(f["column"], cols)
|
|
for v in (f.get("options") or {}).values():
|
|
if v is not None and "values" in cols[f["column"]]:
|
|
self.assertIn(str(v), cols[f["column"]]["values"], f["id"])
|
|
|
|
def test_reports_become_valid_runs(self):
|
|
good = self.issue(7, {"company": "Alone", "direction": "Toward the forest centre", "first_sighting": "Red",
|
|
"scream": "Investigated, and helped my squadmate", "branch": "Valley: passed the Red, the meadow",
|
|
"outcome": "Bonded", "outcome_color": "Blue", "tail": "Clubtail", "attempt": "3rd"})
|
|
spam = self.issue(8, {"company": "Alone"}, state="closed", state_reason="not_planned")
|
|
dup = self.issue(9, {"company": "Alone"}, labels=[{"name": "run report"}, {"name": "duplicate"}])
|
|
raw = TMP / "issues.json"
|
|
raw.write_text(json.dumps([good, spam, dup]), encoding="utf-8")
|
|
spec = read_yaml("collection/pipeline.yaml")["sources"]["reported"]
|
|
out = reports.transform(raw, spec, {})
|
|
self.assertEqual([r["number"] for r in out["reports"]], [7])
|
|
self.assertNotIn("rider", json.dumps(out))
|
|
target = TMP / "reports.json"
|
|
target.write_text(json.dumps(out), encoding="utf-8")
|
|
rows = runs.reports("reported", {**spec, "load": str(target)})
|
|
r = rows[0]
|
|
self.assertEqual((r["run_id"], r["source"], r["date"]), ("reported-7", "reported", "2026-11-05"))
|
|
self.assertEqual((r["company"], r["direction"], r["hour"], r["attempt"]), ("Alone", "Center", "Valley", 3))
|
|
self.assertEqual((r["outcome"], r["outcome_color"], r["enc_Blue"], r["saw_blue_meadow"]), ("Bonded", "Blue", "Bonded!", True))
|
|
self.assertEqual(validate.problems(rows, runs.DICT), [])
|
|
|
|
|
|
class Build(unittest.TestCase):
|
|
def test_published_files_are_up_to_date(self):
|
|
self.assertEqual(run.build(check=True), [], "run `python -m collection build`")
|
|
|
|
def test_every_source_value_is_described(self):
|
|
pipe = read_yaml("collection/pipeline.yaml")
|
|
described = runs.DICT["columns"]["source"]["values"]
|
|
for name, spec in pipe["sources"].items():
|
|
if spec.get("run_source"):
|
|
self.assertIn(spec["run_source"], described, name)
|
|
|
|
def test_validation_catches_unknown_values(self):
|
|
row = runs.blank_row(run_id="x", source="reported", company="Five squadmates")
|
|
self.assertTrue(any("company" in p for p in validate.problems([row], runs.DICT)))
|
|
self.assertTrue(any("unique" in p for p in validate.problems([row, row], runs.DICT)))
|
|
|
|
def test_spreadsheet_rows_keep_their_entry_label(self):
|
|
with open(ROOT / "data/public/runs_clean.csv", encoding="utf-8", newline="") as f:
|
|
rows = list(csv.DictReader(f))
|
|
sheet_rows = [r for r in rows if r["source"] == "reddit spreadsheet"]
|
|
self.assertTrue(sheet_rows and all(r["entry"] and r["run_id"].startswith("sheet-") for r in sheet_rows))
|
|
self.assertTrue(all(not r["entry"] for r in rows if r["source"] != "reddit spreadsheet"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|