CI / test (pull_request) Canceled after 0s
- analysis/tree/build_tree.py -> data/public/tree.json, analysis/reports/TREE.md: every screen the bot has seen, every option, where each led (fixed or dice, with ranges), best known picks per colour chosen cautiously, where any picks end, and the untested options. solver.js screenPick(): least-tried and Thompson picks from the tree. - The bot logs a pick it then overrode as a step: 37 runs were coded with choices they never made. Importer and graph.js now use the played steps only. - runs_clean.csv: the Unofficial Dragon Register's 288 bonded dragons are in the master list as outcome-only rows, flagged (no analysis uses them). - Cycle, refresh and the bot kit rebuild and carry the tree; tests check every generated report is current. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
329 lines
18 KiB
Python
329 lines
18 KiB
Python
"""The collection pipeline: extract's raw store, every transform, the coders, validation and the generated files."""
|
|
import csv
|
|
import json
|
|
import re
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from collection import run # noqa: E402
|
|
from collection.etl import codegen, extract, report_form, runs, validate # noqa: E402
|
|
from collection.etl.common import read_yaml # noqa: E402
|
|
from collection.etl.transform import bot, reddit, reports, sheet # noqa: E402
|
|
|
|
FX = ROOT / "tests" / "fixtures" / "bot"
|
|
TMP = Path(tempfile.mkdtemp(prefix="collection-test-"))
|
|
|
|
|
|
def fixture_import(salt="test-salt"):
|
|
kn = json.loads((FX / "data/knowledge.json").read_text(encoding="utf-8"))
|
|
enc = bot._jsonl((FX / "logs/encounters.jsonl").read_text(encoding="utf-8"))
|
|
rn = bot._jsonl((FX / "data/runs.jsonl").read_text(encoding="utf-8"))
|
|
return bot.import_bot(kn, enc, rn, salt)
|
|
|
|
|
|
class RawStore(unittest.TestCase):
|
|
def setUp(self):
|
|
self.old = extract.RAW_DIR
|
|
extract.RAW_DIR = TMP / "raw"
|
|
shutil.rmtree(extract.RAW_DIR, ignore_errors=True)
|
|
|
|
def tearDown(self):
|
|
extract.RAW_DIR = self.old
|
|
|
|
def test_current_and_one_previous_only(self):
|
|
m = {}
|
|
self.assertTrue(extract.store("s", "txt", b"one", m, today="2026-11-01"))
|
|
self.assertFalse(extract.store("s", "txt", b"one", m, today="2026-11-02"))
|
|
self.assertEqual(m["s"]["changed"], "2026-11-01")
|
|
self.assertTrue(extract.store("s", "txt", b"two", m, today="2026-11-03"))
|
|
self.assertTrue(extract.store("s", "txt", b"three", m, today="2026-11-04"))
|
|
cur, prev = extract.raw_paths("s", "txt")
|
|
self.assertEqual((cur.read_bytes(), prev.read_bytes()), (b"three", b"two"))
|
|
self.assertEqual(sorted(p.name for p in cur.parent.iterdir()), ["current.txt", "previous.txt"])
|
|
self.assertEqual(set(m["s"]), {"sha256", "bytes", "changed"}) # the current version only
|
|
|
|
def test_missing_secrets_skip_not_fail(self):
|
|
with self.assertRaises(extract.Skip):
|
|
extract.reddit({"secrets": ["A", "B"], "threads": []}, {})
|
|
with self.assertRaises(extract.Skip):
|
|
extract.local_dir({"env": "DRAGONKIND_BOT_DIR", "files": []}, {})
|
|
|
|
|
|
class Bot(unittest.TestCase):
|
|
def test_same_output_as_the_javascript_importer(self):
|
|
scenes, trajs, unlabeled = fixture_import()
|
|
expected = json.loads((FX / "expected.json").read_text(encoding="utf-8"))
|
|
self.assertEqual(json.dumps(scenes, ensure_ascii=False), json.dumps(expected["scenes"], ensure_ascii=False))
|
|
self.assertEqual(json.dumps(trajs, ensure_ascii=False), json.dumps(expected["trajectories"], ensure_ascii=False))
|
|
self.assertEqual(unlabeled, expected["unlabeled"])
|
|
|
|
def test_one_trajectory_per_attempt(self):
|
|
_, (t1, t2), _ = fixture_import()
|
|
self.assertEqual(len(t1["steps"]), 3)
|
|
self.assertEqual(t1["steps"][-1]["to"], "BURNED")
|
|
self.assertEqual(t1["steps"][0]["to"], t1["steps"][1]["scene"])
|
|
self.assertEqual(t2["steps"][-1]["to"], "BONDED:green")
|
|
self.assertTrue(t2["steps"][0]["manual"])
|
|
|
|
def test_accounts_hashed_and_no_prose(self):
|
|
scenes, trajs, _ = fixture_import()
|
|
self.assertRegex(trajs[0]["meta"]["accountKey"], r"^[0-9a-f]{10}$")
|
|
text = json.dumps({"s": scenes, "t": trajs})
|
|
self.assertNotIn("acct-1", text)
|
|
self.assertNotIn("Synthetic", text)
|
|
P = read_yaml("collection/pipeline.yaml")["sources"]["bot"]["privacy"]
|
|
bot.check_public(scenes, set(P["refuse_fields"]), P["max_string"])
|
|
with self.assertRaises(ValueError):
|
|
bot.check_public({"prose": "x"}, set(P["refuse_fields"]), 200)
|
|
with self.assertRaises(ValueError):
|
|
bot.check_public({"note": "y" * 201}, set(), 200)
|
|
|
|
def test_crosswalk_matches_graph_js(self):
|
|
"""The bot's advisor maps choices with core/graph.js; the importer with bot.yaml. Same answers."""
|
|
trajs = runs.bot_trajectories(read_yaml("collection/pipeline.yaml")["sources"]["bot"])
|
|
trajs += json.loads((FX / "expected.json").read_text(encoding="utf-8"))["trajectories"]
|
|
script = ("const G=require('./core/graph.js');let s='';process.stdin.on('data',d=>s+=d).on('end',()=>"
|
|
"console.log(JSON.stringify(JSON.parse(s).map(t=>G.trajectoryVariables(t).vars))))")
|
|
js = json.loads(subprocess.run(["node", "-e", script], input=json.dumps(trajs), capture_output=True,
|
|
text=True, cwd=ROOT, check=True).stdout)
|
|
import re
|
|
cw = [(re.compile(rx), c, v) for rx, c, v in read_yaml("collection/specs/bot.yaml")["crosswalk"]]
|
|
for t, j in zip(trajs, js):
|
|
py, _ = runs.bot_variables(t, cw)
|
|
self.assertEqual(py, {k: v for k, v in j.items() if k not in ("outcome", "outcome_color")})
|
|
|
|
def test_every_bot_choice_is_mapped(self):
|
|
"""Every route choice the bot has seen before the hour maps onto a column (answers inside the bonding
|
|
encounters are not route choices), unless bot.yaml lists it as not coded."""
|
|
import re
|
|
spec = read_yaml("collection/specs/bot.yaml")
|
|
cw = [re.compile(rx) for rx, _, _ in spec["crosswalk"]] + [re.compile(rx) for rx in spec.get("not_coded", [])]
|
|
scenes = json.loads((ROOT / "data/bot/scenes.json").read_text(encoding="utf-8"))
|
|
route = {"ENTER_THRESHING", "FIRST_SIGHTING", "SCREAM_AND_CADETS"}
|
|
unmapped = [f"{s['label']}: {o['text']}" for s in scenes.values()
|
|
if len(s["options"]) > 1 and s["stage"] in route and "two Greens in the clearing" not in s["label"]
|
|
for o in s["options"] if not any(rx.search(bot.normalize(o["text"])) for rx in cw)]
|
|
self.assertEqual(unmapped, [])
|
|
|
|
def test_runs_with_steps_and_accounts(self):
|
|
"""The bot's newer run records (one folder per account) become trajectories without account names or
|
|
prose; the top-level files mirror the first account and are skipped."""
|
|
run = {"runId": "run-1", "attempt": 3, "accountAttempt": 3, "accountId": "someone-example-com",
|
|
"accountRole": "research", "startedAt": "2026-11-09T22:02:14.430Z", "targetDragon": "black",
|
|
"decisionMode": "atlas_policy", "policyId": "atlas:x", "outcome": "BURNED", "intentionalBurn": True,
|
|
"burn": {"reason": "TREE_NO_SHADOW_AUTO_BURN"}, "dragonColor": None,
|
|
"steps": [{"nodeId": "a68f681076c6", "stage": "ENTER_THRESHING", "choiceText": "Find the tallest tree and climb for a better view",
|
|
"offered": [{"text": "Find the tallest tree and climb for a better view"}, {"text": "Continue toward the center of the valley"}],
|
|
"observed": {"shadow": False, "dragons": ["Green"]}, "propensity": 0.5, "toKey": "TERMINAL:BURNED",
|
|
"label": "x" * 300, "prose": "game text"}]}
|
|
line = json.dumps(run) + "\n"
|
|
files = {"data/runs.jsonl": line, "data/accounts/someone-example-com/runs.jsonl": line,
|
|
"logs/accounts/someone-example-com/encounters.jsonl": ""}
|
|
accts = bot._account_files(files)
|
|
self.assertEqual(list(accts), ["someone-example-com"])
|
|
t = bot.trajectory_from_run(run, "abc123")
|
|
self.assertEqual(t["steps"][0]["seen"], {"dragons": ["green"], "shadow": False})
|
|
self.assertEqual((t["steps"][0]["to"], t["meta"]["accountKey"], t["outcome"]["reason"]), ("BURNED", "abc123", "TREE_NO_SHADOW_AUTO_BURN"))
|
|
text = json.dumps(t)
|
|
self.assertNotIn("someone", text)
|
|
self.assertNotIn("game text", text)
|
|
bot.check_public(t, {"prose", "accountId"}, 200)
|
|
|
|
def test_graph_js_crosswalk_is_generated(self):
|
|
self.assertEqual(codegen.graph_js(), (ROOT / codegen.GRAPH_JS).read_text(encoding="utf-8"))
|
|
|
|
|
|
class Handoff(unittest.TestCase):
|
|
def test_bot_zip_imports_like_the_folder(self):
|
|
import zipfile
|
|
spec = read_yaml("collection/pipeline.yaml")["sources"]["bot"]["extract"]
|
|
z = TMP / "ken.zip"
|
|
with zipfile.ZipFile(z, "w") as f: # files under a top folder, as a zipped repo would have them
|
|
for rel in spec["files"]:
|
|
if "*" not in rel:
|
|
f.write(FX / rel, f"DragonKind-main/{rel}")
|
|
self.assertEqual(extract.pack_local(spec, z), extract.pack_local(spec, FX))
|
|
empty = TMP / "empty.zip"
|
|
with zipfile.ZipFile(empty, "w") as f:
|
|
f.writestr("notes.txt", "hi")
|
|
with self.assertRaises(RuntimeError):
|
|
extract.pack_local(spec, empty)
|
|
|
|
def test_overridden_picks_are_not_played(self):
|
|
from collection.etl import runs
|
|
steps = [{"scene": "a", "to": "BURNED", "optionText": "west"}, {"scene": "a", "to": "b", "optionText": "centre"},
|
|
{"scene": "b", "to": "BURNED", "optionText": "last"}]
|
|
self.assertEqual([s["optionText"] for s in runs.played(steps)], ["centre", "last"])
|
|
self.assertEqual(runs.played(steps[1:]), steps[1:]) # a last step that ends the run is kept
|
|
|
|
def test_register_rows_are_outcome_only_and_flagged(self):
|
|
import csv
|
|
with open(ROOT / "data/public/runs_clean.csv", encoding="utf-8", newline="") as f:
|
|
reg = [r for r in csv.DictReader(f) if r["source"] == "register"]
|
|
self.assertTrue(reg)
|
|
for r in reg:
|
|
self.assertEqual((r["outcome"], r["company"], r["scream"]), ("Bonded", "Unknown", "Unknown"))
|
|
self.assertIn("outcome only", r["data_flags"])
|
|
|
|
def test_bot_kit(self):
|
|
import zipfile
|
|
from collection.etl import handoff
|
|
zpath = handoff.export_bot_kit(str(TMP / "exports"))
|
|
with zipfile.ZipFile(zpath) as z:
|
|
self.assertEqual(sorted(z.namelist()), ["README.txt", "experiments.json", "graph.js", "policy.json", "solver.js", "tree.json"])
|
|
self.assertEqual(z.read("solver.js"), (ROOT / "core/solver.js").read_bytes())
|
|
self.assertEqual(json.loads(z.read("policy.json"))["id"],
|
|
json.loads((ROOT / "data/public/policy.json").read_text(encoding="utf-8"))["id"])
|
|
|
|
|
|
class Forge(unittest.TestCase):
|
|
GITEA = {"kind": "gitea", "url": "https://git.example.net/", "repo": "me/threshing-atlas", "branch": "main"}
|
|
GITHUB = {"kind": "github", "url": "https://github.com", "repo": "me/threshing-atlas", "branch": "main"}
|
|
|
|
def test_links(self):
|
|
from collection.etl import forge
|
|
self.assertEqual(forge.file_url("data/x.csv", self.GITEA), "https://git.example.net/me/threshing-atlas/raw/branch/main/data/x.csv")
|
|
self.assertEqual(forge.file_url("data/x.csv", self.GITHUB), "https://raw.githubusercontent.com/me/threshing-atlas/main/data/x.csv")
|
|
self.assertIn("template=.github/ISSUE_TEMPLATE/report-a-run.yml", forge.new_issue_url(".github/ISSUE_TEMPLATE/report-a-run.yml", self.GITEA))
|
|
self.assertTrue(forge.new_issue_url(".github/ISSUE_TEMPLATE/report-a-run.yml", self.GITHUB).endswith("template=report-a-run.yml"))
|
|
self.assertEqual(forge.api_base(self.GITEA), "https://git.example.net/api/v1/repos/me/threshing-atlas")
|
|
|
|
def test_gitea_calls(self):
|
|
from collection.etl import forge
|
|
calls = []
|
|
|
|
def fake(method, url, body=None, tok=None, f=None):
|
|
calls.append((method, url, body, tok))
|
|
if url.endswith("/pulls"):
|
|
return {"number": 7}
|
|
if "/issues?" in url:
|
|
page = int(url.rsplit("page=", 1)[1])
|
|
return [{"number": 50 * (page - 1) + i, "body": "", "labels": [], "state": "open"} for i in range(50 if page == 1 else 3)]
|
|
return None
|
|
old = forge._call
|
|
forge._call = fake
|
|
try:
|
|
issues = forge.list_issues("run report", {"ATLAS_BOT_TOKEN": "t"}, self.GITEA)
|
|
self.assertEqual(len(issues), 53)
|
|
self.assertIsNone(issues[0]["state_reason"])
|
|
self.assertIn("type=issues", calls[0][1])
|
|
self.assertEqual(forge.open_pr("data/x", "T", "B", True, {"ATLAS_BOT_TOKEN": "t"}, self.GITEA), 7)
|
|
merge = calls[-1]
|
|
self.assertTrue(merge[1].endswith("/pulls/7/merge") and merge[2]["merge_when_checks_succeed"])
|
|
self.assertEqual(merge[3], "t")
|
|
finally:
|
|
forge._call = old
|
|
|
|
def test_page_links_follow_forge_yaml(self):
|
|
from collection.etl import codegen, forge
|
|
links = codegen.links()
|
|
self.assertEqual(links["repo"], forge.repo_url())
|
|
self.assertTrue(links["report"].startswith(forge.repo_url()))
|
|
page = (ROOT / "site/atlas/index.html").read_text(encoding="utf-8")
|
|
for key in set(re.findall(r"\{\{(\w+)\}\}", page)):
|
|
self.assertIn(key, links)
|
|
|
|
|
|
class Sheets(unittest.TestCase):
|
|
def test_privacy_rules(self):
|
|
p = TMP / "sheet.csv"
|
|
p.write_text("Timestamp,Cadet name,Photo upload,Notes\n2026-10-01,Violet,http://x,mail me at a@b.co\n", encoding="utf-8")
|
|
spec = {"label": "t", "transform": {"name": "sheet"},
|
|
"privacy": {"redact_emails": True, "drop_columns_matching": ["photo", "cadet name"]}}
|
|
out = sheet.transform(p, spec, {"changed": "2026-11-01"})
|
|
self.assertEqual(out["headers"], ["Timestamp", "Notes"])
|
|
self.assertEqual(out["rows"], [{"Timestamp": "2026-10-01", "Notes": "mail me at [email removed]"}])
|
|
self.assertEqual(out["exported"], "2026-11-01")
|
|
|
|
|
|
class Reddit(unittest.TestCase):
|
|
def test_new_result_links_only_and_no_usernames(self):
|
|
known = read_yaml("collection/manual/reddit_results.yaml")["results"][0]["id"]
|
|
tree = [{"kind": "Listing", "data": {"children": [
|
|
{"kind": "t1", "data": {"author": "someone", "permalink": "/r/fourthwing/comments/x/y/c1/",
|
|
"body": f"got mine! https://dragonkind.com/results/{known} and https://dragonkind.com/results/THRESH-NEW123",
|
|
"replies": {"kind": "Listing", "data": {"children": [
|
|
{"kind": "t1", "data": {"author": "other", "permalink": "/c2/", "body": "dragonkind.com/results/thresh-new456"}}]}}}}]}}]
|
|
p = TMP / "reddit.json"
|
|
p.write_text(json.dumps({"x": tree}), encoding="utf-8")
|
|
out = reddit.transform(p, {"transform": {"manual": "collection/manual/reddit_results.yaml"}}, {})
|
|
self.assertEqual([r["id"] for r in out["new_results"]], ["THRESH-NEW123", "THRESH-NEW456"])
|
|
self.assertNotIn("someone", json.dumps(out))
|
|
|
|
|
|
class Reports(unittest.TestCase):
|
|
def issue(self, number, answers, **kw):
|
|
form = report_form.spec()
|
|
by_id = {f["id"]: f for f in form["fields"]}
|
|
body = "\n\n".join(f"### {f['label']}\n\n{answers.get(f['id'], '_No response_')}" for f in by_id.values())
|
|
return {"number": number, "body": body, "created_at": "2026-11-05T10:00:00Z", "user": {"login": "rider"},
|
|
"labels": [{"name": "run report"}], "state": "open", **kw}
|
|
|
|
def test_issue_form_is_generated(self):
|
|
self.assertEqual(report_form.render(), (ROOT / report_form.TEMPLATE).read_text(encoding="utf-8"))
|
|
|
|
def test_every_option_is_a_dictionary_value(self):
|
|
cols = read_yaml("data/dictionary.yaml")["columns"]
|
|
for f in report_form.spec()["fields"]:
|
|
self.assertIn(f["column"], cols)
|
|
for v in (f.get("options") or {}).values():
|
|
if v is not None and "values" in cols[f["column"]]:
|
|
self.assertIn(str(v), cols[f["column"]]["values"], f["id"])
|
|
|
|
def test_reports_become_valid_runs(self):
|
|
good = self.issue(7, {"company": "Alone", "direction": "Toward the forest centre", "first_sighting": "Red",
|
|
"scream": "Investigated, and helped my squadmate", "branch": "Valley: passed the Red, the meadow",
|
|
"outcome": "Bonded", "outcome_color": "Blue", "tail": "Clubtail", "attempt": "3rd"})
|
|
spam = self.issue(8, {"company": "Alone"}, state="closed", state_reason="not_planned")
|
|
dup = self.issue(9, {"company": "Alone"}, labels=[{"name": "run report"}, {"name": "duplicate"}])
|
|
raw = TMP / "issues.json"
|
|
raw.write_text(json.dumps([good, spam, dup]), encoding="utf-8")
|
|
spec = read_yaml("collection/pipeline.yaml")["sources"]["reported"]
|
|
out = reports.transform(raw, spec, {})
|
|
self.assertEqual([r["number"] for r in out["reports"]], [7])
|
|
self.assertNotIn("rider", json.dumps(out))
|
|
target = TMP / "reports.json"
|
|
target.write_text(json.dumps(out), encoding="utf-8")
|
|
rows = runs.reports("reported", {**spec, "load": str(target)})
|
|
r = rows[0]
|
|
self.assertEqual((r["run_id"], r["source"], r["date"]), ("reported-7", "reported", "2026-11-05"))
|
|
self.assertEqual((r["company"], r["direction"], r["hour"], r["attempt"]), ("Alone", "Center", "Valley", 3))
|
|
self.assertEqual((r["outcome"], r["outcome_color"], r["enc_Blue"], r["saw_blue_meadow"]), ("Bonded", "Blue", "Bonded!", True))
|
|
self.assertEqual(validate.problems(rows, runs.DICT), [])
|
|
|
|
|
|
class Build(unittest.TestCase):
|
|
def test_published_files_are_up_to_date(self):
|
|
self.assertEqual(run.build(check=True), [], "run `python -m collection build`")
|
|
|
|
def test_every_source_value_is_described(self):
|
|
pipe = read_yaml("collection/pipeline.yaml")
|
|
described = runs.DICT["columns"]["source"]["values"]
|
|
for name, spec in pipe["sources"].items():
|
|
if spec.get("run_source"):
|
|
self.assertIn(spec["run_source"], described, name)
|
|
|
|
def test_validation_catches_unknown_values(self):
|
|
row = runs.blank_row(run_id="x", source="reported", company="Five squadmates")
|
|
self.assertTrue(any("company" in p for p in validate.problems([row], runs.DICT)))
|
|
self.assertTrue(any("unique" in p for p in validate.problems([row, row], runs.DICT)))
|
|
|
|
def test_spreadsheet_rows_keep_their_entry_label(self):
|
|
with open(ROOT / "data/public/runs_clean.csv", encoding="utf-8", newline="") as f:
|
|
rows = list(csv.DictReader(f))
|
|
sheet_rows = [r for r in rows if r["source"] == "reddit spreadsheet"]
|
|
self.assertTrue(sheet_rows and all(r["entry"] and r["run_id"].startswith("sheet-") for r in sheet_rows))
|
|
self.assertTrue(all(not r["entry"] for r in rows if r["source"] != "reddit spreadsheet"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|