Threshing Atlas: monorepo, route graph, bot importer, daily source checks

- apps/atlas: the web page (valley map, route flow, Dragon Register)
- packages/core: coding rules and statistics (engine.js) and the shared route
  graph (graph.js): scene graph, decision-variable graph, bot-to-form crosswalk,
  Beta credible intervals
- packages/ingest: DragonKind bot importer to anonymised trajectories (no prose,
  no account identifiers); data/bot holds Ken's sample
- research/python: independent reference implementation + update_sources.py
- tests: JS/Python parity checks and graph/importer unit tests
- CI, GitHub Pages deploy, daily check of the community sheets and route PDF
- Email addresses in free-text form answers are redacted; the raw spreadsheet
  is not committed
- MIT code licence, CC BY 4.0 data licence

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-01 21:54:54 +00:00
co-authored by Claude Opus 5.5
parent 0052d1a5fd
commit c3d067b88c
48 changed files with 8334 additions and 0 deletions
+37
View File
@@ -0,0 +1,37 @@
"""Compare factor tables and models between the JS engine and the Python reference."""
import json, sys
js = json.load(open("tests/out/results_js.json"))
py = json.load(open("tests/out/results_py.json"))
TOL = 1e-6
fails = 0
def close(a, b, rel=TOL):
if a is None or b is None:
return a is None and b is None
return abs(a - b) <= rel * max(1, abs(a), abs(b))
for key, ft in py["factors"].items():
jrows = {(r["factor"], r["level"]): r for r in js["factors"][key]["rows"]}
for r in ft["rows"]:
j = jrows.get((r["factor"], r["level"]))
if j is None:
print("missing in JS:", key, r["factor"], r["level"]); fails += 1; continue
for f in ("k", "n", "p", "lo", "hi", "or", "or_lo", "or_hi", "fisher_p", "q"):
if (f in r) != (f in j) or (f in r and not close(r[f], j[f], 1e-5)):
print("diff", key, r["factor"], r["level"], f, r.get(f), j.get(f)); fails += 1
for key, m in py["models"].items():
jm = js["models"][key]
if m["n"] != jm["n"] or m["events"] != jm["events"]:
print("model size diff", key); fails += 1
jt = {t["term"]: t for t in jm["terms"]}
for t in m["terms"]:
j = jt.get(t["term"])
if not j:
print("term missing", key, t["term"]); fails += 1; continue
for f in ("or", "lo", "hi"):
if not close(t[f], j[f], 1e-4):
print("model diff", key, t["term"], f, t[f], j[f]); fails += 1
if abs(t["p"] - j["p"]) > 1e-6:
print("model p diff", key, t["term"], t["p"], j["p"]); fails += 1
n_rows = sum(len(f["rows"]) for f in py["factors"].values())
n_terms = sum(len(m["terms"]) for m in py["models"].values())
print(f"checked {n_rows} factor rows and {n_terms} model terms: {'ALL MATCH' if not fails else str(fails) + ' differences'}")
sys.exit(1 if fails else 0)
+53
View File
@@ -0,0 +1,53 @@
"""Recompute route rankings in Python (crude Fisher and stratified Mantel-Haenszel) and compare with engine.js."""
import json, subprocess, sys
sys.path.insert(0, "research/python")
import pandas as pd
from stats_core import odds_ratio, mantel_haenszel, bh
d = pd.DataFrame(json.load(open("tests/out/runs_js.json")))
js = json.loads(subprocess.check_output(["node", "-e", """
const E=require('./packages/core/src/engine.js'),fs=require('fs');const raw=JSON.parse(fs.readFileSync('data/public/form_responses.json'));
const {runs}=E.codeRows(raw.rows,raw.headers);const o={};for(const t of ['see_Blue','bond_Blue','see_shadow','bond_Green']){o[t]=E.rankRoutes(runs,t,10)}
console.log(JSON.stringify(o));"""]))
def stage(r):
if r.scream.startswith("DIED"): cad = None # noqa
else: cad = ("Help → " + r.resolution) if r.scream == "Investigate & help" else (r.scream + " → " + r.resolution)
hour = {"Tree": "Climb the tree", "Valley": "Valley centre"}.get(r.hour)
fs = None if r.scream == "DIED at first sighting" else r.first_sighting
return " › ".join(v if isinstance(v, str) and v else "—" for v in [r.company, r.direction, fs, cad, r.post_path, hour])
def stratum(r):
if r.source != "Form": return "bulk"
return {"Blue": "blue", "Black": "black"}.get(r.goal, "other")
d["route"] = d.apply(stage, axis=1); d["stratum"] = d.apply(stratum, axis=1)
d["branch"] = d.branch.where(d.branch.notna(), None)
T = {"see_Blue": (lambda x: x.branch == "Passed the Red: meadow", lambda x: x.saw_blue_meadow),
"bond_Blue": (lambda x: x.hour == "Valley", lambda x: (x.outcome == "Bonded") & (x.outcome_color == "Blue")),
"see_shadow": (lambda x: (x.hour == "Tree") & x.branch.notna(), lambda x: x.saw_shadow),
"bond_Green": (lambda x: x.hour == "Tree", lambda x: (x.outcome == "Bonded") & (x.outcome_color == "Green"))}
fails = 0
for t, (pool, ev) in T.items():
P = d[pool(d)].copy(); P["e"] = ev(P).astype(bool)
K, N = int(P.e.sum()), len(P)
rows = {}
for route, g in P.groupby("route"):
if len(g) < 10: continue
k, n = int(g.e.sum()), len(g)
orr = odds_ratio(k, n - k, K - k, N - n - (K - k))
tabs = []
for s, sg in P.groupby("stratum"):
inr = sg.route == route
a = int((inr & sg.e).sum()); nin = int(inr.sum()); c = int((~inr & sg.e).sum()); nout = len(sg) - nin
tabs.append([a, nin - a, c, nout - c])
rows[route] = (k, n, orr, mantel_haenszel(tabs))
jr = {r["route"]: r for r in js[t]["rows"]}
if set(jr) != set(rows): print(t, "route sets differ", set(jr) ^ set(rows)); fails += 1
for route, (k, n, orr, mh) in rows.items():
j = jr.get(route)
if not j: continue
if (k, n) != (j["k"], j["n"]) or abs(orr[0] - j["or"]) > 1e-6 * max(1, orr[0]) or abs(orr[3] - j["fisher_p"]) > 1e-6:
print("crude diff", t, route); fails += 1
if mh is None:
if j["mh_or"] is not None: print("mh none diff", t, route); fails += 1
elif j["mh_or"] is None or abs(mh[0] - j["mh_or"]) > 1e-6 * max(1, mh[0]) or abs(mh[3] - j["mh_p"]) > 1e-5:
print("mh diff", t, route, mh, j["mh_or"], j["mh_p"]); fails += 1
print(f"route rankings for {len(T)} outcomes: {'ALL MATCH' if not fails else str(fails) + ' differences'}")
sys.exit(1 if fails else 0)
+21
View File
@@ -0,0 +1,21 @@
"""Check that the JS engine (packages/core/src/engine.js) and the Python reference (python/clean.py) code every run identically."""
import json, sys
import pandas as pd
js = pd.DataFrame(json.load(open(sys.argv[1] if len(sys.argv) > 1 else "tests/out/runs_js.json")))
py = pd.read_csv(sys.argv[2] if len(sys.argv) > 2 else "data/derived/runs_clean.csv", keep_default_na=False, na_values=[""])
def norm(v):
if v is None or (not isinstance(v, str) and pd.isna(v)) or v == "":
return None
if isinstance(v, float) and v.is_integer():
return str(int(v))
if isinstance(v, bool):
return str(v)
return str(v)
bad = {}
for c in py.columns:
a, b = [norm(v) for v in py[c]], [norm(v) for v in js[c]]
d = [(i, x, y) for i, (x, y) in enumerate(zip(a, b)) if x != y]
if d:
bad[c] = (len(d), d[:3])
print("runs:", len(py), len(js), "| mismatching columns:", bad or "NONE")
sys.exit(1 if bad else 0)
+1
View File
@@ -0,0 +1 @@
{"version":1,"nodes":{},"edges":{}}
+1
View File
@@ -0,0 +1 @@
{"runId":"r1","attempt":1,"startedAt":"2026-01-01T00:00:00.000Z","targetDragon":"black","decisionMode":"explore","outcome":"BURNED","intentionalBurn":true,"accountId":"acct-1","steps":[]}
+10
View File
@@ -0,0 +1,10 @@
{"ts":"2026-01-01T00:00:00.000Z","attempt":1,"event":"DECISION","nodeId":"b0fa671066ec","prose":"Synthetic start scene.","choices":["Enter the valley alone","Enter the valley with one of your squadmates"],"decision":{"action":"choose","choiceIndex":0,"choiceText":"Enter the valley alone","stage":"TEST","rule":"R","source":"guide"}}
{"ts":"2026-01-01T00:00:01.000Z","attempt":1,"event":"DECISION","nodeId":"b0fa671066ec","prose":"Synthetic start scene.","choices":["Enter the valley alone","Enter the valley with one of your squadmates"],"decision":{"action":"choose","choiceIndex":0,"choiceText":"Enter the valley alone","stage":"TEST","rule":"R","source":"guide"}}
{"ts":"2026-01-01T00:00:02.000Z","attempt":1,"event":"DECISION","nodeId":"e0b5530601be","prose":"Synthetic fork.","choices":["Head east toward the roar","Head toward the center of the forest"],"decision":{"action":"choose","choiceIndex":1,"choiceText":"Head toward the center of the forest","stage":"TEST","rule":"R","source":"guide"}}
{"ts":"2026-01-01T00:00:03.000Z","attempt":1,"event":"DECISION","nodeId":"1c86a35c091f","prose":"Synthetic dragon.","choices":["Lower your eyes in respect","Hide behind a tree"],"decision":{"action":"choose","choiceIndex":1,"choiceText":"Hide behind a tree","stage":"TEST","rule":"R","source":"guide"}}
{"ts":"2026-01-01T00:00:04.000Z","attempt":1,"event":"BURNED","prose":"x","stage":"TEST","intentional":false}
{"ts":"2026-01-01T00:00:05.000Z","attempt":1,"event":"BURNED","prose":"x","stage":"TEST","intentional":false}
{"ts":"2026-01-01T00:00:06.000Z","attempt":2,"event":"DECISION","prose":"Synthetic start scene.","choices":["Enter the valley alone","Enter the valley with one of your squadmates"],"decision":{"action":"pause","stage":"TEST"}}
{"ts":"2026-01-01T00:00:07.000Z","attempt":2,"event":"MANUAL_DECISION","prose":"Synthetic start scene.","choices":["Enter the valley alone","Enter the valley with one of your squadmates"],"choiceIndex":1,"choiceText":"Enter the valley with one of your squadmates","source":"manual"}
{"ts":"2026-01-01T00:00:08.000Z","attempt":2,"event":"DECISION","nodeId":"e0b5530601be","prose":"Synthetic fork.","choices":["Head east toward the roar","Head toward the center of the forest"],"decision":{"action":"choose","choiceIndex":0,"choiceText":"Head east toward the roar","stage":"TEST","rule":"R","source":"guide"}}
{"ts":"2026-01-01T00:00:09.000Z","attempt":2,"event":"DRAGON_BONDED","dragon":"Green"}
+72
View File
@@ -0,0 +1,72 @@
// node --test tests/
const test = require("node:test");
const assert = require("node:assert");
const path = require("path");
const { importBot, assertPublicSafe } = require("../packages/ingest/src/bot");
const G = require("../packages/core/src/graph");
const fx = path.join(__dirname, "fixtures/bot");
const { scenes, trajectories } = importBot({ dataDir: path.join(fx, "data"), logsDir: path.join(fx, "logs"), salt: "test-salt" });
test("one trajectory per attempt, duplicates and repeated burns collapsed", () => {
assert.strictEqual(trajectories.length, 2);
const [t1, t2] = trajectories;
assert.strictEqual(t1.steps.length, 3);
assert.deepStrictEqual(t1.steps.map((s) => s.to).slice(-1), ["BURNED"]);
assert.strictEqual(t1.steps[0].to, t1.steps[1].scene);
assert.strictEqual(t2.outcome.type, "BONDED");
assert.strictEqual(t2.steps.at(-1).to, "BONDED:green");
assert.strictEqual(t2.steps[0].manual, true, "manual picks are flagged");
});
test("run metadata attached, account id hashed with the salt", () => {
const t1 = trajectories[0];
assert.strictEqual(t1.meta.target, "black");
assert.strictEqual(t1.meta.decisionMode, "explore");
assert.strictEqual(t1.outcome.intentional, true);
assert.match(t1.meta.accountKey, /^[0-9a-f]{10}$/);
assert.ok(!JSON.stringify(trajectories).includes("acct-1"));
});
test("public export contains no prose", () => {
assertPublicSafe(scenes);
assertPublicSafe(trajectories);
assert.ok(!JSON.stringify({ scenes, trajectories }).includes("Synthetic"));
assert.throws(() => assertPublicSafe({ prose: "x" }));
assert.throws(() => assertPublicSafe({ note: "y".repeat(201) }));
});
test("crosswalk maps bot choices to form variables", () => {
const { vars } = G.trajectoryVariables(trajectories[0]);
assert.strictEqual(vars.company, "Alone");
assert.strictEqual(vars.direction, "Center");
assert.strictEqual(vars.outcome, "Incinerated");
const v2 = G.trajectoryVariables(trajectories[1]).vars;
assert.strictEqual(v2.company, "1 Squadmate");
assert.strictEqual(v2.outcome_color, "Green");
});
test("variable graph counts by source", () => {
const recs = trajectories.map((t) => ({ source: "bot", vars: G.trajectoryVariables(t).vars }));
recs.push({ source: "form", vars: { company: "Alone", direction: "Center", outcome: "Bonded", outcome_color: "Blue" } });
const g = G.buildVariableGraph(recs);
const e = g.edges.find((x) => x.from === "company=Alone" && x.to === "direction=Center");
assert.deepStrictEqual(e.bySource, { bot: 1, form: 1 });
assert.ok(Math.abs(e.p.mean - 2.5 / 3) < 1e-12);
});
test("scene graph marks chance, burns and untested options", () => {
const g = G.buildSceneGraph(scenes, trajectories);
const start = Object.values(g).find((n) => Object.values(n.options).some((o) => /alone/i.test(o.text)));
assert.strictEqual(Object.values(start.options).every((o) => o.n === 1), true);
const dragon = Object.values(g).find((n) => Object.values(n.options).some((o) => /hide behind/i.test(o.text)));
const hide = Object.values(dragon.options).find((o) => /hide/i.test(o.text));
const respect = Object.values(dragon.options).find((o) => /respect/i.test(o.text));
assert.strictEqual(hide.to.BURNED, 1);
assert.strictEqual(respect.untested, true);
});
test("Beta credible interval", () => {
const p = G.betaPosterior(0, 4);
assert.ok(p.lo >= 0 && p.hi < 0.5 && p.mean === 0.1);
});
+13
View File
@@ -0,0 +1,13 @@
// node tests/run_engine.js data/public/form_responses.json out_dir
// Runs the browser engine in Node and writes coded runs + results, for comparison with Python.
const fs = require("fs");
const path = require("path");
const E = require("../packages/core/src/engine.js");
const [, , src = "data/public/form_responses.json", outDir = "tests/out"] = process.argv;
const raw = JSON.parse(fs.readFileSync(src, "utf8"));
const { runs, missing } = E.codeRows(raw.rows, raw.headers);
fs.mkdirSync(outDir, { recursive: true });
fs.writeFileSync(path.join(outDir, "runs_js.json"), JSON.stringify(runs));
const res = E.analyse(runs, {});
fs.writeFileSync(path.join(outDir, "results_js.json"), JSON.stringify(res, null, 1));
console.log(`${runs.length} runs coded; missing columns: ${missing.join(", ") || "none"}`);