Core: a small JS advisor reads the game from policy.json; engine.js is gone (#13)

Step 4b of the Python-first core.

- core/solver.js is now a 250-line advisor (was 900 lines): it reads the game (core/spec/game.yaml,
  published inside policy.json as model.game) and the count tables, and re-solves from any point in
  a run with the same interpreter as the Python model. Same API for the site and Ken's bot
  (advisor, recommendOption, solverHistory). The bot's on-screen text for each option moves to
  game.yaml (bot_text); the deadly rolls carry their risk labels there too.
- tests/core/advisor.test.js: the advisor gives exactly the Python model's values and choices for
  every colour and goal (to 1e-12, every decision on every published route), walks a run in the
  game's order, and maps advice onto the bot's screen.
- Removed core/engine.js and everything that only compared it with Python (tests/parity/run_engine,
  compare_runs, compare_results, compare_routes, compare_solver, compare_stats, analysis/stats/
  analyze.py). In their place: tests/core/test_stats.py checks the statistics against known answers,
  test_solver.py checks the route model on a synthetic world (tests/fixtures/synthetic_policy.py).
- analysis/network/build-graph.js reads the Python-coded runs; graph.json is byte-identical.
- docs/BOT_INTEGRATION.md: what changed for the bot (history keys, new choices, re-sync all three files).


Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ekcSchiano
2026-11-03 22:01:32 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent d04b88e7b4
commit 11b4a0398e
27 changed files with 578 additions and 2260 deletions
-37
View File
@@ -1,37 +0,0 @@
"""Compare factor tables and models between the JS engine and the Python reference."""
import json, sys
js = json.load(open("tests/out/results_js.json"))
py = json.load(open("tests/out/results_py.json"))
TOL = 1e-6
fails = 0
def close(a, b, rel=TOL):
if a is None or b is None:
return a is None and b is None
return abs(a - b) <= rel * max(1, abs(a), abs(b))
for key, ft in py["factors"].items():
jrows = {(r["factor"], r["level"]): r for r in js["factors"][key]["rows"]}
for r in ft["rows"]:
j = jrows.get((r["factor"], r["level"]))
if j is None:
print("missing in JS:", key, r["factor"], r["level"]); fails += 1; continue
for f in ("k", "n", "p", "lo", "hi", "or", "or_lo", "or_hi", "fisher_p", "q"):
if (f in r) != (f in j) or (f in r and not close(r[f], j[f], 1e-5)):
print("diff", key, r["factor"], r["level"], f, r.get(f), j.get(f)); fails += 1
for key, m in py["models"].items():
jm = js["models"][key]
if m["n"] != jm["n"] or m["events"] != jm["events"]:
print("model size diff", key); fails += 1
jt = {t["term"]: t for t in jm["terms"]}
for t in m["terms"]:
j = jt.get(t["term"])
if not j:
print("term missing", key, t["term"]); fails += 1; continue
for f in ("or", "lo", "hi"):
if not close(t[f], j[f], 1e-4):
print("model diff", key, t["term"], f, t[f], j[f]); fails += 1
if abs(t["p"] - j["p"]) > 1e-6:
print("model p diff", key, t["term"], t["p"], j["p"]); fails += 1
n_rows = sum(len(f["rows"]) for f in py["factors"].values())
n_terms = sum(len(m["terms"]) for m in py["models"].values())
print(f"checked {n_rows} factor rows and {n_terms} model terms: {'ALL MATCH' if not fails else str(fails) + ' differences'}")
sys.exit(1 if fails else 0)
-60
View File
@@ -1,60 +0,0 @@
"""Recompute route rankings in Python (crude Fisher and stratified Mantel-Haenszel) and compare with engine.js."""
import json, subprocess, sys
sys.path.insert(0, "analysis/stats")
import pandas as pd
from stats_core import odds_ratio, mantel_haenszel, bh
d = pd.DataFrame(json.load(open("tests/out/runs_js.json")))
js = json.loads(subprocess.check_output(["node", "-e", """
const E=require('./core/engine.js'),fs=require('fs');const raw=JSON.parse(fs.readFileSync('data/public/form_responses.json'));
const {runs}=E.codeRows(raw.rows,raw.headers);const o={};for(const t of ['see_Blue','bond_Blue','see_shadow','bond_Green']){o[t]=E.rankRoutes(runs,t,10)}
console.log(JSON.stringify(o));"""]))
MISSING = {"Unknown", "Unsure", "Unsure/unknown", "Multiple", "(not recorded)", "(not asked)", "(not asked/blank)", "Unknown/lost count", "Blank/not asked"}
miss = lambda v: not isinstance(v, str) or v == "" or v in MISSING # noqa: E731
def stage(r):
# a route is only ranked from runs whose every step is known (mirrors rankRoutes in engine.js)
if r.scream.startswith("DIED") or miss(r.scream) or miss(r.resolution): cad = None # noqa
else: cad = ("Help → " + r.resolution) if r.scream == "Investigate & help" else (r.scream + " → " + r.resolution)
hour = {"Tree": "Climb the tree", "Valley": "Valley centre"}.get(r.hour)
fs = None if r.scream == "DIED at first sighting" or miss(r.first_sighting) else r.first_sighting
post_null = not isinstance(r.post_path, str)
post = None if post_null or miss(r.post_path) else r.post_path
vals = [None if miss(r.company) else r.company, None if miss(r.direction) else r.direction, fs, cad, post, hour]
if any(v is None for i, v in enumerate(vals) if not (i == 4 and post_null)): return None
return " › ".join(v if v else "—" for v in vals)
def stratum(r):
if r.source != "Form": return "bulk"
return {"Blue": "blue", "Black": "black", "Blue or Black": "blue-or-black"}.get(r.goal, "other")
d["route"] = d.apply(stage, axis=1); d["stratum"] = d.apply(stratum, axis=1)
d["branch"] = d.branch.where(d.branch.notna(), None)
T = {"see_Blue": (lambda x: x.branch == "Passed the Red: meadow", lambda x: x.saw_blue_meadow),
"bond_Blue": (lambda x: x.hour == "Valley", lambda x: (x.outcome == "Bonded") & (x.outcome_color == "Blue")),
"see_shadow": (lambda x: (x.hour == "Tree") & x.branch.notna(), lambda x: x.saw_shadow),
"bond_Green": (lambda x: x.hour == "Tree", lambda x: (x.outcome == "Bonded") & (x.outcome_color == "Green"))}
fails = 0
for t, (pool, ev) in T.items():
P = d[pool(d)].copy(); P["e"] = ev(P).astype(bool)
K, N = int(P.e.sum()), len(P)
rows = {}
for route, g in P[P.route.notna()].groupby("route"):
if len(g) < 10: continue
k, n = int(g.e.sum()), len(g)
orr = odds_ratio(k, n - k, K - k, N - n - (K - k))
tabs = []
for s, sg in P.groupby("stratum"):
inr = sg.route == route
a = int((inr & sg.e).sum()); nin = int(inr.sum()); c = int((~inr & sg.e).sum()); nout = len(sg) - nin
tabs.append([a, nin - a, c, nout - c])
rows[route] = (k, n, orr, mantel_haenszel(tabs))
jr = {r["route"]: r for r in js[t]["rows"]}
if set(jr) != set(rows): print(t, "route sets differ", set(jr) ^ set(rows)); fails += 1
for route, (k, n, orr, mh) in rows.items():
j = jr.get(route)
if not j: continue
if (k, n) != (j["k"], j["n"]) or abs(orr[0] - j["or"]) > 1e-6 * max(1, orr[0]) or abs(orr[3] - j["fisher_p"]) > 1e-6:
print("crude diff", t, route); fails += 1
if mh is None:
if j["mh_or"] is not None: print("mh none diff", t, route); fails += 1
elif j["mh_or"] is None or abs(mh[0] - j["mh_or"]) > 1e-6 * max(1, mh[0]) or abs(mh[3] - j["mh_p"]) > 1e-5:
print("mh diff", t, route, mh, j["mh_or"], j["mh_p"]); fails += 1
print(f"route rankings for {len(T)} outcomes: {'ALL MATCH' if not fails else str(fails) + ' differences'}")
sys.exit(1 if fails else 0)
-21
View File
@@ -1,21 +0,0 @@
"""Check that the JS engine (core/engine.js) and the Python coding rules (core/atlas/coding.py) code every run identically."""
import json, sys
import pandas as pd
js = pd.DataFrame(json.load(open(sys.argv[1] if len(sys.argv) > 1 else "tests/out/runs_js.json")))
py = pd.read_csv(sys.argv[2] if len(sys.argv) > 2 else "data/derived/runs_clean.csv", keep_default_na=False, na_values=[""])
def norm(v):
if v is None or (not isinstance(v, str) and pd.isna(v)) or v == "":
return None
if isinstance(v, float) and v.is_integer():
return str(int(v))
if isinstance(v, bool):
return str(v)
return str(v)
bad = {}
for c in py.columns:
a, b = [norm(v) for v in py[c]], [norm(v) for v in js[c]]
d = [(i, x, y) for i, (x, y) in enumerate(zip(a, b)) if x != y]
if d:
bad[c] = (len(d), d[:3])
print("runs:", len(py), len(js), "| mismatching columns:", bad or "NONE")
sys.exit(1 if bad else 0)
-37
View File
@@ -1,37 +0,0 @@
"""The Python route model (core/atlas/solver.py, built from core/spec/game.yaml) must give the same policy as the
JavaScript one the site and the bot still use (core/solver.js): same choices everywhere, same numbers.
A quick version for CI: one colour, both samples' coding, 3 posterior draws.
python tests/parity/compare_solver.py
"""
import json
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "core"))
sys.path.insert(0, str(ROOT / "planner"))
sys.path.insert(0, str(Path(__file__).parent))
import build_policy # noqa: E402
from atlas import solver # noqa: E402
from compare_policy import compare # noqa: E402
COLOURS, DRAWS = ["Blue"], 3
runs = build_policy.coded_runs(ROOT / "data/public/form_responses.json")
py = build_policy.clean(solver.build_policy(runs, draws=DRAWS, colours=COLOURS))
js = json.loads(subprocess.check_output(["node", "-e", f"""
const S=require('./core/solver.js'),E=require('./core/engine.js'),fs=require('fs');
const raw=JSON.parse(fs.readFileSync('data/public/form_responses.json'));const {{runs}}=E.codeRows(raw.rows,raw.headers);
console.log(JSON.stringify(S.buildPolicy(runs,{{draws:{DRAWS},colours:{json.dumps(COLOURS)}}})));"""], cwd=ROOT))
with tempfile.TemporaryDirectory() as d:
a, b = Path(d, "js.json"), Path(d, "py.json")
a.write_text(json.dumps(js))
b.write_text(json.dumps(py))
o = compare(a, b, 1e-9)
bad = o["keys"] + o["other"] + o["numbers"]
print(f"route model ({', '.join(COLOURS)}, {DRAWS} draws): " + ("ALL MATCH" if not bad else f"{len(bad)} DIFFER"), f"(largest relative difference {o['max']:.2g})")
for x in bad[:10]:
print(" ", x)
sys.exit(1 if bad else 0)
-122
View File
@@ -1,122 +0,0 @@
"""The precomputed statistics (analysis/stats/site_stats.py -> stats.json) must match what core/engine.js computed
in the browser before, for every preset: factor tables, models, encounters, chance events, route rankings at every
threshold, and the route map.
python tests/parity/compare_stats.py [stats.json]
"""
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
stats = json.load(open(sys.argv[1] if len(sys.argv) > 1 else ROOT / "data/public/stats.json"))
presets = {k: v["analyse"]["filters"] for k, v in stats["presets"].items()}
targets = list(stats["meta"]["targets"])
th = stats["meta"]["route_thresholds"]
js = json.loads(subprocess.run(["node", "-e", f"""
const E=require('./core/engine.js'),fs=require('fs');const raw=JSON.parse(fs.readFileSync('data/public/form_responses.json'));
const {{runs}}=E.codeRows(raw.rows,raw.headers);const out={{}};
for (const [k,f] of Object.entries({json.dumps(presets)})) {{
const sub=E.applyFilters(runs,f); const o={{analyse:E.analyse(runs,f), routes:{{}}, map:{{}}}};
for (const t of {json.dumps(targets)}) {{ o.routes[t]={{}}; for (const m of {json.dumps(th)}) o.routes[t][m]=E.rankRoutes(sub,t,m); o.map[t]=E.buildGraph(sub,t,{{}}); }}
out[k]=o; }}
process.stdout.write(JSON.stringify(out));"""], cwd=ROOT, capture_output=True, text=True, check=True).stdout)
bad = []
def close(a, b, tol):
if a is None or b is None:
return a is None and b is None
if isinstance(a, bool) or isinstance(b, bool) or isinstance(a, str) or isinstance(b, str):
return a == b
return abs(a - b) <= tol * max(1, abs(a), abs(b))
def same(path, a, b, fields, tol=1e-9):
for f in fields:
if not close(a.get(f), b.get(f), tol): # a field left out counts as empty
bad.append(f"{path}.{f}: python {a.get(f)!r} vs js {b.get(f)!r}")
W = ("k", "n", "p", "lo", "hi")
for pid, P in stats["presets"].items():
py, J = P["analyse"], js[pid]["analyse"]
if py["n"] != J["n"]:
bad.append(f"{pid}: n {py['n']} vs {J['n']}")
same(f"{pid}.meta", py["meta"], J["meta"], ("date_min", "date_max", "n_flagged"))
for k in ("sources", "flag_counts", "outcomes"):
if py["meta"][k] != J["meta"][k]:
bad.append(f"{pid}.meta.{k} differ")
for key, c in py["chance"].items():
same(f"{pid}.chance.{key}", c["overall"], J["chance"][key]["overall"], W)
for grp in ("by_stratum", "by_goal", "by_attempt"):
if set(c[grp]) != set(J["chance"][key][grp]):
bad.append(f"{pid}.chance.{key}.{grp} keys differ")
for s in c[grp]:
same(f"{pid}.chance.{key}.{grp}.{s}", c[grp][s], J["chance"][key][grp].get(s, {}), W)
if py["first_sighting"] != J["first_sighting"]:
bad.append(f"{pid}.first_sighting differ")
for a, b in zip(py["encounters"], J["encounters"]):
same(f"{pid}.enc.{a['context']}.overall", a["overall"], b["overall"], W)
same(f"{pid}.enc.{a['context']}.intent", a["intent_matched"], b["intent_matched"], W)
for k in ("green_encounters", "bonded_green", "rescued_orange", "rescued_brown", "incinerated"):
if py["green_rescue"][k] != J["green_rescue"][k]:
bad.append(f"{pid}.green_rescue.{k}")
for key, ft in py["factors"].items():
jr = {(r["factor"], r["level"]): r for r in J["factors"][key]["rows"]}
if len(jr) != len(ft["rows"]):
bad.append(f"{pid}.factors.{key}: {len(ft['rows'])} rows vs {len(jr)}")
for r in ft["rows"]:
j = jr.get((r["factor"], r["level"]))
if j is None:
bad.append(f"{pid}.factors.{key}: {r['factor']}={r['level']} missing in JS")
continue
same(f"{pid}.factors.{key}.{r['factor']}={r['level']}", r, j, W + ("or", "or_lo", "or_hi"))
same(f"{pid}.factors.{key}.{r['factor']}={r['level']}", r, j, ("fisher_p", "q"), 1e-5)
for key, m in py["models"].items():
jm = J["models"][key]
if (m is None) != (jm is None):
bad.append(f"{pid}.models.{key} missing on one side")
continue
if m is None:
continue
same(f"{pid}.models.{key}", m, jm, ("n", "events"))
jt = {t["term"]: t for t in jm["terms"]}
for t in m["terms"]:
same(f"{pid}.models.{key}.{t['term']}", t, jt.get(t["term"], {}), ("or", "lo", "hi"), 1e-4)
same(f"{pid}.models.{key}.{t['term']}", t, jt.get(t["term"], {}), ("p",), 1e-6)
for a, b in zip(py["daily"], J["daily"]):
same(f"{pid}.daily.{a['date']}", a, b, ("runs", "meadow_n", "blue_k", "tree_n", "shadow_k"))
for t, R in P["routes"].items():
for m in th:
jr = js[pid]["routes"][t][str(m)]
bt = R["by_threshold"][str(m)]
if bt["n_routes_tested"] != jr["n_routes_tested"]:
bad.append(f"{pid}.routes.{t}.{m}: {bt['n_routes_tested']} routes vs {jr['n_routes_tested']}")
jrows = {r["route"]: r for r in jr["rows"]}
for i, r in enumerate(R["rows"]):
if r["n"] < m:
continue
j = jrows.get(r["route"])
if j is None:
bad.append(f"{pid}.routes.{t}.{m}: {r['route']} missing in JS")
continue
same(f"{pid}.routes.{t}.{m}.{r['route']}", r, j, W + ("or", "or_lo", "or_hi", "mh_or", "mh_lo", "mh_hi"))
same(f"{pid}.routes.{t}.{m}.{r['route']}", r, j, ("fisher_p", "mh_p"), 1e-5)
same(f"{pid}.routes.{t}.{m}.{r['route']}", {"q": bt["q"].get(str(i)), "mh_q": bt["mh_q"].get(str(i))}, j, ("q", "mh_q"), 1e-5)
for t in targets:
jn = {n["id"]: n for n in js[pid]["map"][t]["nodes"]}
jl = {l["id"]: l for l in js[pid]["map"][t]["links"]}
for n, w in zip(P["map"]["nodes"], P["map"]["by_target"][t]["nodes"]):
same(f"{pid}.map.{t}.{n['id']}", {**n, **w}, jn.get(n["id"], {}), W)
for l, w in zip(P["map"]["links"], P["map"]["by_target"][t]["links"]):
same(f"{pid}.map.{t}.{l['id']}", {**l, **w}, jl.get(l["id"], {}), W + ("chance",))
if len(jn) != len(P["map"]["nodes"]) or len(jl) != len(P["map"]["links"]):
bad.append(f"{pid}.map.{t}: size differs")
print(f"precomputed statistics, {len(stats['presets'])} presets: " + ("ALL MATCH" if not bad else f"{len(bad)} DIFFER"))
for b in bad[:25]:
print(" ", b)
sys.exit(1 if bad else 0)
-13
View File
@@ -1,13 +0,0 @@
// node tests/parity/run_engine.js data/public/form_responses.json out_dir
// Runs the browser engine in Node and writes coded runs + results, for comparison with Python.
const fs = require("fs");
const path = require("path");
const E = require("../../core/engine.js");
const [, , src = "data/public/form_responses.json", outDir = "tests/out"] = process.argv;
const raw = JSON.parse(fs.readFileSync(src, "utf8"));
const { runs, missing } = E.codeRows(raw.rows, raw.headers);
fs.mkdirSync(outDir, { recursive: true });
fs.writeFileSync(path.join(outDir, "runs_js.json"), JSON.stringify(runs));
const res = E.analyse(runs, {});
fs.writeFileSync(path.join(outDir, "results_js.json"), JSON.stringify(res, null, 1));
console.log(`${runs.length} runs coded; missing columns: ${missing.join(", ") || "none"}`);