Site: show precomputed statistics; drop the upload and the click-to-select route map (#12)
Step 4a of the Python-first core. - analysis/stats/measures.yaml (declarative): outcomes and denominators, route targets and comparison groups, encounter contexts, the route map's stages, six filter presets and the route thresholds (5, 10, 20, 30). - analysis/stats/site_stats.py computes every number the page shows, for every preset, and writes data/public/stats.json: factor tables, models, chance events, encounters, the route map for every outcome, route rankings with q-values at every threshold, the valley map's place and step counts, and The Odds' summary cards. - The page now only displays: app.js, ui.js and explorer.js read stats.json; engine.js is no longer loaded. The three filter controls become one "Runs" menu; minimum runs become fixed thresholds. Removed: "Load a newer route export", click-to-select and "Follow" on the Deep Dive route map, and the unused old valley renderer. The coded runs are a CSV download. - tests/parity/compare_stats.py: every number matches what engine.js computed, for all six presets (factor tables, models, encounters, route rankings at every threshold, route map). Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
963b08c522
commit
d04b88e7b4
@@ -0,0 +1,122 @@
|
||||
"""The precomputed statistics (analysis/stats/site_stats.py -> stats.json) must match what core/engine.js computed
|
||||
in the browser before, for every preset: factor tables, models, encounters, chance events, route rankings at every
|
||||
threshold, and the route map.
|
||||
|
||||
python tests/parity/compare_stats.py [stats.json]
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
stats = json.load(open(sys.argv[1] if len(sys.argv) > 1 else ROOT / "data/public/stats.json"))
|
||||
presets = {k: v["analyse"]["filters"] for k, v in stats["presets"].items()}
|
||||
targets = list(stats["meta"]["targets"])
|
||||
th = stats["meta"]["route_thresholds"]
|
||||
js = json.loads(subprocess.run(["node", "-e", f"""
|
||||
const E=require('./core/engine.js'),fs=require('fs');const raw=JSON.parse(fs.readFileSync('data/public/form_responses.json'));
|
||||
const {{runs}}=E.codeRows(raw.rows,raw.headers);const out={{}};
|
||||
for (const [k,f] of Object.entries({json.dumps(presets)})) {{
|
||||
const sub=E.applyFilters(runs,f); const o={{analyse:E.analyse(runs,f), routes:{{}}, map:{{}}}};
|
||||
for (const t of {json.dumps(targets)}) {{ o.routes[t]={{}}; for (const m of {json.dumps(th)}) o.routes[t][m]=E.rankRoutes(sub,t,m); o.map[t]=E.buildGraph(sub,t,{{}}); }}
|
||||
out[k]=o; }}
|
||||
process.stdout.write(JSON.stringify(out));"""], cwd=ROOT, capture_output=True, text=True, check=True).stdout)
|
||||
|
||||
bad = []
|
||||
|
||||
|
||||
def close(a, b, tol):
|
||||
if a is None or b is None:
|
||||
return a is None and b is None
|
||||
if isinstance(a, bool) or isinstance(b, bool) or isinstance(a, str) or isinstance(b, str):
|
||||
return a == b
|
||||
return abs(a - b) <= tol * max(1, abs(a), abs(b))
|
||||
|
||||
|
||||
def same(path, a, b, fields, tol=1e-9):
|
||||
for f in fields:
|
||||
if not close(a.get(f), b.get(f), tol): # a field left out counts as empty
|
||||
bad.append(f"{path}.{f}: python {a.get(f)!r} vs js {b.get(f)!r}")
|
||||
|
||||
|
||||
W = ("k", "n", "p", "lo", "hi")
|
||||
for pid, P in stats["presets"].items():
|
||||
py, J = P["analyse"], js[pid]["analyse"]
|
||||
if py["n"] != J["n"]:
|
||||
bad.append(f"{pid}: n {py['n']} vs {J['n']}")
|
||||
same(f"{pid}.meta", py["meta"], J["meta"], ("date_min", "date_max", "n_flagged"))
|
||||
for k in ("sources", "flag_counts", "outcomes"):
|
||||
if py["meta"][k] != J["meta"][k]:
|
||||
bad.append(f"{pid}.meta.{k} differ")
|
||||
for key, c in py["chance"].items():
|
||||
same(f"{pid}.chance.{key}", c["overall"], J["chance"][key]["overall"], W)
|
||||
for grp in ("by_stratum", "by_goal", "by_attempt"):
|
||||
if set(c[grp]) != set(J["chance"][key][grp]):
|
||||
bad.append(f"{pid}.chance.{key}.{grp} keys differ")
|
||||
for s in c[grp]:
|
||||
same(f"{pid}.chance.{key}.{grp}.{s}", c[grp][s], J["chance"][key][grp].get(s, {}), W)
|
||||
if py["first_sighting"] != J["first_sighting"]:
|
||||
bad.append(f"{pid}.first_sighting differ")
|
||||
for a, b in zip(py["encounters"], J["encounters"]):
|
||||
same(f"{pid}.enc.{a['context']}.overall", a["overall"], b["overall"], W)
|
||||
same(f"{pid}.enc.{a['context']}.intent", a["intent_matched"], b["intent_matched"], W)
|
||||
for k in ("green_encounters", "bonded_green", "rescued_orange", "rescued_brown", "incinerated"):
|
||||
if py["green_rescue"][k] != J["green_rescue"][k]:
|
||||
bad.append(f"{pid}.green_rescue.{k}")
|
||||
for key, ft in py["factors"].items():
|
||||
jr = {(r["factor"], r["level"]): r for r in J["factors"][key]["rows"]}
|
||||
if len(jr) != len(ft["rows"]):
|
||||
bad.append(f"{pid}.factors.{key}: {len(ft['rows'])} rows vs {len(jr)}")
|
||||
for r in ft["rows"]:
|
||||
j = jr.get((r["factor"], r["level"]))
|
||||
if j is None:
|
||||
bad.append(f"{pid}.factors.{key}: {r['factor']}={r['level']} missing in JS")
|
||||
continue
|
||||
same(f"{pid}.factors.{key}.{r['factor']}={r['level']}", r, j, W + ("or", "or_lo", "or_hi"))
|
||||
same(f"{pid}.factors.{key}.{r['factor']}={r['level']}", r, j, ("fisher_p", "q"), 1e-5)
|
||||
for key, m in py["models"].items():
|
||||
jm = J["models"][key]
|
||||
if (m is None) != (jm is None):
|
||||
bad.append(f"{pid}.models.{key} missing on one side")
|
||||
continue
|
||||
if m is None:
|
||||
continue
|
||||
same(f"{pid}.models.{key}", m, jm, ("n", "events"))
|
||||
jt = {t["term"]: t for t in jm["terms"]}
|
||||
for t in m["terms"]:
|
||||
same(f"{pid}.models.{key}.{t['term']}", t, jt.get(t["term"], {}), ("or", "lo", "hi"), 1e-4)
|
||||
same(f"{pid}.models.{key}.{t['term']}", t, jt.get(t["term"], {}), ("p",), 1e-6)
|
||||
for a, b in zip(py["daily"], J["daily"]):
|
||||
same(f"{pid}.daily.{a['date']}", a, b, ("runs", "meadow_n", "blue_k", "tree_n", "shadow_k"))
|
||||
for t, R in P["routes"].items():
|
||||
for m in th:
|
||||
jr = js[pid]["routes"][t][str(m)]
|
||||
bt = R["by_threshold"][str(m)]
|
||||
if bt["n_routes_tested"] != jr["n_routes_tested"]:
|
||||
bad.append(f"{pid}.routes.{t}.{m}: {bt['n_routes_tested']} routes vs {jr['n_routes_tested']}")
|
||||
jrows = {r["route"]: r for r in jr["rows"]}
|
||||
for i, r in enumerate(R["rows"]):
|
||||
if r["n"] < m:
|
||||
continue
|
||||
j = jrows.get(r["route"])
|
||||
if j is None:
|
||||
bad.append(f"{pid}.routes.{t}.{m}: {r['route']} missing in JS")
|
||||
continue
|
||||
same(f"{pid}.routes.{t}.{m}.{r['route']}", r, j, W + ("or", "or_lo", "or_hi", "mh_or", "mh_lo", "mh_hi"))
|
||||
same(f"{pid}.routes.{t}.{m}.{r['route']}", r, j, ("fisher_p", "mh_p"), 1e-5)
|
||||
same(f"{pid}.routes.{t}.{m}.{r['route']}", {"q": bt["q"].get(str(i)), "mh_q": bt["mh_q"].get(str(i))}, j, ("q", "mh_q"), 1e-5)
|
||||
for t in targets:
|
||||
jn = {n["id"]: n for n in js[pid]["map"][t]["nodes"]}
|
||||
jl = {l["id"]: l for l in js[pid]["map"][t]["links"]}
|
||||
for n, w in zip(P["map"]["nodes"], P["map"]["by_target"][t]["nodes"]):
|
||||
same(f"{pid}.map.{t}.{n['id']}", {**n, **w}, jn.get(n["id"], {}), W)
|
||||
for l, w in zip(P["map"]["links"], P["map"]["by_target"][t]["links"]):
|
||||
same(f"{pid}.map.{t}.{l['id']}", {**l, **w}, jl.get(l["id"], {}), W + ("chance",))
|
||||
if len(jn) != len(P["map"]["nodes"]) or len(jl) != len(P["map"]["links"]):
|
||||
bad.append(f"{pid}.map.{t}: size differs")
|
||||
|
||||
print(f"precomputed statistics, {len(stats['presets'])} presets: " + ("ALL MATCH" if not bad else f"{len(bad)} DIFFER"))
|
||||
for b in bad[:25]:
|
||||
print(" ", b)
|
||||
sys.exit(1 if bad else 0)
|
||||
Reference in New Issue
Block a user