Files
threshing-atlas/tests/compare_routes.py
T
ekcSchianoandClaude Opus 5.5 1b0ef9a2c1 Treat unsure and blank answers as missing information, not choices
- engine.js / analyze.py: one MISSING set ("Unknown", "Unsure",
  "Unsure/unknown", "Multiple", "(not recorded)", "(not asked)", ...).
  Factor tables compare only runs that answered each question and report how
  many were left out; models are complete-case for every choice variable;
  route rankings group only fully known routes; the route flow skips
  missing steps.
- "I've lost count" now counts as a later attempt.
- The chase goal keeps its "not asked" stratum as an adjustment only; it is
  no longer displayed as a finding.
- Page: no unknown levels in charts, tables or filters; factor explorer notes
  how many runs were left out; Methods explains the rule.
JS/Python parity still exact.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-02 16:14:08 +00:00

61 lines
4.0 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Recompute route rankings in Python (crude Fisher and stratified Mantel-Haenszel) and compare with engine.js."""
import json, subprocess, sys
sys.path.insert(0, "research/python")
import pandas as pd
from stats_core import odds_ratio, mantel_haenszel, bh
d = pd.DataFrame(json.load(open("tests/out/runs_js.json")))
js = json.loads(subprocess.check_output(["node", "-e", """
const E=require('./packages/core/src/engine.js'),fs=require('fs');const raw=JSON.parse(fs.readFileSync('data/public/form_responses.json'));
const {runs}=E.codeRows(raw.rows,raw.headers);const o={};for(const t of ['see_Blue','bond_Blue','see_shadow','bond_Green']){o[t]=E.rankRoutes(runs,t,10)}
console.log(JSON.stringify(o));"""]))
MISSING = {"Unknown", "Unsure", "Unsure/unknown", "Multiple", "(not recorded)", "(not asked)", "(not asked/blank)", "Unknown/lost count", "Blank/not asked"}
miss = lambda v: not isinstance(v, str) or v == "" or v in MISSING # noqa: E731
def stage(r):
# a route is only ranked from runs whose every step is known (mirrors rankRoutes in engine.js)
if r.scream.startswith("DIED") or miss(r.scream) or miss(r.resolution): cad = None # noqa
else: cad = ("Help → " + r.resolution) if r.scream == "Investigate & help" else (r.scream + " → " + r.resolution)
hour = {"Tree": "Climb the tree", "Valley": "Valley centre"}.get(r.hour)
fs = None if r.scream == "DIED at first sighting" or miss(r.first_sighting) else r.first_sighting
post_null = not isinstance(r.post_path, str)
post = None if post_null or miss(r.post_path) else r.post_path
vals = [None if miss(r.company) else r.company, None if miss(r.direction) else r.direction, fs, cad, post, hour]
if any(v is None for i, v in enumerate(vals) if not (i == 4 and post_null)): return None
return " › ".join(v if v else "—" for v in vals)
def stratum(r):
if r.source != "Form": return "bulk"
return {"Blue": "blue", "Black": "black", "Blue or Black": "blue-or-black"}.get(r.goal, "other")
d["route"] = d.apply(stage, axis=1); d["stratum"] = d.apply(stratum, axis=1)
d["branch"] = d.branch.where(d.branch.notna(), None)
T = {"see_Blue": (lambda x: x.branch == "Passed the Red: meadow", lambda x: x.saw_blue_meadow),
"bond_Blue": (lambda x: x.hour == "Valley", lambda x: (x.outcome == "Bonded") & (x.outcome_color == "Blue")),
"see_shadow": (lambda x: (x.hour == "Tree") & x.branch.notna(), lambda x: x.saw_shadow),
"bond_Green": (lambda x: x.hour == "Tree", lambda x: (x.outcome == "Bonded") & (x.outcome_color == "Green"))}
fails = 0
for t, (pool, ev) in T.items():
P = d[pool(d)].copy(); P["e"] = ev(P).astype(bool)
K, N = int(P.e.sum()), len(P)
rows = {}
for route, g in P[P.route.notna()].groupby("route"):
if len(g) < 10: continue
k, n = int(g.e.sum()), len(g)
orr = odds_ratio(k, n - k, K - k, N - n - (K - k))
tabs = []
for s, sg in P.groupby("stratum"):
inr = sg.route == route
a = int((inr & sg.e).sum()); nin = int(inr.sum()); c = int((~inr & sg.e).sum()); nout = len(sg) - nin
tabs.append([a, nin - a, c, nout - c])
rows[route] = (k, n, orr, mantel_haenszel(tabs))
jr = {r["route"]: r for r in js[t]["rows"]}
if set(jr) != set(rows): print(t, "route sets differ", set(jr) ^ set(rows)); fails += 1
for route, (k, n, orr, mh) in rows.items():
j = jr.get(route)
if not j: continue
if (k, n) != (j["k"], j["n"]) or abs(orr[0] - j["or"]) > 1e-6 * max(1, orr[0]) or abs(orr[3] - j["fisher_p"]) > 1e-6:
print("crude diff", t, route); fails += 1
if mh is None:
if j["mh_or"] is not None: print("mh none diff", t, route); fails += 1
elif j["mh_or"] is None or abs(mh[0] - j["mh_or"]) > 1e-6 * max(1, mh[0]) or abs(mh[3] - j["mh_p"]) > 1e-5:
print("mh diff", t, route, mh, j["mh_or"], j["mh_p"]); fails += 1
print(f"route rankings for {len(T)} outcomes: {'ALL MATCH' if not fails else str(fails) + ' differences'}")
sys.exit(1 if fails else 0)