""" analyze.py — Python reference implementation of the statistics shown on the route-map page. python analyze.py ../../data/derived/runs_clean.csv ../../tests/out/results_py.json It mirrors analyse() in packages/core/src/engine.js (same denominators, same tests, same model); tests/compare_results.py checks that both produce the same numbers. Edit either, then re-run the comparison. """ import json import sys import numpy as np import pandas as pd from stats_core import Z95, bh, logistic_ridge, odds_ratio, p_from_z, wilson COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"] FACTORS = ["company", "direction", "first_sighting", "scream", "resolution", "combat_first", "post_path", "hour", "wing", "squad", "attempt_group", "source", "goal"] def W(k, n): p, lo, hi = wilson(k, n) return {"k": int(k), "n": int(n), "p": p, "lo": lo, "hi": hi} def outcomes(): o = { "blue_meadow": (lambda d: d.branch == "Passed the Red: meadow", lambda d: d.saw_blue_meadow), "shadow": (lambda d: (d.hour == "Tree") & d.branch.notna(), lambda d: d.saw_shadow), "bond_blue_meadow": (lambda d: (d.branch == "Passed the Red: meadow") & d.enc_Blue.isin(["Bonded!", "Incinerated"]), lambda d: d.enc_Blue == "Bonded!"), "reach_hour": (lambda d: pd.Series(True, index=d.index), lambda d: d.hour.isin(["Tree", "Valley"])), } for c in COLORS: o["bonded_" + c] = (lambda d: d.hour.isin(["Tree", "Valley"]), (lambda c: lambda d: (d.outcome == "Bonded") & (d.outcome_color == c))(c)) return o OUT = outcomes() def factor_table(d, key, min_n=5): den, ev = OUT[key] D = d[den(d)].copy() D["_e"] = ev(D).astype(bool) rows = [] for f in FACTORS: sub = D[D[f].notna()] nf, ef = len(sub), int(sub._e.sum()) for lv in sorted(sub[f].astype(str).unique()): g = sub[sub[f].astype(str) == lv] k, n = int(g._e.sum()), len(g) row = {"factor": f, "level": lv, **W(k, n)} rk, rn = ef - k, nf - n if n >= min_n and rn >= min_n and 0 < ef < nf: or_, lo, hi, p, corr = odds_ratio(k, n - k, rk, rn - rk) row.update({"or": or_, "or_lo": lo, "or_hi": hi, "fisher_p": p, "corrected": corr}) rows.append(row) tested = [r for r in rows if "fisher_p" in r] for r, q in zip(tested, bh([r["fisher_p"] for r in tested])): r["q"] = q return {"outcome": key, "overall": W(int(D._e.sum()), len(D)), "rows": rows} def model(d, key, factors, lam=1.0): den, ev = OUT[key] D = d[den(d)].copy() y = ev(D).astype(int).values F = pd.DataFrame({name: D.apply(fn, axis=1) for name, fn in factors.items()}, index=D.index) keep = F.notna().all(axis=1).values F, y = F[keep], y[keep] cols, names = [np.ones(len(F))], [] refs = {} for f in factors: counts = F[f].value_counts() levels = sorted(counts.index, key=lambda lv: (-counts[lv], lv)) refs[f] = levels[0] for lv in levels[1:]: cols.append((F[f] == lv).astype(float).values) names.append((f, lv, int(counts[lv]))) X = np.column_stack(cols) b, se = logistic_ridge(X, y, lam) terms = [{"term": f"{f} = {lv}", "factor": f, "level": lv, "n": n, "or": float(np.exp(b[i + 1])), "lo": float(np.exp(b[i + 1] - Z95 * se[i + 1])), "hi": float(np.exp(b[i + 1] + Z95 * se[i + 1])), "p": float(p_from_z(b[i + 1] / se[i + 1]))} for i, (f, lv, n) in enumerate(names)] return {"outcome": key, "n": int(len(y)), "events": int(y.sum()), "refs": refs, "lambda": lam, "terms": terms} def goal2(r): if r.goal in ("Blue", "Black"): return r.goal return "Blank/not asked" if r.goal == "(not asked/blank)" else "Other colour/any" def src2(r): return "Form submission" if r.source == "Form" else "Bulk log (import)" def first2(r): return r.first_sighting if r.first_sighting in ("Red", "Brown", "Orange", "Green") else "Unsure/unknown" def period(r): if not isinstance(r.date, str): return "No date (import)" return "Oct 4–10" if r.date < "2026-10-11" else "Oct 11 onward" def analyse(d): res = {"n": len(d), "factors": {k: factor_table(d, k) for k in OUT}} res["models"] = { "blue_meadow": model(d, "blue_meadow", { "source": src2, "goal": goal2, "direction": lambda r: r.direction, "first_sighting": first2, "resolution": lambda r: r.resolution, "company": lambda r: None if r.company == "Unknown" else r.company}), "shadow": model(d, "shadow", { "source": src2, "goal": goal2, "attempt": lambda r: r.attempt_group, "direction": lambda r: r.direction}), "bond_blue_meadow": model(d, "bond_blue_meadow", { "source": src2, "goal": goal2, "direction": lambda r: r.direction, "resolution": lambda r: r.resolution}), } return res if __name__ == "__main__": src = sys.argv[1] if len(sys.argv) > 1 else "../../data/derived/runs_clean.csv" out = sys.argv[2] if len(sys.argv) > 2 else "../../tests/out/results_py.json" d = pd.read_csv(src, keep_default_na=False, na_values=[""]) for c in ("saw_blue_meadow", "saw_shadow", "rescued"): d[c] = d[c].astype(str) == "True" res = analyse(d) json.dump(res, open(out, "w"), indent=1, default=float) print("wrote", out)