Files
threshing-atlas/research/python/analyze.py
T
ekcSchianoandClaude Opus 5.5 46ee08a56b Coding rules for the redesigned community form; import the new data
The Threshing Routes form was redesigned (5,651 responses). Both implementations
(engine.js and clean.py) now read either version:
- one direction question + one colour question, with the older per-direction
  columns used only to fill gaps and cross-check
- cadet encounter as an action path (scream, first and second combat move)
  plus a separate outcome
- new fields: sighting_action, combat_second, rare_reported, malek_intent,
  left_squadmates, tail; "Why are you doing this run?" mapped onto the goal
  ("Blue or Black" for rare chasers)
- the rare-dragon question feeds saw_blue_meadow / saw_shadow and the branch
The old export still codes identically. JS/Python parity: all match.

Data: form 2,573 -> 5,651 rows, Dragon Register 216 -> 273, new route PDF
fingerprint. Supersedes the automated pull request #2.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-01 22:10:51 +00:00

135 lines
5.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
analyze.py — Python reference implementation of the statistics shown on the route-map page.
python analyze.py ../../data/derived/runs_clean.csv ../../tests/out/results_py.json
It mirrors analyse() in packages/core/src/engine.js (same denominators, same tests, same model); tests/compare_results.py
checks that both produce the same numbers. Edit either, then re-run the comparison.
"""
import json
import sys
import numpy as np
import pandas as pd
from stats_core import Z95, bh, logistic_ridge, odds_ratio, p_from_z, wilson
COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
FACTORS = ["company", "direction", "first_sighting", "scream", "resolution", "combat_first",
"post_path", "hour", "wing", "squad", "attempt_group", "source", "goal", "sighting_action", "combat_second", "malek_intent"]
def W(k, n):
p, lo, hi = wilson(k, n)
return {"k": int(k), "n": int(n), "p": p, "lo": lo, "hi": hi}
def outcomes():
o = {
"blue_meadow": (lambda d: d.branch == "Passed the Red: meadow", lambda d: d.saw_blue_meadow),
"shadow": (lambda d: (d.hour == "Tree") & d.branch.notna(), lambda d: d.saw_shadow),
"bond_blue_meadow": (lambda d: (d.branch == "Passed the Red: meadow") & d.enc_Blue.isin(["Bonded!", "Incinerated"]),
lambda d: d.enc_Blue == "Bonded!"),
"reach_hour": (lambda d: pd.Series(True, index=d.index), lambda d: d.hour.isin(["Tree", "Valley"])),
}
for c in COLORS:
o["bonded_" + c] = (lambda d: d.hour.isin(["Tree", "Valley"]),
(lambda c: lambda d: (d.outcome == "Bonded") & (d.outcome_color == c))(c))
return o
OUT = outcomes()
def factor_table(d, key, min_n=5):
den, ev = OUT[key]
D = d[den(d)].copy()
D["_e"] = ev(D).astype(bool)
rows = []
for f in FACTORS:
sub = D[D[f].notna()]
nf, ef = len(sub), int(sub._e.sum())
for lv in sorted(sub[f].astype(str).unique()):
g = sub[sub[f].astype(str) == lv]
k, n = int(g._e.sum()), len(g)
row = {"factor": f, "level": lv, **W(k, n)}
rk, rn = ef - k, nf - n
if n >= min_n and rn >= min_n and 0 < ef < nf:
or_, lo, hi, p, corr = odds_ratio(k, n - k, rk, rn - rk)
row.update({"or": or_, "or_lo": lo, "or_hi": hi, "fisher_p": p, "corrected": corr})
rows.append(row)
tested = [r for r in rows if "fisher_p" in r]
for r, q in zip(tested, bh([r["fisher_p"] for r in tested])):
r["q"] = q
return {"outcome": key, "overall": W(int(D._e.sum()), len(D)), "rows": rows}
def model(d, key, factors, lam=1.0):
den, ev = OUT[key]
D = d[den(d)].copy()
y = ev(D).astype(int).values
F = pd.DataFrame({name: D.apply(fn, axis=1) for name, fn in factors.items()}, index=D.index)
keep = F.notna().all(axis=1).values
F, y = F[keep], y[keep]
cols, names = [np.ones(len(F))], []
refs = {}
for f in factors:
counts = F[f].value_counts()
levels = sorted(counts.index, key=lambda lv: (-counts[lv], lv))
refs[f] = levels[0]
for lv in levels[1:]:
cols.append((F[f] == lv).astype(float).values)
names.append((f, lv, int(counts[lv])))
X = np.column_stack(cols)
b, se = logistic_ridge(X, y, lam)
terms = [{"term": f"{f} = {lv}", "factor": f, "level": lv, "n": n, "or": float(np.exp(b[i + 1])),
"lo": float(np.exp(b[i + 1] - Z95 * se[i + 1])), "hi": float(np.exp(b[i + 1] + Z95 * se[i + 1])),
"p": float(p_from_z(b[i + 1] / se[i + 1]))} for i, (f, lv, n) in enumerate(names)]
return {"outcome": key, "n": int(len(y)), "events": int(y.sum()), "refs": refs, "lambda": lam, "terms": terms}
def goal2(r):
if r.goal in ("Blue", "Black", "Blue or Black"):
return r.goal
return "Blank/not asked" if r.goal == "(not asked/blank)" else "Other colour/any"
def src2(r):
return "Form submission" if r.source == "Form" else "Bulk log (import)"
def first2(r):
return r.first_sighting if r.first_sighting in ("Red", "Brown", "Orange", "Green") else "Unsure/unknown"
def period(r):
if not isinstance(r.date, str):
return "No date (import)"
return "Oct 4–10" if r.date < "2026-10-11" else "Oct 11 onward"
def analyse(d):
res = {"n": len(d), "factors": {k: factor_table(d, k) for k in OUT}}
res["models"] = {
"blue_meadow": model(d, "blue_meadow", {
"source": src2, "goal": goal2, "direction": lambda r: r.direction, "first_sighting": first2,
"resolution": lambda r: r.resolution, "company": lambda r: None if r.company == "Unknown" else r.company}),
"shadow": model(d, "shadow", {
"source": src2, "goal": goal2, "attempt": lambda r: r.attempt_group,
"direction": lambda r: r.direction}),
"bond_blue_meadow": model(d, "bond_blue_meadow", {
"source": src2, "goal": goal2, "direction": lambda r: r.direction, "resolution": lambda r: r.resolution}),
}
return res
if __name__ == "__main__":
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/derived/runs_clean.csv"
out = sys.argv[2] if len(sys.argv) > 2 else "../../tests/out/results_py.json"
d = pd.read_csv(src, keep_default_na=False, na_values=[""])
for c in ("saw_blue_meadow", "saw_shadow", "rescued"):
d[c] = d[c].astype(str) == "True"
res = analyse(d)
json.dump(res, open(out, "w"), indent=1, default=float)
print("wrote", out)