Files
threshing-atlas/research/python/analyze.py
T
ekcSchianoandClaude Opus 5.5 c3d067b88c Threshing Atlas: monorepo, route graph, bot importer, daily source checks
- apps/atlas: the web page (valley map, route flow, Dragon Register)
- packages/core: coding rules and statistics (engine.js) and the shared route
  graph (graph.js): scene graph, decision-variable graph, bot-to-form crosswalk,
  Beta credible intervals
- packages/ingest: DragonKind bot importer to anonymised trajectories (no prose,
  no account identifiers); data/bot holds Ken's sample
- research/python: independent reference implementation + update_sources.py
- tests: JS/Python parity checks and graph/importer unit tests
- CI, GitHub Pages deploy, daily check of the community sheets and route PDF
- Email addresses in free-text form answers are redacted; the raw spreadsheet
  is not committed
- MIT code licence, CC BY 4.0 data licence

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-01 21:54:54 +00:00

135 lines
5.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
analyze.py — Python reference implementation of the statistics shown on the route-map page.
python analyze.py ../../data/derived/runs_clean.csv ../../tests/out/results_py.json
It mirrors analyse() in packages/core/src/engine.js (same denominators, same tests, same model); tests/compare_results.py
checks that both produce the same numbers. Edit either, then re-run the comparison.
"""
import json
import sys
import numpy as np
import pandas as pd
from stats_core import Z95, bh, logistic_ridge, odds_ratio, p_from_z, wilson
COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
FACTORS = ["company", "direction", "first_sighting", "scream", "resolution", "combat_first",
"post_path", "hour", "wing", "squad", "attempt_group", "source", "goal"]
def W(k, n):
p, lo, hi = wilson(k, n)
return {"k": int(k), "n": int(n), "p": p, "lo": lo, "hi": hi}
def outcomes():
o = {
"blue_meadow": (lambda d: d.branch == "Passed the Red: meadow", lambda d: d.saw_blue_meadow),
"shadow": (lambda d: (d.hour == "Tree") & d.branch.notna(), lambda d: d.saw_shadow),
"bond_blue_meadow": (lambda d: (d.branch == "Passed the Red: meadow") & d.enc_Blue.isin(["Bonded!", "Incinerated"]),
lambda d: d.enc_Blue == "Bonded!"),
"reach_hour": (lambda d: pd.Series(True, index=d.index), lambda d: d.hour.isin(["Tree", "Valley"])),
}
for c in COLORS:
o["bonded_" + c] = (lambda d: d.hour.isin(["Tree", "Valley"]),
(lambda c: lambda d: (d.outcome == "Bonded") & (d.outcome_color == c))(c))
return o
OUT = outcomes()
def factor_table(d, key, min_n=5):
den, ev = OUT[key]
D = d[den(d)].copy()
D["_e"] = ev(D).astype(bool)
rows = []
for f in FACTORS:
sub = D[D[f].notna()]
nf, ef = len(sub), int(sub._e.sum())
for lv in sorted(sub[f].astype(str).unique()):
g = sub[sub[f].astype(str) == lv]
k, n = int(g._e.sum()), len(g)
row = {"factor": f, "level": lv, **W(k, n)}
rk, rn = ef - k, nf - n
if n >= min_n and rn >= min_n and 0 < ef < nf:
or_, lo, hi, p, corr = odds_ratio(k, n - k, rk, rn - rk)
row.update({"or": or_, "or_lo": lo, "or_hi": hi, "fisher_p": p, "corrected": corr})
rows.append(row)
tested = [r for r in rows if "fisher_p" in r]
for r, q in zip(tested, bh([r["fisher_p"] for r in tested])):
r["q"] = q
return {"outcome": key, "overall": W(int(D._e.sum()), len(D)), "rows": rows}
def model(d, key, factors, lam=1.0):
den, ev = OUT[key]
D = d[den(d)].copy()
y = ev(D).astype(int).values
F = pd.DataFrame({name: D.apply(fn, axis=1) for name, fn in factors.items()}, index=D.index)
keep = F.notna().all(axis=1).values
F, y = F[keep], y[keep]
cols, names = [np.ones(len(F))], []
refs = {}
for f in factors:
counts = F[f].value_counts()
levels = sorted(counts.index, key=lambda lv: (-counts[lv], lv))
refs[f] = levels[0]
for lv in levels[1:]:
cols.append((F[f] == lv).astype(float).values)
names.append((f, lv, int(counts[lv])))
X = np.column_stack(cols)
b, se = logistic_ridge(X, y, lam)
terms = [{"term": f"{f} = {lv}", "factor": f, "level": lv, "n": n, "or": float(np.exp(b[i + 1])),
"lo": float(np.exp(b[i + 1] - Z95 * se[i + 1])), "hi": float(np.exp(b[i + 1] + Z95 * se[i + 1])),
"p": float(p_from_z(b[i + 1] / se[i + 1]))} for i, (f, lv, n) in enumerate(names)]
return {"outcome": key, "n": int(len(y)), "events": int(y.sum()), "refs": refs, "lambda": lam, "terms": terms}
def goal2(r):
if r.goal in ("Blue", "Black"):
return r.goal
return "Blank/not asked" if r.goal == "(not asked/blank)" else "Other colour/any"
def src2(r):
return "Form submission" if r.source == "Form" else "Bulk log (import)"
def first2(r):
return r.first_sighting if r.first_sighting in ("Red", "Brown", "Orange", "Green") else "Unsure/unknown"
def period(r):
if not isinstance(r.date, str):
return "No date (import)"
return "Oct 4–10" if r.date < "2026-10-11" else "Oct 11 onward"
def analyse(d):
res = {"n": len(d), "factors": {k: factor_table(d, k) for k in OUT}}
res["models"] = {
"blue_meadow": model(d, "blue_meadow", {
"source": src2, "goal": goal2, "direction": lambda r: r.direction, "first_sighting": first2,
"resolution": lambda r: r.resolution, "company": lambda r: None if r.company == "Unknown" else r.company}),
"shadow": model(d, "shadow", {
"source": src2, "goal": goal2, "attempt": lambda r: r.attempt_group,
"direction": lambda r: r.direction}),
"bond_blue_meadow": model(d, "bond_blue_meadow", {
"source": src2, "goal": goal2, "direction": lambda r: r.direction, "resolution": lambda r: r.resolution}),
}
return res
if __name__ == "__main__":
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/derived/runs_clean.csv"
out = sys.argv[2] if len(sys.argv) > 2 else "../../tests/out/results_py.json"
d = pd.read_csv(src, keep_default_na=False, na_values=[""])
for c in ("saw_blue_meadow", "saw_shadow", "rescued"):
d[c] = d[c].astype(str) == "True"
res = analyse(d)
json.dump(res, open(out, "w"), indent=1, default=float)
print("wrote", out)