CI / test (pull_request) Successful in 44s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
570 lines
25 KiB
Python
570 lines
25 KiB
Python
"""Every number the Atlas page shows, precomputed: data/public/stats.json.
|
||
|
||
The page only displays these; it computes no statistics itself. What is measured (outcomes, route targets,
|
||
encounters, the route map's stages, the filter presets and route thresholds) is declared in measures.yaml.
|
||
|
||
python analysis/stats/site_stats.py [--out data/public/stats.json]
|
||
"""
|
||
import argparse
|
||
import json
|
||
import math
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
import yaml
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
ROOT = HERE.parents[1]
|
||
sys.path.insert(0, str(ROOT / "core"))
|
||
sys.path.insert(0, str(HERE))
|
||
from atlas import coding, model # noqa: E402
|
||
from atlas import runs as runs_io # noqa: E402
|
||
from stats_core import Z95, bh, logistic_ridge, mantel_haenszel, odds_ratio, p_from_z # noqa: E402
|
||
from stats_core import wilson as _wilson # noqa: E402
|
||
|
||
M = yaml.safe_load(open(HERE / "measures.yaml", encoding="utf-8"))
|
||
COLOURS = M["colours"]
|
||
is_missing = coding.is_missing
|
||
|
||
|
||
# ---------------------------------------------------------------- conditions from measures.yaml
|
||
|
||
def match(r, cond):
|
||
for k, want in cond.items():
|
||
if k == "any":
|
||
if not any(match(r, c) for c in want):
|
||
return False
|
||
continue
|
||
v = r.get(k)
|
||
if isinstance(want, dict):
|
||
if "in" in want and v not in want["in"]:
|
||
return False
|
||
if "not" in want and v == want["not"]:
|
||
return False
|
||
if "exists" in want and (v is not None and v != "") != want["exists"]:
|
||
return False
|
||
elif v != want:
|
||
return False
|
||
return True
|
||
|
||
|
||
def _fill(cond, colour):
|
||
return json.loads(json.dumps(cond).replace("{colour}", colour))
|
||
|
||
|
||
OUTCOMES = {k: v for k, v in M["outcomes"].items()}
|
||
for c in COLOURS:
|
||
b = M["bonded_outcome"]
|
||
OUTCOMES["bonded_" + c] = {"label": b["label"].replace("{colour}", c), "denom": b["denom"], "event": _fill(b["event"], c)}
|
||
TARGETS = M["targets"]
|
||
STAGES = M["stages"]
|
||
|
||
|
||
def W(k, n):
|
||
p, lo, hi = _wilson(k, n)
|
||
return {"k": int(k), "n": int(n), "p": p, "lo": lo, "hi": hi}
|
||
|
||
|
||
def rate(runs, pred):
|
||
return W(sum(1 for r in runs if pred(r)), len(runs))
|
||
|
||
|
||
def group_by(runs, key):
|
||
out = {}
|
||
for r in runs:
|
||
k = key(r) if callable(key) else r.get(key)
|
||
out.setdefault(_key(k), []).append(r)
|
||
return out
|
||
|
||
|
||
def _key(v):
|
||
"""Object keys as JavaScript writes them (true/false/null), so the page can look them up as before."""
|
||
if v is True:
|
||
return "true"
|
||
if v is False:
|
||
return "false"
|
||
if v is None:
|
||
return "null"
|
||
if isinstance(v, float) and v.is_integer():
|
||
return str(int(v))
|
||
return str(v)
|
||
|
||
|
||
def is_rare(goal):
|
||
return goal in M["rare_goals"]
|
||
|
||
|
||
def stratum(r):
|
||
if r["entry"] != "Form":
|
||
return "Bulk logs (imports)"
|
||
g = r["goal"]
|
||
return {"Blue": "Form: chasing Blue", "Black": "Form: chasing Black", "Blue or Black": "Form: chasing Blue or Black"}.get(g, "Form: other/blank goal")
|
||
|
||
|
||
def goal2(r):
|
||
return r["goal"] if is_rare(r["goal"]) else "Blank/not asked" if r["goal"] == "(not asked/blank)" else "Other colour/any"
|
||
|
||
|
||
def apply_filters(runs, f):
|
||
def ok(r):
|
||
if f.get("source") == "form" and r["entry"] != "Form":
|
||
return False
|
||
if f.get("source") == "bulk" and r["entry"] == "Form":
|
||
return False
|
||
if f.get("goal") == "exclude_rare" and is_rare(r["goal"]):
|
||
return False
|
||
if f.get("goal") == "only_rare" and not is_rare(r["goal"]):
|
||
return False
|
||
if f.get("excludeFlagged") and r["data_flags"]:
|
||
return False
|
||
return True
|
||
return [r for r in runs if ok(r)]
|
||
|
||
|
||
# ---------------------------------------------------------------- tables and models
|
||
|
||
def odds(a, b, c, d):
|
||
or_, lo, hi, p, corr = odds_ratio(a, b, c, d)
|
||
return {"or": or_, "or_lo": lo, "or_hi": hi, "fisher_p": p, "corrected": corr}
|
||
|
||
|
||
def factor_table(runs, key, min_n=5):
|
||
O = OUTCOMES[key]
|
||
D = [r for r in runs if match(r, O["denom"])]
|
||
ev = lambda r: match(r, O["event"]) # noqa: E731
|
||
rows, missing = [], {}
|
||
for f in M["factors"]:
|
||
known = [r for r in D if not is_missing(r.get(f))]
|
||
missing[f] = len(D) - len(known)
|
||
groups = group_by(known, f)
|
||
nf, ef = len(known), sum(1 for r in known if ev(r))
|
||
for lv in sorted(groups):
|
||
g = groups[lv]
|
||
k, n = sum(1 for r in g if ev(r)), len(g)
|
||
row = {"factor": f, "level": lv, **W(k, n)}
|
||
rk, rn = ef - k, nf - n
|
||
if n >= min_n and rn >= min_n and 0 < ef < nf:
|
||
row.update(odds(k, n - k, rk, rn - rk))
|
||
rows.append(row)
|
||
tested = [r for r in rows if "fisher_p" in r]
|
||
for r, q in zip(tested, bh([r["fisher_p"] for r in tested])):
|
||
r["q"] = q
|
||
return {"outcome": key, "label": O["label"], "overall": W(sum(1 for r in D if ev(r)), len(D)), "rows": rows, "missing": missing}
|
||
|
||
|
||
def fit_model(runs, key, factors, lam=1.0):
|
||
import numpy as np
|
||
O = OUTCOMES[key]
|
||
D = []
|
||
for r in runs:
|
||
if not match(r, O["denom"]):
|
||
continue
|
||
o = {"y": 1 if match(r, O["event"]) else 0, **{name: fn(r) for name, fn in factors.items()}}
|
||
if all(o[f] is not None for f in factors):
|
||
D.append(o)
|
||
if len(D) < 20:
|
||
return None
|
||
cols, names, refs = [np.ones(len(D))], [], {}
|
||
for f in factors:
|
||
counts = group_by(D, f)
|
||
levels = sorted(counts, key=lambda lv: (-len(counts[lv]), lv))
|
||
refs[f] = levels[0]
|
||
for lv in levels[1:]:
|
||
cols.append(np.array([1.0 if _key(o[f]) == lv else 0.0 for o in D]))
|
||
names.append((f, lv, len(counts[lv])))
|
||
y = np.array([o["y"] for o in D])
|
||
b, se = logistic_ridge(np.column_stack(cols), y, lam)
|
||
terms = [{"term": f"{f} = {lv}", "factor": f, "level": lv, "n": n, "or": math.exp(b[i + 1]),
|
||
"lo": math.exp(b[i + 1] - Z95 * se[i + 1]), "hi": math.exp(b[i + 1] + Z95 * se[i + 1]),
|
||
"p": float(p_from_z(b[i + 1] / se[i + 1]))} for i, (f, lv, n) in enumerate(names)]
|
||
return {"outcome": key, "label": O["label"], "n": len(D), "events": int(y.sum()), "refs": refs, "lambda": lam, "terms": terms}
|
||
|
||
|
||
def route_key(r):
|
||
return [r["company"], r["direction"], r["first_sighting"], r["scream"], r["resolution"], r.get("post_path") or "—", r.get("hour") or "—"]
|
||
|
||
|
||
def analyse(all_runs, filters):
|
||
runs = apply_filters(all_runs, filters)
|
||
res = {"filters": filters, "n": len(runs)}
|
||
|
||
flag_counts = {}
|
||
for r in runs:
|
||
for f in (r["data_flags"].split("; ") if r["data_flags"] else []):
|
||
k = "first sighting outside documented pool" if f.startswith("first sighting") else f
|
||
flag_counts[k] = flag_counts.get(k, 0) + 1
|
||
dates = sorted(r["date"] for r in runs if r.get("date"))
|
||
outs = group_by(runs, lambda r: r["outcome"] + (" " + r["outcome_color"] if r.get("outcome_color") else ""))
|
||
res["meta"] = {
|
||
"sources": {k: len(v) for k, v in group_by(runs, "entry").items()},
|
||
"date_min": dates[0] if dates else None, "date_max": dates[-1] if dates else None,
|
||
"n_flagged": sum(1 for r in runs if r["data_flags"]), "flag_counts": flag_counts,
|
||
"outcomes": {k: len(v) for k, v in outs.items()},
|
||
}
|
||
|
||
res["chance"] = {}
|
||
for key in ("blue_meadow", "shadow"):
|
||
O = OUTCOMES[key]
|
||
D = [r for r in runs if match(r, O["denom"])]
|
||
ev = lambda r, O=O: match(r, O["event"]) # noqa: E731
|
||
by = lambda k: {s: rate(g, ev) for s, g in sorted(group_by(D, k).items())} # noqa: E731
|
||
form = [r for r in D if r["entry"] == "Form"]
|
||
res["chance"][key] = {"label": O["label"], "overall": rate(D, ev), "by_stratum": by(stratum), "by_goal": by("goal"),
|
||
"by_attempt": by("attempt_group"), "form_overall": rate(form, ev)}
|
||
|
||
fs = group_by([r for r in runs if not is_missing(r["direction"]) and not is_missing(r["company"])], lambda r: r["company"] + "|" + r["direction"])
|
||
res["first_sighting"] = [{"key": k, "n": len(g), "colors": {c: len(v) for c, v in sorted(group_by(g, "first_sighting").items())}}
|
||
for k, g in sorted(fs.items())]
|
||
|
||
res["encounters"] = []
|
||
for e in M["encounters"]:
|
||
col, colour = e["column"], e["colour"]
|
||
A = [r for r in runs if match(r, e["where"]) and r.get(col) in ("Bonded!", "Incinerated")]
|
||
win = lambda r, col=col: r.get(col) == "Bonded!" # noqa: E731
|
||
intent = [r for r in A if r["goal"] == colour or r["goal"].startswith("Any") or (r["goal"] == "Blue or Black" and colour in ("Blue", "Black"))]
|
||
res["encounters"].append({"context": e["context"], "color": colour, "overall": rate(A, win), "intent_matched": rate(intent, win),
|
||
"by_stratum": {s: rate(g, win) for s, g in sorted(group_by(A, stratum).items())}})
|
||
greens_where = next(e["where"] for e in M["encounters"] if e["colour"] == "Green")
|
||
greens = [r for r in runs if match(r, greens_where) and r.get("enc_Green")]
|
||
g = {"green_encounters": len(greens),
|
||
"bonded_green": sum(1 for r in greens if r["enc_Green"] == "Bonded!"),
|
||
"rescued_orange": sum(1 for r in greens if r["rescued"] and r.get("outcome_color") == "Orange"),
|
||
"rescued_brown": sum(1 for r in greens if r["rescued"] and r.get("outcome_color") == "Brown"),
|
||
"incinerated": sum(1 for r in greens if r["outcome"] == "Incinerated")}
|
||
g["no_outcome"] = g["green_encounters"] - g["bonded_green"] - g["rescued_orange"] - g["rescued_brown"] - g["incinerated"]
|
||
g["rates"] = {k: W(g[k], g["green_encounters"]) for k in ("bonded_green", "rescued_orange", "rescued_brown", "incinerated", "no_outcome")}
|
||
res["green_rescue"] = g
|
||
|
||
res["factors"] = {k: factor_table(runs, k) for k in OUTCOMES}
|
||
|
||
first2 = lambda r: r["first_sighting"] if r["first_sighting"] in ("Red", "Brown", "Orange", "Green") else None # noqa: E731
|
||
known = lambda f: (lambda r: None if is_missing(r.get(f)) else r[f]) # noqa: E731
|
||
src2 = lambda r: "Form submission" if r["entry"] == "Form" else "Bulk log (import)" # noqa: E731
|
||
res["models"] = {
|
||
"blue_meadow": fit_model(runs, "blue_meadow", {"source": src2, "goal": goal2, "direction": known("direction"),
|
||
"first_sighting": first2, "resolution": known("resolution"), "company": known("company")}),
|
||
"shadow": fit_model(runs, "shadow", {"source": src2, "goal": goal2, "attempt": known("attempt_group"), "direction": known("direction")}),
|
||
"bond_blue_meadow": fit_model(runs, "bond_blue_meadow", {"source": src2, "goal": goal2, "direction": known("direction"), "resolution": known("resolution")}),
|
||
}
|
||
|
||
res["daily"] = []
|
||
for d, gr in sorted(group_by([r for r in runs if r.get("date")], "date").items()):
|
||
m = [r for r in gr if match(r, OUTCOMES["blue_meadow"]["denom"])]
|
||
t = [r for r in gr if match(r, OUTCOMES["shadow"]["denom"])]
|
||
bk, sk = sum(1 for r in m if r["saw_blue_meadow"]), sum(1 for r in t if r["saw_shadow"])
|
||
res["daily"].append({"date": d, "runs": len(gr), "meadow_n": len(m), "blue_k": bk, "tree_n": len(t), "shadow_k": sk,
|
||
"blue": W(bk, len(m)), "shadow": W(sk, len(t))})
|
||
|
||
res["source_rates"] = {key: {s: rate([r for r in g if match(r, OUTCOMES[key]["denom"])], lambda r, k=key: match(r, OUTCOMES[k]["event"]))
|
||
for s, g in group_by(runs, "entry").items()} for key in ("blue_meadow", "shadow")}
|
||
return res
|
||
|
||
|
||
# ---------------------------------------------------------------- route map and route rankings
|
||
|
||
def stage_value(r, key):
|
||
if key in ("company", "direction", "post_path"):
|
||
return None if is_missing(r.get(key)) else r[key]
|
||
if key == "first_sighting":
|
||
return None if r["scream"] == "DIED at first sighting" or is_missing(r["first_sighting"]) else r["first_sighting"]
|
||
if key == "cadet":
|
||
if r["scream"].startswith("DIED") or is_missing(r["scream"]) or is_missing(r["resolution"]):
|
||
return None
|
||
return "Help → " + r["resolution"] if r["scream"] == "Investigate & help" else r["scream"] + " → " + r["resolution"]
|
||
if key == "hour":
|
||
return "Climb the tree" if r.get("hour") == "Tree" else "Valley centre" if r.get("hour") == "Valley" else None
|
||
if key == "encounter":
|
||
if not r.get("branch"):
|
||
return None
|
||
if r["hour"] == "Tree":
|
||
if r["branch"] == "Shadow (rare)":
|
||
return "Tree: shadow → Blue/Black"
|
||
if r["branch"] == "Descend quickly: Red":
|
||
return "Tree: Red below"
|
||
return "Tree: two Greens"
|
||
if r["branch"] == "Valley Red":
|
||
return "Valley: bond the Red"
|
||
for cond, label in ((r.get("enc_Blue"), "Meadow: Blue"), (r.get("enc_Orange") and r.get("enc_Brown"), "Meadow: Orange/Brown"),
|
||
(r.get("enc_Orange"), "Meadow: Orange"), (r.get("enc_Brown"), "Meadow: Brown")):
|
||
if cond:
|
||
return label
|
||
return "Meadow (no attempt recorded)"
|
||
if key == "end":
|
||
if r["outcome"] == "Bonded":
|
||
return "Bonded " + r["outcome_color"]
|
||
if r["outcome"] in ("Incinerated", "Died before the hour"):
|
||
return r["outcome"]
|
||
return None
|
||
|
||
|
||
def route_map(runs):
|
||
"""The route map: nodes and links with run counts, and for every target the hits and their Wilson interval."""
|
||
nodes, links = {}, {}
|
||
hits = {t: {} for t in TARGETS}
|
||
for r in runs:
|
||
prev = None
|
||
hit = {t: match(r, T["event"]) for t, T in TARGETS.items()}
|
||
for si, s in enumerate(STAGES):
|
||
v = stage_value(r, s["key"])
|
||
if v is None:
|
||
continue
|
||
nid = f"{si}|{v}"
|
||
nd = nodes.setdefault(nid, {"id": nid, "stage": si, "stageKey": s["key"], "value": v, "n": 0})
|
||
nd["n"] += 1
|
||
for t in TARGETS:
|
||
hits[t][nid] = hits[t].get(nid, 0) + hit[t]
|
||
if prev:
|
||
lid = prev + "→" + nid
|
||
ln = links.setdefault(lid, {"id": lid, "source": prev, "target": nid, "n": 0, "chance": bool(s.get("chance"))})
|
||
ln["n"] += 1
|
||
for t in TARGETS:
|
||
hits[t][lid] = hits[t].get(lid, 0) + hit[t]
|
||
prev = nid
|
||
nl, ll = list(nodes.values()), list(links.values())
|
||
by_target = {t: {"nodes": [W(hits[t][n["id"]], n["n"]) for n in nl], "links": [W(hits[t][l["id"]], l["n"]) for l in ll]} for t in TARGETS}
|
||
return {"nodes": nl, "links": ll, "by_target": by_target}
|
||
|
||
|
||
def rank_routes(runs, target, thresholds):
|
||
"""Full routes (every choice up to the hour), ranked for one target. Each route with at least min_n runs is
|
||
compared with all other runs that could reach the target (Fisher), and within data-source / chase-goal strata
|
||
(Mantel-Haenszel). q-values are corrected within each threshold. Ranked by the lower end of the 95% interval."""
|
||
T = TARGETS[target]
|
||
ev = lambda r: match(r, T["event"]) # noqa: E731
|
||
key_stages = [s["key"] for s in STAGES[:6]]
|
||
pool = [r for r in runs if match(r, T["pool"])]
|
||
tot_k, tot_n = sum(1 for r in pool if ev(r)), len(pool)
|
||
full = [r for r in pool if all(stage_value(r, k) or (k == "post_path" and r.get("post_path") is None) for k in key_stages)]
|
||
groups = group_by(full, lambda r: " › ".join(stage_value(r, k) or "—" for k in key_stages))
|
||
strata = list(group_by(pool, stratum).values())
|
||
rows = []
|
||
for key, g in groups.items():
|
||
if len(g) < min(thresholds):
|
||
continue
|
||
k, n = sum(1 for r in g if ev(r)), len(g)
|
||
o = {"route": key, "values": [stage_value(g[0], s) for s in key_stages], **W(k, n)}
|
||
o.update(odds(k, n - k, tot_k - k, tot_n - n - (tot_k - k)))
|
||
ids = {id(r) for r in g}
|
||
tabs = []
|
||
for sg in strata:
|
||
a = sum(1 for r in sg if id(r) in ids and ev(r))
|
||
nin = sum(1 for r in sg if id(r) in ids)
|
||
c = sum(1 for r in sg if id(r) not in ids and ev(r))
|
||
tabs.append([a, nin - a, c, len(sg) - nin - c])
|
||
mh = mantel_haenszel(tabs)
|
||
o.update(dict(zip(("mh_or", "mh_lo", "mh_hi", "mh_p"), mh)) if mh else {"mh_or": None, "mh_lo": None, "mh_hi": None, "mh_p": None})
|
||
rows.append(o)
|
||
rows.sort(key=lambda r: (-r["lo"], -r["n"]))
|
||
out = {"target": target, "overall": W(tot_k, tot_n), "rows": rows, "by_threshold": {}}
|
||
for m in thresholds:
|
||
idx = [i for i, r in enumerate(rows) if r["n"] >= m]
|
||
q = bh([rows[i]["fisher_p"] for i in idx])
|
||
mi = [i for i in idx if rows[i]["mh_p"] is not None]
|
||
mq = bh([rows[i]["mh_p"] for i in mi])
|
||
out["by_threshold"][str(m)] = {"n_routes_tested": len(idx), "q": {str(i): v for i, v in zip(idx, q)},
|
||
"mh_q": {str(i): v for i, v in zip(mi, mq)}}
|
||
return out
|
||
|
||
|
||
# ---------------------------------------------------------------- the valley map and the summary cards
|
||
|
||
def place_ids():
|
||
"""The places drawn on the valley map (the keys of L in site/atlas/valley.js)."""
|
||
src = (ROOT / "site/atlas/valley.js").read_text(encoding="utf-8")
|
||
body = src[src.index("const L = {"):]
|
||
body = body[:body.index("\n };")]
|
||
return set(re.findall(r'^\s{4}"?([^":{}\n]+(?::[^":{}\n]+)*)"?:\s*\{\s*x:', body, re.M))
|
||
|
||
|
||
PLACES = None
|
||
|
||
|
||
def place_path(r):
|
||
"""The places a run passed through, as drawn on the valley map (mirrors path() in valley.js)."""
|
||
L = PLACES
|
||
s = []
|
||
if "comp:" + str(r["company"]) in L:
|
||
s.append("comp:" + r["company"])
|
||
if r["direction"] not in ("East", "West", "Center"):
|
||
return s
|
||
s.append("dir:" + r["direction"])
|
||
sight = f"see:{r['direction']}:{r['first_sighting']}"
|
||
if sight in L:
|
||
s.append(sight)
|
||
if r["scream"] in ("DIED at first sighting", "Unknown"):
|
||
return s
|
||
s.append("scream")
|
||
if r["scream"] == "Investigate & help":
|
||
s.append("bully")
|
||
if "res:" + r["resolution"] in L:
|
||
s.append("res:" + r["resolution"])
|
||
if (r.get("resolution") or "").startswith("DIED"):
|
||
return s
|
||
if r.get("post_path") in ("Forest", "West", "East"):
|
||
s.append("fork:" + r["post_path"])
|
||
elif r["scream"] == "Ignore the scream":
|
||
s.append("wounded")
|
||
if "res:" + r["resolution"] in L:
|
||
s.append("res:" + r["resolution"])
|
||
elif r["scream"] == "Investigate & leave":
|
||
s.append("away")
|
||
if r.get("hour") not in ("Tree", "Valley"):
|
||
return s
|
||
s.append("hour")
|
||
b = r.get("branch")
|
||
if r["hour"] == "Tree":
|
||
s.append("tree")
|
||
if b == "Descend quickly: Red":
|
||
s.append("treeRed")
|
||
elif b == "Descend quickly: let the Red pass":
|
||
s += ["treeRed", "greens"]
|
||
elif b == "Descend slowly: two Greens":
|
||
s.append("greens")
|
||
elif b == "Shadow (rare)":
|
||
s.append("stream")
|
||
if r.get("enc_Blue") or r.get("enc_Black"):
|
||
s.append("redwood")
|
||
if r.get("enc_Black"):
|
||
s.append("black")
|
||
else:
|
||
s.append("valley")
|
||
if b == "Valley Red":
|
||
s.append("valleyRed")
|
||
elif b == "Passed the Red: meadow":
|
||
s.append("meadow")
|
||
oc = r.get("outcome_color")
|
||
tried = "Blue" if r.get("enc_Blue") else oc if oc in ("Orange", "Brown") else "Orange" if r.get("enc_Orange") else "Brown" if r.get("enc_Brown") else None
|
||
if tried:
|
||
s.append("meadow:" + tried)
|
||
return s
|
||
|
||
|
||
def valley(runs):
|
||
counts, edges = {}, {}
|
||
for r in runs:
|
||
seq = place_path(r)
|
||
for i, p in enumerate(seq):
|
||
counts[p] = counts.get(p, 0) + 1
|
||
if i:
|
||
k = seq[i - 1] + ">" + p
|
||
edges[k] = edges.get(k, 0) + 1
|
||
return {"counts": counts, "edges": [[*k.split(">"), n] for k, n in sorted(edges.items(), key=lambda kv: -kv[1])]}
|
||
|
||
|
||
DEATHS = ("Incinerated", "Died before the hour")
|
||
|
||
|
||
def deliberate(r):
|
||
"""A deliberate death: burned at a bonding encounter the rider did not genuinely attempt (chasing another colour,
|
||
choosing to die and try again). The route model's own rule."""
|
||
if r["outcome"] != "Incinerated":
|
||
return False
|
||
try:
|
||
v = model.prepare_run(r)
|
||
return bool(v.get("encCtx")) and not model.genuine_attempt(r, v["encCtx"])
|
||
except (KeyError, TypeError, ValueError): # a source coded from other questions (e.g. the bot): not known
|
||
return False
|
||
|
||
|
||
def path_layers(runs):
|
||
"""'Show the paths of all riders' on the valley map: for every layer in measures.yaml path_layers (one source or
|
||
more), how many runs went from each place to the next (like valley()), split by how the runs ended: `lived` (did
|
||
not die), `chose` (a deliberate death) and `failed` (every other death), plus where the deaths happened. Every
|
||
source is used here, not only the statistics' sources. Sources that record no route are counted only."""
|
||
out = []
|
||
for layer in M["path_layers"]:
|
||
sel = [r for r in runs if r["source"] in layer["sources"]]
|
||
kinds = {"lived": [], "chose": [], "failed": []}
|
||
for r in sel:
|
||
kinds["lived" if r["outcome"] not in DEATHS else "chose" if deliberate(r) else "failed"].append(r)
|
||
ends = {}
|
||
for k in ("chose", "failed"):
|
||
e = {}
|
||
for r in kinds[k]:
|
||
seq = place_path(r)
|
||
if seq:
|
||
e[seq[-1]] = e.get(seq[-1], 0) + 1
|
||
ends[k] = dict(sorted(e.items(), key=lambda kv: -kv[1]))
|
||
out.append({"key": layer["key"], "label": layer["label"], "runs": len(sel),
|
||
"counts": {k: len(v) for k, v in kinds.items()},
|
||
"lines": {k: valley(v)["edges"] for k, v in kinds.items()}, "ends": ends})
|
||
unrouted = {s: sum(1 for r in runs if r["source"] == s) for s in M.get("path_unrouted", [])}
|
||
return {"layers": out, "unrouted": unrouted}
|
||
|
||
|
||
def overview(runs):
|
||
"""The Odds' summary cards: cadets who went in and either bonded or died. Deliberate deaths (burned at an
|
||
encounter the rider did not genuinely attempt) and runs with no recorded outcome are left out."""
|
||
ended = [r for r in runs if r["outcome"] in ("Bonded", "Incinerated", "Died before the hour")]
|
||
cadets = [r for r in ended if not deliberate(r)]
|
||
bonded = sum(1 for r in cadets if r["outcome"] == "Bonded")
|
||
return {"cadets": len(cadets), "bonded": bonded, "dead": len(cadets) - bonded, "deliberate": len(ended) - len(cadets),
|
||
"early_deaths": sum(1 for r in cadets if r["outcome"] == "Died before the hour")}
|
||
|
||
|
||
# ---------------------------------------------------------------- everything
|
||
|
||
def clean(o):
|
||
if isinstance(o, dict):
|
||
return {str(k): clean(v) for k, v in o.items()}
|
||
if isinstance(o, (list, tuple)):
|
||
return [clean(v) for v in o]
|
||
if hasattr(o, "item"):
|
||
o = o.item()
|
||
if isinstance(o, float) and (math.isinf(o) or math.isnan(o)):
|
||
return None
|
||
return o
|
||
|
||
|
||
def build(runs, rows, exported=None, every_run=None):
|
||
global PLACES
|
||
PLACES = place_ids()
|
||
th = M["route_thresholds"]
|
||
out = {
|
||
"schema": 1,
|
||
"data": {"form_rows": rows, "runs": len(runs), "exported": exported},
|
||
"meta": {
|
||
"presets": {k: v["label"] for k, v in M["presets"].items()}, "route_thresholds": th,
|
||
"targets": {k: {"label": t["label"], "color": t["colour"]} for k, t in TARGETS.items()},
|
||
"outcomes": {k: {"label": o["label"]} for k, o in OUTCOMES.items()},
|
||
"factors": M["factors"], "stages": [{"key": s["key"], "label": s["label"], "chance": bool(s.get("chance"))} for s in STAGES],
|
||
"missing_values": sorted(coding.MISSING),
|
||
},
|
||
"presets": {},
|
||
}
|
||
for pid, p in M["presets"].items():
|
||
sub = apply_filters(runs, p["filters"])
|
||
out["presets"][pid] = {
|
||
"label": p["label"], "analyse": analyse(runs, p["filters"]),
|
||
"routes": {t: rank_routes(sub, t, th) for t in TARGETS},
|
||
"map": route_map(sub), "valley": valley(sub), "overview": overview(sub),
|
||
}
|
||
if every_run is not None:
|
||
out["paths"] = path_layers(every_run)
|
||
return clean(out)
|
||
|
||
|
||
def main(argv=None):
|
||
ap = argparse.ArgumentParser(description=__doc__)
|
||
ap.add_argument("--data", default=str(runs_io.PATH))
|
||
ap.add_argument("--out", default=str(ROOT / "data/public/stats.json"))
|
||
a = ap.parse_args(argv)
|
||
runs = runs_io.read(a.data, sources=M["sources"])
|
||
out = build(runs, len(runs), exported(), every_run=runs_io.read(a.data))
|
||
Path(a.out).write_text(json.dumps(out, separators=(",", ":"), ensure_ascii=False), encoding="utf-8")
|
||
print(f"{a.out}: {len(runs)} runs, {len(out['presets'])} presets, {Path(a.out).stat().st_size / 1e6:.1f} MB")
|
||
|
||
|
||
def exported():
|
||
"""When the community spreadsheet last changed (data/manifest.json), else the date in its public file."""
|
||
m = ROOT / "data/manifest.json"
|
||
entry = json.loads(m.read_text(encoding="utf-8")).get("reddit_spreadsheet") if m.exists() else None
|
||
if entry:
|
||
return entry["changed"]
|
||
return json.loads((ROOT / "data/public/form_responses.json").read_text(encoding="utf-8")).get("exported")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|