CI / test (pull_request) Successful in 39s
- analysis/stats/measures.yaml: the bot is a statistics source. A deliberate death (`_deliberate`, the route
model's rule: burned at an encounter the rider was not attempting) is left out of every bond rate, encounter
rate and route ranking, for every source, as the model and the summary cards already did. Biggest moves (all
runs): two Greens 8.9% -> 29.1%, meadow Orange 9.2% -> 25.2%, meadow Brown 14.3% -> 28.3%, valley Red 79.2% ->
93.3%; mostly from form riders who burned on purpose, not from the bot.
- analysis/stats/site_stats.py: runs that are not from the spreadsheet are grouped as their own source
("Reddit comments", "DragonKind bot", "Reported on the Atlas"), not as bulk logs; two new presets (Reddit
comments only, DragonKind bot only); the route map's bond denominators skip deliberate deaths.
- site/atlas/app.js: the bulk-import share counts only spreadsheet imports.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
594 lines
26 KiB
Python
594 lines
26 KiB
Python
"""Every number the Atlas page shows, precomputed: data/public/stats.json.
|
||
|
||
The page only displays these; it computes no statistics itself. What is measured (outcomes, route targets,
|
||
encounters, the route map's stages, the filter presets and route thresholds) is declared in measures.yaml.
|
||
|
||
python analysis/stats/site_stats.py [--out data/public/stats.json]
|
||
"""
|
||
import argparse
|
||
import json
|
||
import math
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
import yaml
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
ROOT = HERE.parents[1]
|
||
sys.path.insert(0, str(ROOT / "core"))
|
||
sys.path.insert(0, str(HERE))
|
||
from atlas import coding, model # noqa: E402
|
||
from atlas import runs as runs_io # noqa: E402
|
||
from stats_core import Z95, bh, logistic_ridge, mantel_haenszel, odds_ratio, p_from_z # noqa: E402
|
||
from stats_core import wilson as _wilson # noqa: E402
|
||
|
||
M = yaml.safe_load(open(HERE / "measures.yaml", encoding="utf-8"))
|
||
COLOURS = M["colours"]
|
||
is_missing = coding.is_missing
|
||
|
||
|
||
# ---------------------------------------------------------------- conditions from measures.yaml
|
||
|
||
def match(r, cond):
|
||
for k, want in cond.items():
|
||
if k == "any":
|
||
if not any(match(r, c) for c in want):
|
||
return False
|
||
continue
|
||
v = r.get(k)
|
||
if isinstance(want, dict):
|
||
if "in" in want and v not in want["in"]:
|
||
return False
|
||
if "not" in want and v == want["not"]:
|
||
return False
|
||
if "exists" in want and (v is not None and v != "") != want["exists"]:
|
||
return False
|
||
elif v != want:
|
||
return False
|
||
return True
|
||
|
||
|
||
def _fill(cond, colour):
|
||
return json.loads(json.dumps(cond).replace("{colour}", colour))
|
||
|
||
|
||
OUTCOMES = {k: v for k, v in M["outcomes"].items()}
|
||
for c in COLOURS:
|
||
b = M["bonded_outcome"]
|
||
OUTCOMES["bonded_" + c] = {"label": b["label"].replace("{colour}", c), "denom": b["denom"], "event": _fill(b["event"], c)}
|
||
TARGETS = M["targets"]
|
||
STAGES = M["stages"]
|
||
|
||
|
||
def W(k, n):
|
||
p, lo, hi = _wilson(k, n)
|
||
return {"k": int(k), "n": int(n), "p": p, "lo": lo, "hi": hi}
|
||
|
||
|
||
def rate(runs, pred):
|
||
return W(sum(1 for r in runs if pred(r)), len(runs))
|
||
|
||
|
||
def group_by(runs, key):
|
||
out = {}
|
||
for r in runs:
|
||
k = key(r) if callable(key) else r.get(key)
|
||
out.setdefault(_key(k), []).append(r)
|
||
return out
|
||
|
||
|
||
def _key(v):
|
||
"""Object keys as JavaScript writes them (true/false/null), so the page can look them up as before."""
|
||
if v is True:
|
||
return "true"
|
||
if v is False:
|
||
return "false"
|
||
if v is None:
|
||
return "null"
|
||
if isinstance(v, float) and v.is_integer():
|
||
return str(int(v))
|
||
return str(v)
|
||
|
||
|
||
def is_rare(goal):
|
||
return goal in M["rare_goals"]
|
||
|
||
|
||
# sources other than the community spreadsheet: their own group wherever runs are split by where they came from
|
||
SOURCE_LABEL = {"reddit comment": "Reddit comments", "bot": "DragonKind bot", "reported": "Reported on the Atlas"}
|
||
|
||
|
||
def entry_group(r):
|
||
"""Where a run came from: the spreadsheet's entry type (Form, Data Import, ...) or the source's own label."""
|
||
return SOURCE_LABEL.get(r["source"]) or r["entry"]
|
||
|
||
|
||
def stratum(r):
|
||
if r["source"] in SOURCE_LABEL:
|
||
return SOURCE_LABEL[r["source"]]
|
||
if r["entry"] != "Form":
|
||
return "Bulk logs (imports)"
|
||
g = r["goal"]
|
||
return {"Blue": "Form: chasing Blue", "Black": "Form: chasing Black", "Blue or Black": "Form: chasing Blue or Black"}.get(g, "Form: other/blank goal")
|
||
|
||
|
||
def goal2(r):
|
||
return r["goal"] if is_rare(r["goal"]) else "Blank/not asked" if r["goal"] == "(not asked/blank)" else "Other colour/any"
|
||
|
||
|
||
def apply_filters(runs, f):
|
||
def ok(r):
|
||
sheet = r["source"] == "reddit spreadsheet"
|
||
if f.get("source") == "form" and not (sheet and r["entry"] == "Form"):
|
||
return False
|
||
if f.get("source") == "bulk" and not (sheet and r["entry"] != "Form"):
|
||
return False
|
||
if f.get("source") == "reddit" and r["source"] != "reddit comment":
|
||
return False
|
||
if f.get("source") == "bot" and r["source"] != "bot":
|
||
return False
|
||
if f.get("goal") == "exclude_rare" and is_rare(r["goal"]):
|
||
return False
|
||
if f.get("goal") == "only_rare" and not is_rare(r["goal"]):
|
||
return False
|
||
if f.get("excludeFlagged") and r["data_flags"]:
|
||
return False
|
||
return True
|
||
return [r for r in runs if ok(r)]
|
||
|
||
|
||
# ---------------------------------------------------------------- tables and models
|
||
|
||
def odds(a, b, c, d):
|
||
or_, lo, hi, p, corr = odds_ratio(a, b, c, d)
|
||
return {"or": or_, "or_lo": lo, "or_hi": hi, "fisher_p": p, "corrected": corr}
|
||
|
||
|
||
def factor_table(runs, key, min_n=5):
|
||
O = OUTCOMES[key]
|
||
D = [r for r in runs if match(r, O["denom"])]
|
||
ev = lambda r: match(r, O["event"]) # noqa: E731
|
||
rows, missing = [], {}
|
||
for f in M["factors"]:
|
||
known = [r for r in D if not is_missing(r.get(f))]
|
||
missing[f] = len(D) - len(known)
|
||
groups = group_by(known, f)
|
||
nf, ef = len(known), sum(1 for r in known if ev(r))
|
||
for lv in sorted(groups):
|
||
g = groups[lv]
|
||
k, n = sum(1 for r in g if ev(r)), len(g)
|
||
row = {"factor": f, "level": lv, **W(k, n)}
|
||
rk, rn = ef - k, nf - n
|
||
if n >= min_n and rn >= min_n and 0 < ef < nf:
|
||
row.update(odds(k, n - k, rk, rn - rk))
|
||
rows.append(row)
|
||
tested = [r for r in rows if "fisher_p" in r]
|
||
for r, q in zip(tested, bh([r["fisher_p"] for r in tested])):
|
||
r["q"] = q
|
||
return {"outcome": key, "label": O["label"], "overall": W(sum(1 for r in D if ev(r)), len(D)), "rows": rows, "missing": missing}
|
||
|
||
|
||
def fit_model(runs, key, factors, lam=1.0):
|
||
import numpy as np
|
||
O = OUTCOMES[key]
|
||
D = []
|
||
for r in runs:
|
||
if not match(r, O["denom"]):
|
||
continue
|
||
o = {"y": 1 if match(r, O["event"]) else 0, **{name: fn(r) for name, fn in factors.items()}}
|
||
if all(o[f] is not None for f in factors):
|
||
D.append(o)
|
||
if len(D) < 20:
|
||
return None
|
||
cols, names, refs = [np.ones(len(D))], [], {}
|
||
for f in factors:
|
||
counts = group_by(D, f)
|
||
levels = sorted(counts, key=lambda lv: (-len(counts[lv]), lv))
|
||
refs[f] = levels[0]
|
||
for lv in levels[1:]:
|
||
cols.append(np.array([1.0 if _key(o[f]) == lv else 0.0 for o in D]))
|
||
names.append((f, lv, len(counts[lv])))
|
||
y = np.array([o["y"] for o in D])
|
||
b, se = logistic_ridge(np.column_stack(cols), y, lam)
|
||
terms = [{"term": f"{f} = {lv}", "factor": f, "level": lv, "n": n, "or": math.exp(b[i + 1]),
|
||
"lo": math.exp(b[i + 1] - Z95 * se[i + 1]), "hi": math.exp(b[i + 1] + Z95 * se[i + 1]),
|
||
"p": float(p_from_z(b[i + 1] / se[i + 1]))} for i, (f, lv, n) in enumerate(names)]
|
||
return {"outcome": key, "label": O["label"], "n": len(D), "events": int(y.sum()), "refs": refs, "lambda": lam, "terms": terms}
|
||
|
||
|
||
def route_key(r):
|
||
return [r["company"], r["direction"], r["first_sighting"], r["scream"], r["resolution"], r.get("post_path") or "—", r.get("hour") or "—"]
|
||
|
||
|
||
def analyse(all_runs, filters):
|
||
runs = apply_filters(all_runs, filters)
|
||
res = {"filters": filters, "n": len(runs)}
|
||
|
||
flag_counts = {}
|
||
for r in runs:
|
||
for f in (r["data_flags"].split("; ") if r["data_flags"] else []):
|
||
k = "first sighting outside documented pool" if f.startswith("first sighting") else f
|
||
flag_counts[k] = flag_counts.get(k, 0) + 1
|
||
dates = sorted(r["date"] for r in runs if r.get("date"))
|
||
outs = group_by(runs, lambda r: r["outcome"] + (" " + r["outcome_color"] if r.get("outcome_color") else ""))
|
||
res["meta"] = {
|
||
"sources": {k: len(v) for k, v in group_by(runs, entry_group).items()},
|
||
"date_min": dates[0] if dates else None, "date_max": dates[-1] if dates else None,
|
||
"n_flagged": sum(1 for r in runs if r["data_flags"]), "flag_counts": flag_counts,
|
||
"outcomes": {k: len(v) for k, v in outs.items()},
|
||
}
|
||
|
||
res["chance"] = {}
|
||
for key in ("blue_meadow", "shadow"):
|
||
O = OUTCOMES[key]
|
||
D = [r for r in runs if match(r, O["denom"])]
|
||
ev = lambda r, O=O: match(r, O["event"]) # noqa: E731
|
||
by = lambda k: {s: rate(g, ev) for s, g in sorted(group_by(D, k).items())} # noqa: E731
|
||
form = [r for r in D if r["entry"] == "Form"]
|
||
res["chance"][key] = {"label": O["label"], "overall": rate(D, ev), "by_stratum": by(stratum), "by_goal": by("goal"),
|
||
"by_attempt": by("attempt_group"), "form_overall": rate(form, ev)}
|
||
|
||
fs = group_by([r for r in runs if not is_missing(r["direction"]) and not is_missing(r["company"])], lambda r: r["company"] + "|" + r["direction"])
|
||
res["first_sighting"] = [{"key": k, "n": len(g), "colors": {c: len(v) for c, v in sorted(group_by(g, "first_sighting").items())}}
|
||
for k, g in sorted(fs.items())]
|
||
|
||
res["encounters"] = []
|
||
for e in M["encounters"]:
|
||
col, colour = e["column"], e["colour"]
|
||
# a rider who chose to burn here (chasing another colour) did not fail this encounter
|
||
A = [r for r in runs if match(r, e["where"]) and r.get(col) in ("Bonded!", "Incinerated") and not r.get("_deliberate")]
|
||
win = lambda r, col=col: r.get(col) == "Bonded!" # noqa: E731
|
||
intent = [r for r in A if r["goal"] == colour or r["goal"].startswith("Any") or (r["goal"] == "Blue or Black" and colour in ("Blue", "Black"))]
|
||
res["encounters"].append({"context": e["context"], "color": colour, "overall": rate(A, win), "intent_matched": rate(intent, win),
|
||
"by_stratum": {s: rate(g, win) for s, g in sorted(group_by(A, stratum).items())}})
|
||
greens_where = next(e["where"] for e in M["encounters"] if e["colour"] == "Green")
|
||
greens = [r for r in runs if match(r, greens_where) and r.get("enc_Green") and not r.get("_deliberate")]
|
||
g = {"green_encounters": len(greens),
|
||
"bonded_green": sum(1 for r in greens if r["enc_Green"] == "Bonded!"),
|
||
"rescued_orange": sum(1 for r in greens if r["rescued"] and r.get("outcome_color") == "Orange"),
|
||
"rescued_brown": sum(1 for r in greens if r["rescued"] and r.get("outcome_color") == "Brown"),
|
||
"incinerated": sum(1 for r in greens if r["outcome"] == "Incinerated")}
|
||
g["no_outcome"] = g["green_encounters"] - g["bonded_green"] - g["rescued_orange"] - g["rescued_brown"] - g["incinerated"]
|
||
g["rates"] = {k: W(g[k], g["green_encounters"]) for k in ("bonded_green", "rescued_orange", "rescued_brown", "incinerated", "no_outcome")}
|
||
res["green_rescue"] = g
|
||
|
||
res["factors"] = {k: factor_table(runs, k) for k in OUTCOMES}
|
||
|
||
first2 = lambda r: r["first_sighting"] if r["first_sighting"] in ("Red", "Brown", "Orange", "Green") else None # noqa: E731
|
||
known = lambda f: (lambda r: None if is_missing(r.get(f)) else r[f]) # noqa: E731
|
||
src2 = lambda r: SOURCE_LABEL.get(r["source"]) or ("Form submission" if r["entry"] == "Form" else "Bulk log (import)") # noqa: E731
|
||
res["models"] = {
|
||
"blue_meadow": fit_model(runs, "blue_meadow", {"source": src2, "goal": goal2, "direction": known("direction"),
|
||
"first_sighting": first2, "resolution": known("resolution"), "company": known("company")}),
|
||
"shadow": fit_model(runs, "shadow", {"source": src2, "goal": goal2, "attempt": known("attempt_group"), "direction": known("direction")}),
|
||
"bond_blue_meadow": fit_model(runs, "bond_blue_meadow", {"source": src2, "goal": goal2, "direction": known("direction"), "resolution": known("resolution")}),
|
||
}
|
||
|
||
res["daily"] = []
|
||
for d, gr in sorted(group_by([r for r in runs if r.get("date")], "date").items()):
|
||
m = [r for r in gr if match(r, OUTCOMES["blue_meadow"]["denom"])]
|
||
t = [r for r in gr if match(r, OUTCOMES["shadow"]["denom"])]
|
||
bk, sk = sum(1 for r in m if r["saw_blue_meadow"]), sum(1 for r in t if r["saw_shadow"])
|
||
res["daily"].append({"date": d, "runs": len(gr), "meadow_n": len(m), "blue_k": bk, "tree_n": len(t), "shadow_k": sk,
|
||
"blue": W(bk, len(m)), "shadow": W(sk, len(t))})
|
||
|
||
res["source_rates"] = {key: {s: rate([r for r in g if match(r, OUTCOMES[key]["denom"])], lambda r, k=key: match(r, OUTCOMES[k]["event"]))
|
||
for s, g in group_by(runs, entry_group).items()} for key in ("blue_meadow", "shadow")}
|
||
return res
|
||
|
||
|
||
# ---------------------------------------------------------------- route map and route rankings
|
||
|
||
def stage_value(r, key):
|
||
if key in ("company", "direction", "post_path"):
|
||
return None if is_missing(r.get(key)) else r[key]
|
||
if key == "first_sighting":
|
||
return None if r["scream"] == "DIED at first sighting" or is_missing(r["first_sighting"]) else r["first_sighting"]
|
||
if key == "cadet":
|
||
if r["scream"].startswith("DIED") or is_missing(r["scream"]) or is_missing(r["resolution"]):
|
||
return None
|
||
return "Help → " + r["resolution"] if r["scream"] == "Investigate & help" else r["scream"] + " → " + r["resolution"]
|
||
if key == "hour":
|
||
return "Climb the tree" if r.get("hour") == "Tree" else "Valley centre" if r.get("hour") == "Valley" else None
|
||
if key == "encounter":
|
||
if not r.get("branch"):
|
||
return None
|
||
if r["hour"] == "Tree":
|
||
if r["branch"] == "Shadow (rare)":
|
||
return "Tree: shadow → Blue/Black"
|
||
if r["branch"] == "Descend quickly: Red":
|
||
return "Tree: Red below"
|
||
return "Tree: two Greens"
|
||
if r["branch"] == "Valley Red":
|
||
return "Valley: bond the Red"
|
||
for cond, label in ((r.get("enc_Blue"), "Meadow: Blue"), (r.get("enc_Orange") and r.get("enc_Brown"), "Meadow: Orange/Brown"),
|
||
(r.get("enc_Orange"), "Meadow: Orange"), (r.get("enc_Brown"), "Meadow: Brown")):
|
||
if cond:
|
||
return label
|
||
return "Meadow (no attempt recorded)"
|
||
if key == "end":
|
||
if r["outcome"] == "Bonded":
|
||
return "Bonded " + r["outcome_color"]
|
||
if r["outcome"] in ("Incinerated", "Died before the hour"):
|
||
return r["outcome"]
|
||
return None
|
||
|
||
|
||
def route_map(runs):
|
||
"""The route map: nodes and links with run counts, and for every target the hits and their Wilson interval."""
|
||
nodes, links = {}, {}
|
||
hits = {t: {} for t in TARGETS}
|
||
tries = {t: {} for t in TARGETS} # runs that count for the target (no deliberate deaths for a bond)
|
||
for r in runs:
|
||
prev = None
|
||
hit = {t: match(r, T["event"]) for t, T in TARGETS.items()}
|
||
counts = {t: not (r.get("_deliberate") and t.startswith("bond_")) for t in TARGETS}
|
||
for si, s in enumerate(STAGES):
|
||
v = stage_value(r, s["key"])
|
||
if v is None:
|
||
continue
|
||
nid = f"{si}|{v}"
|
||
nd = nodes.setdefault(nid, {"id": nid, "stage": si, "stageKey": s["key"], "value": v, "n": 0})
|
||
nd["n"] += 1
|
||
for t in TARGETS:
|
||
hits[t][nid] = hits[t].get(nid, 0) + hit[t]
|
||
tries[t][nid] = tries[t].get(nid, 0) + counts[t]
|
||
if prev:
|
||
lid = prev + "→" + nid
|
||
ln = links.setdefault(lid, {"id": lid, "source": prev, "target": nid, "n": 0, "chance": bool(s.get("chance"))})
|
||
ln["n"] += 1
|
||
for t in TARGETS:
|
||
hits[t][lid] = hits[t].get(lid, 0) + hit[t]
|
||
tries[t][lid] = tries[t].get(lid, 0) + counts[t]
|
||
prev = nid
|
||
nl, ll = list(nodes.values()), list(links.values())
|
||
by_target = {t: {"nodes": [W(hits[t][n["id"]], tries[t][n["id"]]) for n in nl],
|
||
"links": [W(hits[t][l["id"]], tries[t][l["id"]]) for l in ll]} for t in TARGETS}
|
||
return {"nodes": nl, "links": ll, "by_target": by_target}
|
||
|
||
|
||
def rank_routes(runs, target, thresholds):
|
||
"""Full routes (every choice up to the hour), ranked for one target. Each route with at least min_n runs is
|
||
compared with all other runs that could reach the target (Fisher), and within data-source / chase-goal strata
|
||
(Mantel-Haenszel). q-values are corrected within each threshold. Ranked by the lower end of the 95% interval."""
|
||
T = TARGETS[target]
|
||
ev = lambda r: match(r, T["event"]) # noqa: E731
|
||
key_stages = [s["key"] for s in STAGES[:6]]
|
||
pool = [r for r in runs if match(r, T["pool"])]
|
||
tot_k, tot_n = sum(1 for r in pool if ev(r)), len(pool)
|
||
full = [r for r in pool if all(stage_value(r, k) or (k == "post_path" and r.get("post_path") is None) for k in key_stages)]
|
||
groups = group_by(full, lambda r: " › ".join(stage_value(r, k) or "—" for k in key_stages))
|
||
strata = list(group_by(pool, stratum).values())
|
||
rows = []
|
||
for key, g in groups.items():
|
||
if len(g) < min(thresholds):
|
||
continue
|
||
k, n = sum(1 for r in g if ev(r)), len(g)
|
||
o = {"route": key, "values": [stage_value(g[0], s) for s in key_stages], **W(k, n)}
|
||
o.update(odds(k, n - k, tot_k - k, tot_n - n - (tot_k - k)))
|
||
ids = {id(r) for r in g}
|
||
tabs = []
|
||
for sg in strata:
|
||
a = sum(1 for r in sg if id(r) in ids and ev(r))
|
||
nin = sum(1 for r in sg if id(r) in ids)
|
||
c = sum(1 for r in sg if id(r) not in ids and ev(r))
|
||
tabs.append([a, nin - a, c, len(sg) - nin - c])
|
||
mh = mantel_haenszel(tabs)
|
||
o.update(dict(zip(("mh_or", "mh_lo", "mh_hi", "mh_p"), mh)) if mh else {"mh_or": None, "mh_lo": None, "mh_hi": None, "mh_p": None})
|
||
rows.append(o)
|
||
rows.sort(key=lambda r: (-r["lo"], -r["n"]))
|
||
out = {"target": target, "overall": W(tot_k, tot_n), "rows": rows, "by_threshold": {}}
|
||
for m in thresholds:
|
||
idx = [i for i, r in enumerate(rows) if r["n"] >= m]
|
||
q = bh([rows[i]["fisher_p"] for i in idx])
|
||
mi = [i for i in idx if rows[i]["mh_p"] is not None]
|
||
mq = bh([rows[i]["mh_p"] for i in mi])
|
||
out["by_threshold"][str(m)] = {"n_routes_tested": len(idx), "q": {str(i): v for i, v in zip(idx, q)},
|
||
"mh_q": {str(i): v for i, v in zip(mi, mq)}}
|
||
return out
|
||
|
||
|
||
# ---------------------------------------------------------------- the valley map and the summary cards
|
||
|
||
def place_ids():
|
||
"""The places drawn on the valley map (the keys of L in site/atlas/valley.js)."""
|
||
src = (ROOT / "site/atlas/valley.js").read_text(encoding="utf-8")
|
||
body = src[src.index("const L = {"):]
|
||
body = body[:body.index("\n };")]
|
||
return set(re.findall(r'^\s{4}"?([^":{}\n]+(?::[^":{}\n]+)*)"?:\s*\{\s*x:', body, re.M))
|
||
|
||
|
||
PLACES = None
|
||
|
||
|
||
def place_path(r):
|
||
"""The places a run passed through, as drawn on the valley map (mirrors path() in valley.js)."""
|
||
L = PLACES
|
||
s = []
|
||
if "comp:" + str(r["company"]) in L:
|
||
s.append("comp:" + r["company"])
|
||
if r["direction"] not in ("East", "West", "Center"):
|
||
return s
|
||
s.append("dir:" + r["direction"])
|
||
sight = f"see:{r['direction']}:{r['first_sighting']}"
|
||
if sight in L:
|
||
s.append(sight)
|
||
if r["scream"] in ("DIED at first sighting", "Unknown"):
|
||
return s
|
||
s.append("scream")
|
||
if r["scream"] == "Investigate & help":
|
||
s.append("bully")
|
||
if "res:" + r["resolution"] in L:
|
||
s.append("res:" + r["resolution"])
|
||
if (r.get("resolution") or "").startswith("DIED"):
|
||
return s
|
||
if r.get("post_path") in ("Forest", "West", "East"):
|
||
s.append("fork:" + r["post_path"])
|
||
elif r["scream"] == "Ignore the scream":
|
||
s.append("wounded")
|
||
if "res:" + r["resolution"] in L:
|
||
s.append("res:" + r["resolution"])
|
||
elif r["scream"] == "Investigate & leave":
|
||
s.append("away")
|
||
if r.get("hour") not in ("Tree", "Valley"):
|
||
return s
|
||
s.append("hour")
|
||
b = r.get("branch")
|
||
if r["hour"] == "Tree":
|
||
s.append("tree")
|
||
if b == "Descend quickly: Red":
|
||
s.append("treeRed")
|
||
elif b == "Descend quickly: let the Red pass":
|
||
s += ["treeRed", "greens"]
|
||
elif b == "Descend slowly: two Greens":
|
||
s.append("greens")
|
||
elif b == "Shadow (rare)":
|
||
s.append("stream")
|
||
if r.get("enc_Blue") or r.get("enc_Black"):
|
||
s.append("redwood")
|
||
if r.get("enc_Black"):
|
||
s.append("black")
|
||
else:
|
||
s.append("valley")
|
||
if b == "Valley Red":
|
||
s.append("valleyRed")
|
||
elif b == "Passed the Red: meadow":
|
||
s.append("meadow")
|
||
oc = r.get("outcome_color")
|
||
tried = "Blue" if r.get("enc_Blue") else oc if oc in ("Orange", "Brown") else "Orange" if r.get("enc_Orange") else "Brown" if r.get("enc_Brown") else None
|
||
if tried:
|
||
s.append("meadow:" + tried)
|
||
return s
|
||
|
||
|
||
def valley(runs):
|
||
counts, edges = {}, {}
|
||
for r in runs:
|
||
seq = place_path(r)
|
||
for i, p in enumerate(seq):
|
||
counts[p] = counts.get(p, 0) + 1
|
||
if i:
|
||
k = seq[i - 1] + ">" + p
|
||
edges[k] = edges.get(k, 0) + 1
|
||
return {"counts": counts, "edges": [[*k.split(">"), n] for k, n in sorted(edges.items(), key=lambda kv: -kv[1])]}
|
||
|
||
|
||
DEATHS = ("Incinerated", "Died before the hour")
|
||
|
||
|
||
def deliberate(r):
|
||
"""A deliberate death: burned at a bonding encounter the rider did not genuinely attempt (chasing another colour,
|
||
choosing to die and try again). The route model's own rule."""
|
||
if r["outcome"] != "Incinerated":
|
||
return False
|
||
try:
|
||
v = model.prepare_run(r)
|
||
return bool(v.get("encCtx")) and not model.genuine_attempt(r, v["encCtx"])
|
||
except (KeyError, TypeError, ValueError): # a source coded from other questions (e.g. the bot): not known
|
||
return False
|
||
|
||
|
||
def path_layers(runs):
|
||
"""'Show the paths of all riders' on the valley map: for every layer in measures.yaml path_layers (one source or
|
||
more), how many runs went from each place to the next (like valley()), split by how the runs ended: `lived` (did
|
||
not die), `chose` (a deliberate death) and `failed` (every other death), plus where the deaths happened. Every
|
||
source is used here, not only the statistics' sources. Sources that record no route are counted only."""
|
||
out = []
|
||
for layer in M["path_layers"]:
|
||
sel = [r for r in runs if r["source"] in layer["sources"]]
|
||
kinds = {"lived": [], "chose": [], "failed": []}
|
||
for r in sel:
|
||
kinds["lived" if r["outcome"] not in DEATHS else "chose" if deliberate(r) else "failed"].append(r)
|
||
ends = {}
|
||
for k in ("chose", "failed"):
|
||
e = {}
|
||
for r in kinds[k]:
|
||
seq = place_path(r)
|
||
if seq:
|
||
e[seq[-1]] = e.get(seq[-1], 0) + 1
|
||
ends[k] = dict(sorted(e.items(), key=lambda kv: -kv[1]))
|
||
out.append({"key": layer["key"], "label": layer["label"], "runs": len(sel),
|
||
"counts": {k: len(v) for k, v in kinds.items()},
|
||
"lines": {k: valley(v)["edges"] for k, v in kinds.items()}, "ends": ends})
|
||
unrouted = {s: sum(1 for r in runs if r["source"] == s) for s in M.get("path_unrouted", [])}
|
||
return {"layers": out, "unrouted": unrouted}
|
||
|
||
|
||
def overview(runs):
|
||
"""The Odds' summary cards: cadets who went in and either bonded or died. Deliberate deaths (burned at an
|
||
encounter the rider did not genuinely attempt) and runs with no recorded outcome are left out."""
|
||
ended = [r for r in runs if r["outcome"] in ("Bonded", "Incinerated", "Died before the hour")]
|
||
cadets = [r for r in ended if not deliberate(r)]
|
||
bonded = sum(1 for r in cadets if r["outcome"] == "Bonded")
|
||
return {"cadets": len(cadets), "bonded": bonded, "dead": len(cadets) - bonded, "deliberate": len(ended) - len(cadets),
|
||
"early_deaths": sum(1 for r in cadets if r["outcome"] == "Died before the hour")}
|
||
|
||
|
||
# ---------------------------------------------------------------- everything
|
||
|
||
def clean(o):
|
||
if isinstance(o, dict):
|
||
return {str(k): clean(v) for k, v in o.items()}
|
||
if isinstance(o, (list, tuple)):
|
||
return [clean(v) for v in o]
|
||
if hasattr(o, "item"):
|
||
o = o.item()
|
||
if isinstance(o, float) and (math.isinf(o) or math.isnan(o)):
|
||
return None
|
||
return o
|
||
|
||
|
||
def build(runs, rows, exported=None, every_run=None):
|
||
global PLACES
|
||
PLACES = place_ids()
|
||
th = M["route_thresholds"]
|
||
out = {
|
||
"schema": 1,
|
||
"data": {"form_rows": rows, "runs": len(runs), "exported": exported},
|
||
"meta": {
|
||
"presets": {k: v["label"] for k, v in M["presets"].items()}, "route_thresholds": th,
|
||
"targets": {k: {"label": t["label"], "color": t["colour"]} for k, t in TARGETS.items()},
|
||
"outcomes": {k: {"label": o["label"]} for k, o in OUTCOMES.items()},
|
||
"factors": M["factors"], "stages": [{"key": s["key"], "label": s["label"], "chance": bool(s.get("chance"))} for s in STAGES],
|
||
"missing_values": sorted(coding.MISSING),
|
||
},
|
||
"presets": {},
|
||
}
|
||
for pid, p in M["presets"].items():
|
||
sub = apply_filters(runs, p["filters"])
|
||
out["presets"][pid] = {
|
||
"label": p["label"], "analyse": analyse(runs, p["filters"]),
|
||
"routes": {t: rank_routes(sub, t, th) for t in TARGETS},
|
||
"map": route_map(sub), "valley": valley(sub), "overview": overview(sub),
|
||
}
|
||
if every_run is not None:
|
||
out["paths"] = path_layers(every_run)
|
||
return clean(out)
|
||
|
||
|
||
def main(argv=None):
|
||
ap = argparse.ArgumentParser(description=__doc__)
|
||
ap.add_argument("--data", default=str(runs_io.PATH))
|
||
ap.add_argument("--out", default=str(ROOT / "data/public/stats.json"))
|
||
a = ap.parse_args(argv)
|
||
runs = runs_io.read(a.data, sources=M["sources"])
|
||
for r in runs:
|
||
r["_deliberate"] = deliberate(r)
|
||
out = build(runs, len(runs), exported(), every_run=runs_io.read(a.data))
|
||
Path(a.out).write_text(json.dumps(out, separators=(",", ":"), ensure_ascii=False), encoding="utf-8")
|
||
print(f"{a.out}: {len(runs)} runs, {len(out['presets'])} presets, {Path(a.out).stat().st_size / 1e6:.1f} MB")
|
||
|
||
|
||
def exported():
|
||
"""When the community spreadsheet last changed (data/manifest.json), else the date in its public file."""
|
||
m = ROOT / "data/manifest.json"
|
||
entry = json.loads(m.read_text(encoding="utf-8")).get("reddit_spreadsheet") if m.exists() else None
|
||
if entry:
|
||
return entry["changed"]
|
||
return json.loads((ROOT / "data/public/form_responses.json").read_text(encoding="utf-8")).get("exported")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|