site built from 6ba87691
This commit is contained in:
@@ -0,0 +1,595 @@
|
||||
"""Every number the Atlas page shows, precomputed: data/public/stats.json.
|
||||
|
||||
The page only displays these; it computes no statistics itself. What is measured (outcomes, route targets,
|
||||
encounters, the route map's stages, the filter presets and route thresholds) is declared in measures.yaml.
|
||||
|
||||
python analysis/stats/site_stats.py [--out data/public/stats.json]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parents[1]
|
||||
sys.path.insert(0, str(ROOT / "core"))
|
||||
sys.path.insert(0, str(HERE))
|
||||
from atlas import coding, model # noqa: E402
|
||||
from atlas import runs as runs_io # noqa: E402
|
||||
from stats_core import Z95, bh, logistic_ridge, mantel_haenszel, odds_ratio, p_from_z # noqa: E402
|
||||
from stats_core import wilson as _wilson # noqa: E402
|
||||
|
||||
M = yaml.safe_load(open(HERE / "measures.yaml", encoding="utf-8"))
|
||||
COLOURS = M["colours"]
|
||||
is_missing = coding.is_missing
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- conditions from measures.yaml
|
||||
|
||||
def match(r, cond):
|
||||
for k, want in cond.items():
|
||||
if k == "any":
|
||||
if not any(match(r, c) for c in want):
|
||||
return False
|
||||
continue
|
||||
v = r.get(k)
|
||||
if isinstance(want, dict):
|
||||
if "in" in want and v not in want["in"]:
|
||||
return False
|
||||
if "not" in want and v == want["not"]:
|
||||
return False
|
||||
if "exists" in want and (v is not None and v != "") != want["exists"]:
|
||||
return False
|
||||
elif v != want:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _fill(cond, colour):
|
||||
return json.loads(json.dumps(cond).replace("{colour}", colour))
|
||||
|
||||
|
||||
OUTCOMES = {k: v for k, v in M["outcomes"].items()}
|
||||
for c in COLOURS:
|
||||
b = M["bonded_outcome"]
|
||||
OUTCOMES["bonded_" + c] = {"label": b["label"].replace("{colour}", c), "denom": b["denom"], "event": _fill(b["event"], c)}
|
||||
TARGETS = M["targets"]
|
||||
STAGES = M["stages"]
|
||||
|
||||
|
||||
def W(k, n):
|
||||
p, lo, hi = _wilson(k, n)
|
||||
return {"k": int(k), "n": int(n), "p": p, "lo": lo, "hi": hi}
|
||||
|
||||
|
||||
def rate(runs, pred):
|
||||
return W(sum(1 for r in runs if pred(r)), len(runs))
|
||||
|
||||
|
||||
def group_by(runs, key):
|
||||
out = {}
|
||||
for r in runs:
|
||||
k = key(r) if callable(key) else r.get(key)
|
||||
out.setdefault(_key(k), []).append(r)
|
||||
return out
|
||||
|
||||
|
||||
def _key(v):
|
||||
"""Object keys as JavaScript writes them (true/false/null), so the page can look them up as before."""
|
||||
if v is True:
|
||||
return "true"
|
||||
if v is False:
|
||||
return "false"
|
||||
if v is None:
|
||||
return "null"
|
||||
if isinstance(v, float) and v.is_integer():
|
||||
return str(int(v))
|
||||
return str(v)
|
||||
|
||||
|
||||
def is_rare(goal):
|
||||
return goal in M["rare_goals"]
|
||||
|
||||
|
||||
# sources other than the community spreadsheet: their own group wherever runs are split by where they came from
|
||||
SOURCE_LABEL = {"reddit comment": "Reddit comments", "bot": "DragonKind bot", "reported": "Reported on the Atlas"}
|
||||
|
||||
|
||||
def entry_group(r):
|
||||
"""Where a run came from: the spreadsheet's entry type (Form, Data Import, ...) or the source's own label."""
|
||||
return SOURCE_LABEL.get(r["source"]) or r["entry"]
|
||||
|
||||
|
||||
def stratum(r):
|
||||
if r["source"] in SOURCE_LABEL:
|
||||
return SOURCE_LABEL[r["source"]]
|
||||
if r["entry"] != "Form":
|
||||
return "Bulk logs (imports)"
|
||||
g = r["goal"]
|
||||
return {"Blue": "Form: chasing Blue", "Black": "Form: chasing Black", "Blue or Black": "Form: chasing Blue or Black"}.get(g, "Form: other/blank goal")
|
||||
|
||||
|
||||
def goal2(r):
|
||||
return r["goal"] if is_rare(r["goal"]) else "Blank/not asked" if r["goal"] == "(not asked/blank)" else "Other colour/any"
|
||||
|
||||
|
||||
def apply_filters(runs, f):
|
||||
def ok(r):
|
||||
sheet = r["source"] == "reddit spreadsheet"
|
||||
if f.get("source") == "form" and not (sheet and r["entry"] == "Form"):
|
||||
return False
|
||||
if f.get("source") == "bulk" and not (sheet and r["entry"] != "Form"):
|
||||
return False
|
||||
if f.get("source") == "reddit" and r["source"] != "reddit comment":
|
||||
return False
|
||||
if f.get("source") == "bot" and r["source"] != "bot":
|
||||
return False
|
||||
if f.get("goal") == "exclude_rare" and is_rare(r["goal"]):
|
||||
return False
|
||||
if f.get("goal") == "only_rare" and not is_rare(r["goal"]):
|
||||
return False
|
||||
if f.get("excludeFlagged") and r["data_flags"]:
|
||||
return False
|
||||
return True
|
||||
return [r for r in runs if ok(r)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- tables and models
|
||||
|
||||
def odds(a, b, c, d):
|
||||
or_, lo, hi, p, corr = odds_ratio(a, b, c, d)
|
||||
return {"or": or_, "or_lo": lo, "or_hi": hi, "fisher_p": p, "corrected": corr}
|
||||
|
||||
|
||||
def factor_table(runs, key, min_n=5):
|
||||
O = OUTCOMES[key]
|
||||
D = [r for r in runs if match(r, O["denom"])]
|
||||
ev = lambda r: match(r, O["event"]) # noqa: E731
|
||||
rows, missing = [], {}
|
||||
for f in M["factors"]:
|
||||
known = [r for r in D if not is_missing(r.get(f))]
|
||||
missing[f] = len(D) - len(known)
|
||||
groups = group_by(known, f)
|
||||
nf, ef = len(known), sum(1 for r in known if ev(r))
|
||||
for lv in sorted(groups):
|
||||
g = groups[lv]
|
||||
k, n = sum(1 for r in g if ev(r)), len(g)
|
||||
row = {"factor": f, "level": lv, **W(k, n)}
|
||||
rk, rn = ef - k, nf - n
|
||||
if n >= min_n and rn >= min_n and 0 < ef < nf:
|
||||
row.update(odds(k, n - k, rk, rn - rk))
|
||||
rows.append(row)
|
||||
tested = [r for r in rows if "fisher_p" in r]
|
||||
for r, q in zip(tested, bh([r["fisher_p"] for r in tested])):
|
||||
r["q"] = q
|
||||
return {"outcome": key, "label": O["label"], "overall": W(sum(1 for r in D if ev(r)), len(D)), "rows": rows, "missing": missing}
|
||||
|
||||
|
||||
def fit_model(runs, key, factors, lam=1.0):
|
||||
import numpy as np
|
||||
O = OUTCOMES[key]
|
||||
D = []
|
||||
for r in runs:
|
||||
if not match(r, O["denom"]):
|
||||
continue
|
||||
o = {"y": 1 if match(r, O["event"]) else 0, **{name: fn(r) for name, fn in factors.items()}}
|
||||
if all(o[f] is not None for f in factors):
|
||||
D.append(o)
|
||||
if len(D) < 20:
|
||||
return None
|
||||
cols, names, refs = [np.ones(len(D))], [], {}
|
||||
for f in factors:
|
||||
counts = group_by(D, f)
|
||||
levels = sorted(counts, key=lambda lv: (-len(counts[lv]), lv))
|
||||
refs[f] = levels[0]
|
||||
for lv in levels[1:]:
|
||||
cols.append(np.array([1.0 if _key(o[f]) == lv else 0.0 for o in D]))
|
||||
names.append((f, lv, len(counts[lv])))
|
||||
y = np.array([o["y"] for o in D])
|
||||
b, se = logistic_ridge(np.column_stack(cols), y, lam)
|
||||
terms = [{"term": f"{f} = {lv}", "factor": f, "level": lv, "n": n, "or": math.exp(b[i + 1]),
|
||||
"lo": math.exp(b[i + 1] - Z95 * se[i + 1]), "hi": math.exp(b[i + 1] + Z95 * se[i + 1]),
|
||||
"p": float(p_from_z(b[i + 1] / se[i + 1]))} for i, (f, lv, n) in enumerate(names)]
|
||||
return {"outcome": key, "label": O["label"], "n": len(D), "events": int(y.sum()), "refs": refs, "lambda": lam, "terms": terms}
|
||||
|
||||
|
||||
def route_key(r):
|
||||
return [r["company"], r["direction"], r["first_sighting"], r["scream"], r["resolution"], r.get("post_path") or "—", r.get("hour") or "—"]
|
||||
|
||||
|
||||
def analyse(all_runs, filters):
|
||||
runs = apply_filters(all_runs, filters)
|
||||
res = {"filters": filters, "n": len(runs)}
|
||||
|
||||
flag_counts = {}
|
||||
for r in runs:
|
||||
for f in (r["data_flags"].split("; ") if r["data_flags"] else []):
|
||||
k = "first sighting outside documented pool" if f.startswith("first sighting") else f
|
||||
flag_counts[k] = flag_counts.get(k, 0) + 1
|
||||
dates = sorted(r["date"] for r in runs if r.get("date"))
|
||||
outs = group_by(runs, lambda r: r["outcome"] + (" " + r["outcome_color"] if r.get("outcome_color") else ""))
|
||||
res["meta"] = {
|
||||
"sources": {k: len(v) for k, v in group_by(runs, entry_group).items()},
|
||||
"date_min": dates[0] if dates else None, "date_max": dates[-1] if dates else None,
|
||||
"n_flagged": sum(1 for r in runs if r["data_flags"]), "flag_counts": flag_counts,
|
||||
"outcomes": {k: len(v) for k, v in outs.items()},
|
||||
}
|
||||
|
||||
res["chance"] = {}
|
||||
for key in ("blue_meadow", "shadow"):
|
||||
O = OUTCOMES[key]
|
||||
D = [r for r in runs if match(r, O["denom"])]
|
||||
ev = lambda r, O=O: match(r, O["event"]) # noqa: E731
|
||||
by = lambda k: {s: rate(g, ev) for s, g in sorted(group_by(D, k).items())} # noqa: E731
|
||||
form = [r for r in D if r["entry"] == "Form"]
|
||||
res["chance"][key] = {"label": O["label"], "overall": rate(D, ev), "by_stratum": by(stratum), "by_goal": by("goal"),
|
||||
"by_attempt": by("attempt_group"), "form_overall": rate(form, ev)}
|
||||
|
||||
fs = group_by([r for r in runs if not is_missing(r["direction"]) and not is_missing(r["company"])], lambda r: r["company"] + "|" + r["direction"])
|
||||
res["first_sighting"] = [{"key": k, "n": len(g), "colors": {c: len(v) for c, v in sorted(group_by(g, "first_sighting").items())}}
|
||||
for k, g in sorted(fs.items())]
|
||||
|
||||
res["encounters"] = []
|
||||
for e in M["encounters"]:
|
||||
col, colour = e["column"], e["colour"]
|
||||
# a rider who chose to burn here (chasing another colour) did not fail this encounter
|
||||
A = [r for r in runs if match(r, e["where"]) and r.get(col) in ("Bonded!", "Incinerated") and not r.get("_deliberate")]
|
||||
win = lambda r, col=col: r.get(col) == "Bonded!" # noqa: E731
|
||||
intent = [r for r in A if r["goal"] == colour or r["goal"].startswith("Any") or (r["goal"] == "Blue or Black" and colour in ("Blue", "Black"))]
|
||||
res["encounters"].append({"context": e["context"], "color": colour, "overall": rate(A, win), "intent_matched": rate(intent, win),
|
||||
"by_stratum": {s: rate(g, win) for s, g in sorted(group_by(A, stratum).items())}})
|
||||
greens_where = next(e["where"] for e in M["encounters"] if e["colour"] == "Green")
|
||||
greens = [r for r in runs if match(r, greens_where) and r.get("enc_Green") and not r.get("_deliberate")]
|
||||
g = {"green_encounters": len(greens),
|
||||
"bonded_green": sum(1 for r in greens if r["enc_Green"] == "Bonded!"),
|
||||
"rescued_orange": sum(1 for r in greens if r["rescued"] and r.get("outcome_color") == "Orange"),
|
||||
"rescued_brown": sum(1 for r in greens if r["rescued"] and r.get("outcome_color") == "Brown"),
|
||||
"incinerated": sum(1 for r in greens if r["outcome"] == "Incinerated")}
|
||||
g["no_outcome"] = g["green_encounters"] - g["bonded_green"] - g["rescued_orange"] - g["rescued_brown"] - g["incinerated"]
|
||||
g["rates"] = {k: W(g[k], g["green_encounters"]) for k in ("bonded_green", "rescued_orange", "rescued_brown", "incinerated", "no_outcome")}
|
||||
res["green_rescue"] = g
|
||||
|
||||
res["factors"] = {k: factor_table(runs, k) for k in OUTCOMES}
|
||||
|
||||
first2 = lambda r: r["first_sighting"] if r["first_sighting"] in ("Red", "Brown", "Orange", "Green") else None # noqa: E731
|
||||
known = lambda f: (lambda r: None if is_missing(r.get(f)) else r[f]) # noqa: E731
|
||||
src2 = lambda r: SOURCE_LABEL.get(r["source"]) or ("Form submission" if r["entry"] == "Form" else "Bulk log (import)") # noqa: E731
|
||||
res["models"] = {
|
||||
"blue_meadow": fit_model(runs, "blue_meadow", {"source": src2, "goal": goal2, "direction": known("direction"),
|
||||
"first_sighting": first2, "resolution": known("resolution"), "company": known("company")}),
|
||||
"shadow": fit_model(runs, "shadow", {"source": src2, "goal": goal2, "attempt": known("attempt_group"), "direction": known("direction")}),
|
||||
"bond_blue_meadow": fit_model(runs, "bond_blue_meadow", {"source": src2, "goal": goal2, "direction": known("direction"), "resolution": known("resolution")}),
|
||||
}
|
||||
|
||||
res["daily"] = []
|
||||
for d, gr in sorted(group_by([r for r in runs if r.get("date")], "date").items()):
|
||||
m = [r for r in gr if match(r, OUTCOMES["blue_meadow"]["denom"])]
|
||||
t = [r for r in gr if match(r, OUTCOMES["shadow"]["denom"])]
|
||||
bk, sk = sum(1 for r in m if r["saw_blue_meadow"]), sum(1 for r in t if r["saw_shadow"])
|
||||
res["daily"].append({"date": d, "runs": len(gr), "meadow_n": len(m), "blue_k": bk, "tree_n": len(t), "shadow_k": sk,
|
||||
"blue": W(bk, len(m)), "shadow": W(sk, len(t))})
|
||||
|
||||
res["source_rates"] = {key: {s: rate([r for r in g if match(r, OUTCOMES[key]["denom"])], lambda r, k=key: match(r, OUTCOMES[k]["event"]))
|
||||
for s, g in group_by(runs, entry_group).items()} for key in ("blue_meadow", "shadow")}
|
||||
return res
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- route map and route rankings
|
||||
|
||||
def stage_value(r, key):
|
||||
if key in ("company", "direction", "post_path"):
|
||||
return None if is_missing(r.get(key)) else r[key]
|
||||
if key == "first_sighting":
|
||||
return None if r["scream"] == "DIED at first sighting" or is_missing(r["first_sighting"]) else r["first_sighting"]
|
||||
if key == "cadet":
|
||||
if r["scream"].startswith("DIED") or is_missing(r["scream"]) or is_missing(r["resolution"]):
|
||||
return None
|
||||
return "Help → " + r["resolution"] if r["scream"] == "Investigate & help" else r["scream"] + " → " + r["resolution"]
|
||||
if key == "hour":
|
||||
return "Climb the tree" if r.get("hour") == "Tree" else "Valley centre" if r.get("hour") == "Valley" else None
|
||||
if key == "encounter":
|
||||
if not r.get("branch"):
|
||||
return None
|
||||
if r["hour"] == "Tree":
|
||||
if r["branch"] == "Shadow (rare)":
|
||||
return "Tree: shadow → Blue/Black"
|
||||
if r["branch"] == "Descend quickly: Red":
|
||||
return "Tree: Red below"
|
||||
return "Tree: two Greens"
|
||||
if r["branch"] == "Valley Red":
|
||||
return "Valley: bond the Red"
|
||||
for cond, label in ((r.get("enc_Blue"), "Meadow: Blue"), (r.get("enc_Orange") and r.get("enc_Brown"), "Meadow: Orange/Brown"),
|
||||
(r.get("enc_Orange"), "Meadow: Orange"), (r.get("enc_Brown"), "Meadow: Brown")):
|
||||
if cond:
|
||||
return label
|
||||
return "Meadow (no attempt recorded)"
|
||||
if key == "end":
|
||||
if r["outcome"] == "Bonded":
|
||||
return "Bonded " + r["outcome_color"]
|
||||
if r["outcome"] in ("Incinerated", "Died before the hour"):
|
||||
return r["outcome"]
|
||||
return None
|
||||
|
||||
|
||||
def route_map(runs):
|
||||
"""The route map: nodes and links with run counts, and for every target the hits and their Wilson interval."""
|
||||
nodes, links = {}, {}
|
||||
hits = {t: {} for t in TARGETS}
|
||||
tries = {t: {} for t in TARGETS} # runs that count for the target (no deliberate deaths for a bond)
|
||||
for r in runs:
|
||||
prev = None
|
||||
hit = {t: match(r, T["event"]) for t, T in TARGETS.items()}
|
||||
counts = {t: not (r.get("_deliberate") and t.startswith("bond_")) for t in TARGETS}
|
||||
for si, s in enumerate(STAGES):
|
||||
v = stage_value(r, s["key"])
|
||||
if v is None:
|
||||
continue
|
||||
nid = f"{si}|{v}"
|
||||
nd = nodes.setdefault(nid, {"id": nid, "stage": si, "stageKey": s["key"], "value": v, "n": 0})
|
||||
nd["n"] += 1
|
||||
for t in TARGETS:
|
||||
hits[t][nid] = hits[t].get(nid, 0) + hit[t]
|
||||
tries[t][nid] = tries[t].get(nid, 0) + counts[t]
|
||||
if prev:
|
||||
lid = prev + "→" + nid
|
||||
ln = links.setdefault(lid, {"id": lid, "source": prev, "target": nid, "n": 0, "chance": bool(s.get("chance"))})
|
||||
ln["n"] += 1
|
||||
for t in TARGETS:
|
||||
hits[t][lid] = hits[t].get(lid, 0) + hit[t]
|
||||
tries[t][lid] = tries[t].get(lid, 0) + counts[t]
|
||||
prev = nid
|
||||
nl, ll = list(nodes.values()), list(links.values())
|
||||
by_target = {t: {"nodes": [W(hits[t][n["id"]], tries[t][n["id"]]) for n in nl],
|
||||
"links": [W(hits[t][l["id"]], tries[t][l["id"]]) for l in ll]} for t in TARGETS}
|
||||
return {"nodes": nl, "links": ll, "by_target": by_target}
|
||||
|
||||
|
||||
def rank_routes(runs, target, thresholds):
|
||||
"""Full routes (every choice up to the hour), ranked for one target. Each route with at least min_n runs is
|
||||
compared with all other runs that could reach the target (Fisher), and within data-source / chase-goal strata
|
||||
(Mantel-Haenszel). q-values are corrected within each threshold. Ranked by the lower end of the 95% interval."""
|
||||
T = TARGETS[target]
|
||||
ev = lambda r: match(r, T["event"]) # noqa: E731
|
||||
key_stages = [s["key"] for s in STAGES[:6]]
|
||||
pool = [r for r in runs if match(r, T["pool"])]
|
||||
tot_k, tot_n = sum(1 for r in pool if ev(r)), len(pool)
|
||||
full = [r for r in pool if all(stage_value(r, k) or (k == "post_path" and r.get("post_path") is None) for k in key_stages)]
|
||||
groups = group_by(full, lambda r: " › ".join(stage_value(r, k) or "—" for k in key_stages))
|
||||
strata = list(group_by(pool, stratum).values())
|
||||
rows = []
|
||||
for key, g in groups.items():
|
||||
if len(g) < min(thresholds):
|
||||
continue
|
||||
k, n = sum(1 for r in g if ev(r)), len(g)
|
||||
o = {"route": key, "values": [stage_value(g[0], s) for s in key_stages], **W(k, n)}
|
||||
o.update(odds(k, n - k, tot_k - k, tot_n - n - (tot_k - k)))
|
||||
ids = {id(r) for r in g}
|
||||
tabs = []
|
||||
for sg in strata:
|
||||
a = sum(1 for r in sg if id(r) in ids and ev(r))
|
||||
nin = sum(1 for r in sg if id(r) in ids)
|
||||
c = sum(1 for r in sg if id(r) not in ids and ev(r))
|
||||
tabs.append([a, nin - a, c, len(sg) - nin - c])
|
||||
mh = mantel_haenszel(tabs)
|
||||
o.update(dict(zip(("mh_or", "mh_lo", "mh_hi", "mh_p"), mh)) if mh else {"mh_or": None, "mh_lo": None, "mh_hi": None, "mh_p": None})
|
||||
rows.append(o)
|
||||
rows.sort(key=lambda r: (-r["lo"], -r["n"]))
|
||||
out = {"target": target, "overall": W(tot_k, tot_n), "rows": rows, "by_threshold": {}}
|
||||
for m in thresholds:
|
||||
idx = [i for i, r in enumerate(rows) if r["n"] >= m]
|
||||
q = bh([rows[i]["fisher_p"] for i in idx])
|
||||
mi = [i for i in idx if rows[i]["mh_p"] is not None]
|
||||
mq = bh([rows[i]["mh_p"] for i in mi])
|
||||
out["by_threshold"][str(m)] = {"n_routes_tested": len(idx), "q": {str(i): v for i, v in zip(idx, q)},
|
||||
"mh_q": {str(i): v for i, v in zip(mi, mq)}}
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the valley map and the summary cards
|
||||
|
||||
def place_ids():
|
||||
"""The places drawn on the valley map (the keys of L in site/atlas/valley.js)."""
|
||||
src = (ROOT / "site/atlas/valley.js").read_text(encoding="utf-8")
|
||||
body = src[src.index("const L = {"):]
|
||||
body = body[:body.index("\n };")]
|
||||
return set(re.findall(r'^\s{4}"?([^":{}\n]+(?::[^":{}\n]+)*)"?:\s*\{\s*x:', body, re.M))
|
||||
|
||||
|
||||
PLACES = None
|
||||
|
||||
|
||||
def place_path(r):
|
||||
"""The places a run passed through, as drawn on the valley map (mirrors path() in valley.js)."""
|
||||
L = PLACES
|
||||
s = []
|
||||
if "comp:" + str(r["company"]) in L:
|
||||
s.append("comp:" + r["company"])
|
||||
if r["direction"] not in ("East", "West", "Center"):
|
||||
return s
|
||||
s.append("dir:" + r["direction"])
|
||||
sight = f"see:{r['direction']}:{r['first_sighting']}"
|
||||
if sight in L:
|
||||
s.append(sight)
|
||||
if r["scream"] in ("DIED at first sighting", "Unknown"):
|
||||
return s
|
||||
s.append("scream")
|
||||
if r["scream"] == "Investigate & help":
|
||||
s.append("bully")
|
||||
if "res:" + r["resolution"] in L:
|
||||
s.append("res:" + r["resolution"])
|
||||
if (r.get("resolution") or "").startswith("DIED"):
|
||||
return s
|
||||
if r.get("post_path") in ("Forest", "West", "East"):
|
||||
s.append("fork:" + r["post_path"])
|
||||
elif r["scream"] == "Ignore the scream":
|
||||
s.append("wounded")
|
||||
if "res:" + r["resolution"] in L:
|
||||
s.append("res:" + r["resolution"])
|
||||
elif r["scream"] == "Investigate & leave":
|
||||
s.append("away")
|
||||
if r.get("hour") not in ("Tree", "Valley"):
|
||||
return s
|
||||
s.append("hour")
|
||||
b = r.get("branch")
|
||||
if r["hour"] == "Tree":
|
||||
s.append("tree")
|
||||
if b == "Descend quickly: Red":
|
||||
s.append("treeRed")
|
||||
elif b == "Descend quickly: let the Red pass":
|
||||
s += ["treeRed", "greens"]
|
||||
elif b == "Descend slowly: two Greens":
|
||||
s.append("greens")
|
||||
elif b == "Shadow (rare)":
|
||||
s.append("stream")
|
||||
if r.get("enc_Blue") or r.get("enc_Black"):
|
||||
s.append("redwood")
|
||||
if r.get("enc_Black"):
|
||||
s.append("black")
|
||||
else:
|
||||
s.append("valley")
|
||||
if b == "Valley Red":
|
||||
s.append("valleyRed")
|
||||
elif b == "Passed the Red: meadow":
|
||||
s.append("meadow")
|
||||
oc = r.get("outcome_color")
|
||||
tried = "Blue" if r.get("enc_Blue") else oc if oc in ("Orange", "Brown") else "Orange" if r.get("enc_Orange") else "Brown" if r.get("enc_Brown") else None
|
||||
if tried:
|
||||
s.append("meadow:" + tried)
|
||||
return s
|
||||
|
||||
|
||||
def valley(runs):
|
||||
counts, edges = {}, {}
|
||||
for r in runs:
|
||||
seq = place_path(r)
|
||||
for i, p in enumerate(seq):
|
||||
counts[p] = counts.get(p, 0) + 1
|
||||
if i:
|
||||
k = seq[i - 1] + ">" + p
|
||||
edges[k] = edges.get(k, 0) + 1
|
||||
return {"counts": counts, "edges": [[*k.split(">"), n] for k, n in sorted(edges.items(), key=lambda kv: -kv[1])]}
|
||||
|
||||
|
||||
DEATHS = ("Incinerated", "Died before the hour")
|
||||
|
||||
|
||||
def deliberate(r):
|
||||
"""A deliberate death: burned at a bonding encounter the rider did not genuinely attempt (chasing another colour,
|
||||
choosing to die and try again). The route model's own rule."""
|
||||
if r["outcome"] != "Incinerated":
|
||||
return False
|
||||
try:
|
||||
v = model.prepare_run(r)
|
||||
return bool(v.get("encCtx")) and not model.genuine_attempt(r, v["encCtx"])
|
||||
except (KeyError, TypeError, ValueError): # a source coded from other questions (e.g. the bot): not known
|
||||
return False
|
||||
|
||||
|
||||
def path_layers(runs):
|
||||
"""'Show the paths of all riders' on the valley map: for every layer in measures.yaml path_layers (one source or
|
||||
more), how many runs went from each place to the next (like valley()), split by how the runs ended: `lived` (did
|
||||
not die), `chose` (a deliberate death) and `failed` (every other death), plus where the deaths happened. Every
|
||||
source is used here, not only the statistics' sources. Sources that record no route are counted only."""
|
||||
out = []
|
||||
for layer in M["path_layers"]:
|
||||
sel = [r for r in runs if r["source"] in layer["sources"]]
|
||||
kinds = {"lived": [], "chose": [], "failed": []}
|
||||
for r in sel:
|
||||
kinds["lived" if r["outcome"] not in DEATHS else "chose" if deliberate(r) else "failed"].append(r)
|
||||
ends = {}
|
||||
for k in ("chose", "failed"):
|
||||
e = {}
|
||||
for r in kinds[k]:
|
||||
seq = place_path(r)
|
||||
if seq:
|
||||
e[seq[-1]] = e.get(seq[-1], 0) + 1
|
||||
ends[k] = dict(sorted(e.items(), key=lambda kv: -kv[1]))
|
||||
out.append({"key": layer["key"], "label": layer["label"], "runs": len(sel),
|
||||
"counts": {k: len(v) for k, v in kinds.items()},
|
||||
"lines": {k: valley(v)["edges"] for k, v in kinds.items()}, "ends": ends})
|
||||
unrouted = {s: sum(1 for r in runs if r["source"] == s) for s in M.get("path_unrouted", [])}
|
||||
return {"layers": out, "unrouted": unrouted}
|
||||
|
||||
|
||||
def overview(runs):
|
||||
"""The Odds' summary cards: cadets who went in and either bonded or died. Deliberate deaths (burned at an
|
||||
encounter the rider did not genuinely attempt) and runs with no recorded outcome are left out."""
|
||||
ended = [r for r in runs if r["outcome"] in ("Bonded", "Incinerated", "Died before the hour")]
|
||||
cadets = [r for r in ended if not deliberate(r)]
|
||||
bonded = sum(1 for r in cadets if r["outcome"] == "Bonded")
|
||||
return {"cadets": len(cadets), "bonded": bonded, "dead": len(cadets) - bonded, "deliberate": len(ended) - len(cadets),
|
||||
"early_deaths": sum(1 for r in cadets if r["outcome"] == "Died before the hour")}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- everything
|
||||
|
||||
def clean(o):
|
||||
if isinstance(o, dict):
|
||||
return {str(k): clean(v) for k, v in o.items()}
|
||||
if isinstance(o, (list, tuple)):
|
||||
return [clean(v) for v in o]
|
||||
if hasattr(o, "item"):
|
||||
o = o.item()
|
||||
if isinstance(o, float) and (math.isinf(o) or math.isnan(o)):
|
||||
return None
|
||||
return o
|
||||
|
||||
|
||||
def build(runs, rows, exported=None, every_run=None):
|
||||
global PLACES
|
||||
PLACES = place_ids()
|
||||
th = M["route_thresholds"]
|
||||
out = {
|
||||
"schema": 1,
|
||||
"data": {"form_rows": rows, "runs": len(runs), "exported": exported},
|
||||
"meta": {
|
||||
"presets": {k: v["label"] for k, v in M["presets"].items()}, "route_thresholds": th,
|
||||
"targets": {k: {"label": t["label"], "color": t["colour"]} for k, t in TARGETS.items()},
|
||||
"outcomes": {k: {"label": o["label"]} for k, o in OUTCOMES.items()},
|
||||
"factors": M["factors"], "stages": [{"key": s["key"], "label": s["label"], "chance": bool(s.get("chance"))} for s in STAGES],
|
||||
"missing_values": sorted(coding.MISSING),
|
||||
},
|
||||
"presets": {},
|
||||
}
|
||||
for pid, p in M["presets"].items():
|
||||
sub = apply_filters(runs, p["filters"])
|
||||
out["presets"][pid] = {
|
||||
"label": p["label"], "analyse": analyse(runs, p["filters"]),
|
||||
"routes": {t: rank_routes(sub, t, th) for t in TARGETS},
|
||||
"map": route_map(sub), "valley": valley(sub), "overview": overview(sub),
|
||||
}
|
||||
if every_run is not None:
|
||||
out["paths"] = path_layers(every_run)
|
||||
return clean(out)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--data", default=str(runs_io.PATH))
|
||||
ap.add_argument("--out", default=str(ROOT / "data/public/stats.json"))
|
||||
a = ap.parse_args(argv)
|
||||
# a run that cannot have happened as recorded (core/spec/form.yaml impossible_flags) is left out, as in the model
|
||||
runs = [r for r in runs_io.read(a.data, sources=M["sources"]) if not runs_io.impossible(r)]
|
||||
for r in runs:
|
||||
r["_deliberate"] = deliberate(r)
|
||||
every = [r for r in runs_io.read(a.data) if not runs_io.impossible(r)]
|
||||
out = build(runs, len(runs), exported(), every_run=every)
|
||||
Path(a.out).write_text(json.dumps(out, separators=(",", ":"), ensure_ascii=False), encoding="utf-8")
|
||||
print(f"{a.out}: {len(runs)} runs, {len(out['presets'])} presets, {Path(a.out).stat().st_size / 1e6:.1f} MB")
|
||||
|
||||
|
||||
def exported():
|
||||
"""When the community spreadsheet last changed (data/manifest.json), else the date in its public file."""
|
||||
m = ROOT / "data/manifest.json"
|
||||
entry = json.loads(m.read_text(encoding="utf-8")).get("reddit_spreadsheet") if m.exists() else None
|
||||
if entry:
|
||||
return entry["changed"]
|
||||
return json.loads((ROOT / "data/public/form_responses.json").read_text(encoding="utf-8")).get("exported")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user