site built from 87217037

This commit is contained in:
Atlas data bot
2026-11-21 10:32:21 +00:00
commit 489f6e03b4
29 changed files with 22582 additions and 0 deletions
+323
View File
@@ -0,0 +1,323 @@
"""Community form rows -> one coded run per row.
The tables (how each question is found, what every answer means, which answers count as missing) live in
core/spec/form.yaml. This module holds the logic that combines answers into a run: which way the rider went,
which branch they took after the hour, how the run ended, and which rows look inconsistent (data_flags).
python -m atlas.coding [form_responses.json|.xlsx|.csv] [out.csv]
Prints a summary of the coded export (default: data/public/form_responses.json), and writes the coded runs to
out.csv when given. The published runs (every source) are written by `python -m collection build`.
"""
import json
import re
import sys
from datetime import datetime
from pathlib import Path
import pandas as pd
from . import spec
ROOT = Path(__file__).resolve().parents[2]
F = spec.form()
A = F["answers"]
COLORS = F["colours"]
MISSING = frozenset(F["missing_values"])
ENC = F["encounter_answers"]
_EMAIL = re.compile(F["redact"]["pattern"])
def is_missing(v):
"""True for answers that carry no information about the choice (blank, "Unsure", not asked...)."""
return v is None or v == "" or v in MISSING or (isinstance(v, float) and v != v)
def redact(v):
"""Replace email addresses in free text."""
return v if v is None else _EMAIL.sub(F["redact"]["replacement"], str(v))
# ---------------------------------------------------------------- reading cells
def find_columns(cols):
"""Map each question key in form.yaml to the export's column header. Returns (columns, missing keys)."""
out = {}
low = {c: str(c).lower() for c in cols}
for key, col in F["columns"].items():
words = col["match"]
hits = [c for c in cols if all((w[1:] not in low[c]) if w[0] == "!" else (w in low[c]) for w in words)]
if hits:
out[key] = hits[0]
missing = [k for k, col in F["columns"].items()
if k not in out and not col.get("optional") and col.get("replaced_by") not in out]
return out, missing
def blank(v):
if v is None or v is pd.NaT:
return True
if not isinstance(v, str):
try:
if pd.isna(v):
return True
except (TypeError, ValueError):
pass
return str(v).strip() in ("", "nan", "NaN", "NaT")
def text(v):
return None if blank(v) else str(v).strip()
def parse_timestamp(v):
"""Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'.
Returns 'YYYY-MM-DD' or None."""
if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v):
return pd.Timestamp(v).strftime("%Y-%m-%d")
if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date
return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d")
t = text(v)
if not t:
return None
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t)
if m:
return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY
if m:
return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
return None
def parse_attempt(v):
v = text(v)
if v is None:
return None
t = v.lower()
if "lost count" in t:
return None
for w, n in F["attempt_words"].items():
if t.startswith(w):
return n
m = re.search(r"\d+", t)
return int(m.group()) if m else None
# ---------------------------------------------------------------- one row -> one run
def code_row(r, C):
"""Code one export row. r: mapping header -> cell; C: from find_columns()."""
g = lambda k: text(r[C[k]]) if k in C else None # noqa: E731
flags = []
ts = r[C["timestamp"]] if "timestamp" in C else None
d = parse_timestamp(ts)
source, date = ("Form", d) if d else ((text(ts) or "Unknown import"), None)
wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)]
wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown")
section = g("wing_" + wings[0]) if len(wings) == 1 else None
if len(wings) > 1:
flags.append("multiple wings")
if "first_dir" in C:
# Redesigned form: one direction question and one colour question. The older per-direction colour
# columns are still in the sheet for some rows and are used only to fill gaps and to cross-check.
direction = A["direction"].get(g("first_dir"))
first = g("first_colour")
legacy = [d for d in ("West", "Center") if g("leg_" + d)]
if not direction and len(legacy) == 1:
direction = legacy[0]
flags.append("direction taken from the older sighting column")
leg = g("leg_" + direction) if direction else None
if not first and leg:
first = leg
elif first and leg and first != leg and first != "Unsure" and leg != "Unsure":
flags.append("first sighting differs between the old and new columns")
if not direction:
direction = "Unknown"
flags.append("direction missing")
else:
dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)]
if len(dirs) == 1:
direction, first = dirs[0], g("dir_" + dirs[0])
else:
direction, first = ("Multiple" if dirs else "Unknown"), None
flags.append("direction " + ("multiple" if dirs else "missing"))
company_raw = g("company") or "Unknown"
suffix = F["company_left_suffix"]
left_squad = company_raw.endswith(suffix)
company = company_raw[: -len(suffix)] if left_squad else company_raw
if not left_squad and company in ("1 Squadmate", "2 Squadmates") and not (date and date >= F["left_squadmates_asked_from"]):
left_squad = None # the form did not offer "left them" yet: unknown, not "stayed together"
alias = F.get("first_sighting_alias") or {}
if first in alias:
flags.append(f"first sighting answered as {first!r}")
first = alias[first]
pools = F["sighting_pool"]
pool = list(pools.get(company, {}).get(direction, [])) + (list(pools["Alone"].get(direction, [])) if left_squad else [])
if first and first != "Unsure" and pool and first not in pool:
flags.append(f"first sighting {first} not in documented pool for {company}/{direction}")
cadet_raw = g("cadet")
combat = combat2 = None
if "cadet_actions" in C:
act = g("cadet_actions")
a = A["cadet_actions"].get(act, ["Unknown", None, None, "Unknown"])
if act and act not in A["cadet_actions"]:
flags.append("unrecognised cadet answer")
scream, combat, combat2 = a[0], a[1], a[2]
out = A["cadet_outcome"].get(cadet_raw)
if a[3] is not None:
resolution = a[3]
if out == "DIED in combat" and not resolution.startswith("DIED"):
flags.append("died in combat on a path without combat")
else:
resolution = out or "Unknown"
else:
scream, resolution = A["cadet_legacy"].get(cadet_raw, ["Unknown", "Unknown"])
if cadet_raw and cadet_raw not in A["cadet_legacy"]:
flags.append("unrecognised cadet answer")
combat = A["combat_legacy"].get(g("combat")) if g("combat") else None
post_raw = g("post")
if scream == "Investigate & help" and not resolution.startswith("DIED"):
post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown"
elif scream in ("Ignore the scream", "Investigate & leave"):
post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch
else:
post = None
hour = A["hour"].get(g("hour"))
died_early = scream.startswith("DIED") or resolution.startswith("DIED")
if died_early:
hour = None
elif hour is None:
hour = "Unknown"
enc = {c: g("enc_" + c) for c in COLORS}
rare = A["rare"].get(g("rare"))
bonded = [c for c in COLORS if enc[c] == ENC["bonded"]]
burned = [c for c in COLORS if enc[c] == ENC["burned"]]
# Which encounter branch did the run take after the hour?
branch = None
if hour == "Tree":
if enc["Blue"] or enc["Black"] or rare == "Shadow":
branch = "Shadow (rare)"
elif enc["Red"] == ENC["passed"] and enc["Green"]:
# only climbing down quickly puts you above the Red: passing on it and meeting the Greens means the
# rider let it pass
branch = "Descend quickly: let the Red pass"
elif enc["Green"]:
branch = "Descend slowly: two Greens"
if enc["Red"]:
flags.append("tree run with both the Red and the two Greens attempted")
elif enc["Red"]:
branch = "Descend quickly: Red"
elif enc["Orange"] or enc["Brown"]:
branch = "Descend slowly: two Greens"
flags.append("tree run with Orange/Brown but no Green recorded")
elif hour == "Valley":
if enc["Green"]:
flags.append("Green recorded on a Valley run (no Greens in the valley)")
meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]] or rare == "Blue (valley)"
if meadow or enc["Red"] == ENC["passed"]:
branch = "Passed the Red: meadow"
elif enc["Red"]:
branch = "Valley Red"
# How the run ended
outcome, outcome_color = None, None
if died_early:
outcome = "Died before the hour"
elif len(bonded) == 1:
outcome, outcome_color = "Bonded", bonded[0]
elif len(bonded) > 1:
outcome = "Ambiguous"
flags.append("more than one dragon marked Bonded")
elif burned:
order = F["burn_order"]["valley" if hour == "Valley" else "tree"]
outcome, outcome_color = "Incinerated", [c for c in order if c in burned][-1]
if len(burned) > 1:
flags.append("more than one dragon marked Incinerated")
elif hour == "DIED before the hour":
outcome = "Died before the hour"
else:
outcome = "No outcome recorded"
rescued = enc["Green"] == ENC["rescued"] and outcome == "Bonded" and outcome_color in ("Orange", "Brown")
attempt = parse_attempt(g("attempt"))
if (rare == "Blue (valley)" and hour == "Tree") or (rare == "Shadow" and hour == "Valley"):
flags.append("rare-dragon answer does not match the path")
goal_raw = g("goal_v2")
goal = g("goal") or (A["goal_v2"].get(goal_raw, goal_raw) if goal_raw and goal_raw != "Unknown" else None) or "(not asked/blank)"
return {
"source": source, "date": date, "wing": wing, "section": section,
"squad": A["squad_any_case"].get((g("squad") or "").strip().lower(), g("squad") or "Unknown"), "attempt": attempt,
# "I've lost count" means many attempts, so it is a later attempt, not missing
"attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if (attempt and attempt > 1) or "lost count" in (g("attempt") or "").lower() else "Unknown"),
"goal": goal, "company": company, "left_squadmates": left_squad,
"direction": direction, "first_sighting": first or "Unknown",
"sighting_action": (A["sighting_action"].get(g("sighting_action")) or g("sighting_action") or "Unknown") if "sighting_action" in C else "(not asked)",
"scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)",
"combat_second": combat2 or "(not recorded)",
"post_path": post, "hour": hour, "branch": branch,
**{"enc_" + c: enc[c] for c in COLORS},
"outcome": outcome, "outcome_color": outcome_color, "rescued": rescued,
"saw_blue_meadow": bool(hour == "Valley" and (enc["Blue"] or rare == "Blue (valley)")),
"saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"] or rare == "Shadow")),
"rare_reported": (rare or "Unknown") if "rare" in C else "(not asked)",
"malek_intent": (g("malek") or "Unknown") if "malek" in C else "(not asked)",
"tail": g("tail"),
"data_flags": "; ".join(flags),
"death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")),
}
# ---------------------------------------------------------------- files
def read_export(path):
"""An export as a DataFrame: the bundled JSON (data/public/form_responses.json), a CSV or the .xlsx."""
p = str(path).lower()
if p.endswith(".json"):
with open(path, encoding="utf-8") as f:
d = json.load(f)
return pd.DataFrame(d["rows"], columns=d["headers"])
if p.endswith(".csv"):
return pd.read_csv(path)
xl = pd.ExcelFile(path)
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
return pd.read_excel(xl, sheet)
def code_frame(raw):
"""Code every row of an export. Returns (runs DataFrame with run_id, missing question keys)."""
C, missing = find_columns(raw.columns)
df = pd.DataFrame([code_row(r, C) for _, r in raw.iterrows()])
df.insert(0, "run_id", range(1, len(df) + 1))
return df, missing
def load(path):
return code_frame(read_export(path))
def main(argv=None):
argv = sys.argv[1:] if argv is None else argv
src = argv[0] if argv else ROOT / "data/public/form_responses.json"
out = argv[1] if len(argv) > 1 else None
df, missing = load(src)
if missing:
print("WARNING: columns not found:", missing)
# a whole-number column with gaps would otherwise be written as 1.0, 2.0, ... (data/dictionary.yaml: integer)
df["attempt"] = df["attempt"].astype("Int64")
if out:
df.to_csv(out, index=False)
print(f"{len(df)} runs" + (f" -> {out}" if out else ""))
print(df.outcome.value_counts().to_string())
print("rows with flags:", (df["data_flags"] != "").sum())
if __name__ == "__main__":
main()
+193
View File
@@ -0,0 +1,193 @@
# The community "Threshing Routes" form: how each question is found in an export and how every answer is coded.
#
# This file is the single source of truth for the coding rules' tables. core/atlas/coding.py reads it; the logic
# that combines answers (which branch a run took, how it ended) lives there and is described on the page's
# Methods tab. To support a new or renamed answer, add it here and add a case to tests/core/test_coding.py.
version: 1
colours: [Red, Green, Orange, Brown, Blue, Black]
# Answers that carry no information about the choice. They are kept in the coded data for transparency, but every
# analysis treats them as missing: never shown or compared as if they were a choice.
missing_values: ["Unknown", "Unsure", "Unsure/unknown", "Multiple", "(not recorded)", "(not asked)",
"(not asked/blank)", "Unknown/lost count", "Blank/not asked"]
# Free-text answers sometimes contain an email address; it is replaced before anything is displayed or exported.
redact:
pattern: '[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}'
replacement: "[email removed]"
# Columns are found by words in the question text (case-insensitive), so the coding keeps working when the form
# owner reorders, renames slightly or adds questions. A word starting with "!" must NOT appear in the header.
columns:
timestamp: { match: ["timestamp"] }
wing_First: { match: ["wing and section", "[first]"] }
wing_Second: { match: ["wing and section", "[second]"] }
wing_Third: { match: ["wing and section", "[third]"] }
wing_Fourth: { match: ["wing and section", "[fourth]"] }
squad: { match: ["what squad"] }
attempt: { match: ["attempt number"] }
company: { match: ["who accompanied"] }
dir_East: { match: ["initial direction", "[east"], replaced_by: first_dir }
dir_West: { match: ["initial direction", "[west"], replaced_by: first_dir }
dir_Center: { match: ["initial direction", "[forest center"], replaced_by: first_dir }
cadet: { match: ["outcome of your cadet encounter"] }
post: { match: ["path did you take after the bully"] }
hour: { match: ["after walking an hour"] }
enc_Red: { match: ["attempt to bond", "[red"] }
enc_Green: { match: ["attempt to bond", "[green"] }
enc_Orange: { match: ["attempt to bond", "[orange"] }
enc_Brown: { match: ["attempt to bond", "[brown"] }
enc_Blue: { match: ["attempt to bond", "[blue"] }
enc_Black: { match: ["attempt to bond", "[black"] }
death_note: { match: ["those who died"] }
bond_note: { match: ["those who bonded"] }
goal: { match: ["colour dragon are you trying"], replaced_by: goal_v2 }
combat: { match: ["combat choice"], replaced_by: cadet_actions }
# Questions added in the redesigned form (late October 2026)
first_dir: { match: ["what was your first direction"], replaced_by: dir_Center }
first_colour: { match: ["colour was your initial dragon sighting", "!["], replaced_by: dir_Center }
leg_West: { match: ["colour was your initial dragon sighting", "[west"], optional: true }
leg_Center: { match: ["colour was your initial dragon sighting", "[forest center"], optional: true }
sighting_action: { match: ["action did you take during the first dragon sighting"], optional: true }
cadet_actions: { match: ["during your cadet encounter"], replaced_by: combat }
goal_v2: { match: ["why are you doing this run"], replaced_by: goal }
malek: { match: ["seek malek"], optional: true }
rare: { match: ["encounter a rare dragon"], optional: true }
tail: { match: ["what tail does your dragon have"], optional: true }
# Answer → coded value, one table per question.
answers:
# Older form: one answer for the whole cadet encounter → [scream, resolution]
cadet_legacy:
"Bullies > You dealt with them": ["Investigate & help", "Killed the bullies yourself"]
"Bullies > Squadmate's revenge": ["Investigate & help", "Squadmate's revenge"]
"Bullies > Left to the Dragons": ["Investigate & help", "Left bullies to the dragons"]
"Bullies > Assessed the situation": ["Investigate & help", "Unknown"]
"Injured Cadet > Tourniquet": ["Ignore the scream", "Tourniquet"]
"Injured Cadet > Left them there": ["Ignore the scream", "Left injured cadet"]
"Injured Cadet > Took their weapons": ["Ignore the scream", "Took their weapons"]
"Avoided combat (Investigated scream > Left them to it)": ["Investigate & leave", "Left them to it"]
"You died in combat": ["Investigate & help", "DIED in combat"]
"N/A - You died with initial dragon sighting": ["DIED at first sighting", "DIED at first sighting"]
# Older form: the first move in the fight
combat_legacy:
"Bullies > Assessed the situation": "Assess the situation"
"Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)"
"Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)": "Put yourself in the middle (bigger cadet)"
# Redesigned form: the encounter as a path of actions → [scream, first move, second move, resolution or null]
# (null resolution: the separate outcome question decides it, see cadet_outcome)
cadet_actions:
"Bullies > Assess > Unknown": ["Investigate & help", "Assess the situation", null, null]
"Bullies > Assess > Smaller Cadet": ["Investigate & help", "Assess the situation", "Engage the smaller cadet", null]
"Bullies > Assess > Daggers or not": ["Investigate & help", "Assess the situation", "Daggers or not (protect squadmate)", null]
"Bullies > Unknown": ["Investigate & help", null, null, null]
"Bullies > Smaller Cadet (Did not assess)": ["Investigate & help", "Even the numbers (smaller cadet)", null, null]
"Bullies > Bigger Cadet (Did not assess)": ["Investigate & help", "Put yourself in the middle (bigger cadet)", null, null]
"Injured Cadet > Tourniquet": ["Ignore the scream", null, null, "Tourniquet"]
"Injured Cadet > Left them there": ["Ignore the scream", null, null, "Left injured cadet"]
"Injured Cadet > Took their weapons": ["Ignore the scream", null, null, "Took their weapons"]
"Avoided combat (Investigated scream > Left them to it)": ["Investigate & leave", null, null, "Left them to it"]
"N/A - You died with initial dragon sighting": ["DIED at first sighting", null, null, "DIED at first sighting"]
"Unsure": ["Unknown", null, null, "Unknown"]
cadet_outcome:
"Squadmate's revenge": "Squadmate's revenge"
"You ended the bullies yourself": "Killed the bullies yourself"
"Left to the Dragons": "Left bullies to the dragons"
"You died in combat": "DIED in combat"
direction:
"Forest Center": Center
"West / Scent": West
"East / Roar": East
goal_v2:
"Hoping for a Rare Dragon (Blue or Black)": "Blue or Black"
"Trying to bond the Red dragon under the tree": "Red (tree)"
"Daily Run Assignment data": "Daily run data"
"Just here for fun, and maybe a dragon": "Any (for fun)"
# Squad: matched ignoring case and spaces (riders type it); an answer not listed still stops the build
squad_any_case:
"first": "First"
"second": "Second"
"third": "Third"
"don't know": "Don't know"
sighting_action:
"Hold Ground / Stay Still": "Hold still"
"Respect / Face it": "Respect / face it"
"Retreat / Run / Back Away": "Retreat / run"
"Hide / Duck": "Hide"
"Protect / Help Squadmate": "Protect squadmate"
"Draw / Drop Weapon": "Weapon"
"Meet Gaze": "Meet its gaze"
"Protect / Help Sqaudmate": "Protect squadmate"
"Approach": "Approach"
"Stand on your own": "Stand alone"
"Unsure": "Unknown"
"Unsure - You don't remember": "Unknown"
rare:
"Yes - A blue dragon in the valley": "Blue (valley)"
"Yes - Shadow!!!": "Shadow"
"No": "No"
hour:
"Continued to Valley Centre": Valley
"Climbed the tallest tree": Tree
"N/A - You're dead": "DIED before the hour"
# Suffix on the company answer that means the rider later left their squadmates. The answer was only offered from
# this date (none of the 504 squad runs before it chose it; 40 of 144 after it did): earlier and undated squad runs
# are coded as unknown (empty), not as "stayed together".
company_left_suffix: " > Left them"
left_squadmates_asked_from: "2026-10-27"
# The encounter answers in the "attempt to bond" grid
encounter_answers: { bonded: "Bonded!", burned: "Incinerated", passed: "Changed my mind", rescued: "Saved by another" }
# When more than one dragon is marked Incinerated, the run ended on the last one met, in this order.
burn_order:
tree: [Red, Green, Orange, Brown, Blue, Black]
valley: [Red, Orange, Brown, Blue, Green, Black]
# First-dragon pools per company and direction, from the community PDF. Only used to FLAG rows that report a
# dragon outside the documented pool. Leaving your squadmates sends you into the solo pools too.
sighting_pool:
Alone: { East: [Red, Orange], West: [Red, Brown, Green], Center: [Red, Brown] }
1 Squadmate: { East: [Green, Orange, Red], West: [Red, Brown, Green], Center: [Red, Brown] }
2 Squadmates: { East: [Red, Orange], West: [Green, Brown, Red], Center: [Red, Brown] }
# The dragons each bonding encounter can end with (bond or burn), from the community PDF. Only used to FLAG runs
# answered through "Report a run" (reported, reddit comment) that end with a dragon the encounter does not have.
# Orange and Brown after the two Greens: a Green's rescue, then the Orange or Brown.
# Flags (data_flags, matched as text inside a flag) that mean a run cannot have happened as recorded: the game's own
# rules, from the form coder (core/atlas/coding.py) and the checks of collection/etl/runs.py rule_flags. A run with one
# is kept in runs_clean.csv, flagged, and left out of the route model and the site statistics (atlas.runs.impossible).
# Everything else, however partial, is used. Not here on purpose: a first sighting outside the documented pool (the
# pool is what riders have seen, not a rule), a missing direction, several wings, a missing Green before a rescue.
impossible_flags:
- "Green recorded on a Valley run"
- "Black recorded on a Valley run"
- "encounter, which has only"
- "more than one dragon marked Bonded"
- "more than one dragon marked Incinerated"
- "tree run with both the Red and the two Greens attempted"
- "died in combat on a path without combat"
- "rare-dragon answer does not match the path"
- "does not follow"
- "fight answers on a run that did not help"
- "a path after the fight on a run without a fight"
- "a second fight move without pausing to assess"
- "died before the hour, yet reached"
encounter_colours:
"Shadow (rare)": [Blue, Black]
"Descend quickly: Red": [Red]
"Descend quickly: let the Red pass": [Green]
"Descend slowly: two Greens": [Green, Orange, Brown]
"Valley Red": [Red]
"Passed the Red: meadow": [Orange, Brown, Blue]
# First-sighting answers that are not a single colour: coded as this colour, and the row is flagged with the
# original answer (data_flags), so nothing is lost.
first_sighting_alias:
"2 Greens": Green
# How a bulk-pasted row's attempt answer is read
attempt_words: { first: 1, second: 2, third: 3, fourth: 4, fifth: 5 }
+108
View File
@@ -0,0 +1,108 @@
# What the site measures: outcomes and their denominators, route targets and their comparison groups, the
# bonding-encounter contexts, the stages of the route map, the filter presets and the minimum-run thresholds.
# analysis/stats/site_stats.py reads this file and writes data/public/stats.json, which the page only displays.
#
# Conditions: {field: value} means equal; {field: {in: [...]}}, {field: {not: value}}, {field: {exists: true}}
# (not blank), {any: [condition, ...]} (at least one). Several fields in one condition must all hold.
version: 1
# Which runs of data/public/runs_clean.csv the statistics use (its `source` column): every source that records the
# route (the community form, "Report a run", the Reddit megathread comments, the DragonKind bot). The Reddit result
# pages and the register record only the end, so they stay out. The bot plays to a strategy and burns on purpose at
# encounters it is not chasing: like every rider's deliberate death (`_deliberate`, the route model's rule), those
# burns are left out of the bond rates, never counted as failed bonds.
sources: [reddit spreadsheet, reported, reddit comment, bot]
colours: [Red, Green, Orange, Brown, Blue, Black]
# Outcomes for the factor tables and models: who could have had the outcome (denom), and the outcome (event).
outcomes:
blue_meadow:
label: Blue appears in the meadow
denom: { branch: "Passed the Red: meadow" }
event: { saw_blue_meadow: true }
shadow:
label: Shadow option appears at the tree
denom: { hour: Tree, branch: { exists: true } }
event: { saw_shadow: true }
bond_blue_meadow:
label: Bond, given you approached the meadow Blue
denom: { branch: "Passed the Red: meadow", enc_Blue: { in: ["Bonded!", "Incinerated"] }, _deliberate: { not: true } }
event: { enc_Blue: "Bonded!" }
reach_hour:
label: Survive to the hour (tree/valley choice)
denom: {}
event: { hour: { in: [Tree, Valley] } }
# one more outcome per colour: "Run ends bonded to <colour>", among runs that reached the hour
bonded_outcome:
label: Run ends bonded to {colour}
denom: { hour: { in: [Tree, Valley] }, _deliberate: { not: true } }
event: { outcome: Bonded, outcome_color: "{colour}" }
# Factors compared in the factor tables (each answer against the other answers to the same question)
factors: [company, direction, first_sighting, scream, resolution, combat_first, post_path, hour, wing, squad,
attempt_group, entry, goal, sighting_action, combat_second, malek_intent]
# Route targets: the outcome (event) and which runs could have reached it at all (pool)
targets:
bond_Blue: { label: Bond Blue, colour: Blue, event: { outcome: Bonded, outcome_color: Blue }, pool: { hour: Valley, _deliberate: { not: true } } }
bond_Black: { label: Bond Black, colour: Black, event: { outcome: Bonded, outcome_color: Black }, pool: { hour: Tree, _deliberate: { not: true } } }
see_Blue: { label: Meet a Blue (meadow), colour: Blue, event: { saw_blue_meadow: true }, pool: { branch: "Passed the Red: meadow" } }
see_shadow: { label: Get the tree shadow, colour: Black, event: { saw_shadow: true }, pool: { hour: Tree, branch: { exists: true } } }
bond_Red: { label: Bond Red, colour: Red, event: { outcome: Bonded, outcome_color: Red }, pool: { hour: { in: [Tree, Valley] }, _deliberate: { not: true } } }
bond_Green: { label: Bond Green, colour: Green, event: { outcome: Bonded, outcome_color: Green }, pool: { hour: Tree, _deliberate: { not: true } } }
bond_Orange: { label: Bond Orange, colour: Orange, event: { outcome: Bonded, outcome_color: Orange }, pool: { hour: { in: [Tree, Valley] }, _deliberate: { not: true } } }
bond_Brown: { label: Bond Brown, colour: Brown, event: { outcome: Bonded, outcome_color: Brown }, pool: { hour: { in: [Tree, Valley] }, _deliberate: { not: true } } }
bond_any: { label: Bond any dragon, colour: null, event: { outcome: Bonded }, pool: { hour: { in: [Tree, Valley] }, _deliberate: { not: true } } }
# Bonding encounters: which recorded attempt (column) belongs to which in-game encounter (where)
encounters:
- { context: "Red (tree, descend quickly)", colour: Red, column: enc_Red, where: { hour: Tree, branch: "Descend quickly: Red" } }
- { context: "Green (two Greens)", colour: Green, column: enc_Green, where: { hour: Tree, any: [ { branch: "Descend slowly: two Greens" }, { branch: "Descend quickly: let the Red pass" } ] } }
- { context: "Blue (shadow)", colour: Blue, column: enc_Blue, where: { hour: Tree, branch: "Shadow (rare)" } }
- { context: "Black (shadow)", colour: Black, column: enc_Black, where: { hour: Tree, branch: "Shadow (rare)" } }
- { context: "Red (valley)", colour: Red, column: enc_Red, where: { hour: Valley, branch: Valley Red } }
- { context: "Orange (meadow)", colour: Orange, column: enc_Orange, where: { hour: Valley, branch: "Passed the Red: meadow" } }
- { context: "Brown (meadow)", colour: Brown, column: enc_Brown, where: { hour: Valley, branch: "Passed the Red: meadow" } }
- { context: "Blue (meadow)", colour: Blue, column: enc_Blue, where: { hour: Valley, branch: "Passed the Red: meadow" } }
# The route map's columns, in order. "chance": steps the game rolls, not the rider.
stages:
- { key: company, label: Company }
- { key: direction, label: Direction }
- { key: first_sighting, label: First sighting, chance: true }
- { key: cadet, label: Cadet encounter }
- { key: post_path, label: Fork in the road }
- { key: hour, label: After an hour }
- { key: encounter, label: Encounter, chance: true }
- { key: end, label: Outcome }
# Filter presets: the page shows one at a time (no statistics are computed in the browser)
presets:
all: { label: All runs, filters: { source: all, goal: all, excludeFlagged: false } }
form: { label: Form submissions only, filters: { source: form, goal: all, excludeFlagged: false } }
bulk: { label: Bulk logs only (imports), filters: { source: bulk, goal: all, excludeFlagged: false } }
reddit: { label: Reddit comments only, filters: { source: reddit, goal: all, excludeFlagged: false } }
bot: { label: DragonKind bot only, filters: { source: bot, goal: all, excludeFlagged: false } }
not_chasing: { label: Not chasing Blue/Black, filters: { source: all, goal: exclude_rare, excludeFlagged: false } }
chasing: { label: Chasing Blue/Black, filters: { source: all, goal: only_rare, excludeFlagged: false } }
unflagged: { label: Rows without data-quality flags, filters: { source: all, goal: all, excludeFlagged: true } }
# Minimum runs a full route needs before it is ranked (q-values are corrected within each threshold)
route_thresholds: [5, 10, 20, 30]
rare_goals: [Blue, Black, Blue or Black]
# "Show the paths of all riders" on the valley map (site/atlas/explorer.js): one layer per source that records a
# route, each with its own switch. Unlike everything above, the layers use every source of runs_clean.csv. Within a
# layer the runs are split by how they ended, drawn as line styles in the layer's colour: solid (did not die), dashed
# (chose to die: burned at an encounter the rider did not genuinely attempt, the route model's rule) and dotted
# (failed: every other death). Discord has no runs yet: its layer is ready for when it is imported.
path_layers:
- { key: sheet, label: "Google Sheet", sources: [reddit spreadsheet] }
- { key: reported, label: "Reported on the Atlas", sources: [reported] }
- { key: reddit, label: "Reddit megathread comments", sources: [reddit comment] }
- { key: bot, label: "DragonKind bot", sources: [bot] }
- { key: discord, label: "Discord", sources: [discord] }
# sources that record which dragon bonded but not the route: counted on the map, not drawn
path_unrouted: [reddit thread, register]
+595
View File
@@ -0,0 +1,595 @@
"""Every number the Atlas page shows, precomputed: data/public/stats.json.
The page only displays these; it computes no statistics itself. What is measured (outcomes, route targets,
encounters, the route map's stages, the filter presets and route thresholds) is declared in measures.yaml.
python analysis/stats/site_stats.py [--out data/public/stats.json]
"""
import argparse
import json
import math
import re
import sys
from pathlib import Path
import yaml
HERE = Path(__file__).resolve().parent
ROOT = HERE.parents[1]
sys.path.insert(0, str(ROOT / "core"))
sys.path.insert(0, str(HERE))
from atlas import coding, model # noqa: E402
from atlas import runs as runs_io # noqa: E402
from stats_core import Z95, bh, logistic_ridge, mantel_haenszel, odds_ratio, p_from_z # noqa: E402
from stats_core import wilson as _wilson # noqa: E402
M = yaml.safe_load(open(HERE / "measures.yaml", encoding="utf-8"))
COLOURS = M["colours"]
is_missing = coding.is_missing
# ---------------------------------------------------------------- conditions from measures.yaml
def match(r, cond):
for k, want in cond.items():
if k == "any":
if not any(match(r, c) for c in want):
return False
continue
v = r.get(k)
if isinstance(want, dict):
if "in" in want and v not in want["in"]:
return False
if "not" in want and v == want["not"]:
return False
if "exists" in want and (v is not None and v != "") != want["exists"]:
return False
elif v != want:
return False
return True
def _fill(cond, colour):
return json.loads(json.dumps(cond).replace("{colour}", colour))
OUTCOMES = {k: v for k, v in M["outcomes"].items()}
for c in COLOURS:
b = M["bonded_outcome"]
OUTCOMES["bonded_" + c] = {"label": b["label"].replace("{colour}", c), "denom": b["denom"], "event": _fill(b["event"], c)}
TARGETS = M["targets"]
STAGES = M["stages"]
def W(k, n):
p, lo, hi = _wilson(k, n)
return {"k": int(k), "n": int(n), "p": p, "lo": lo, "hi": hi}
def rate(runs, pred):
return W(sum(1 for r in runs if pred(r)), len(runs))
def group_by(runs, key):
out = {}
for r in runs:
k = key(r) if callable(key) else r.get(key)
out.setdefault(_key(k), []).append(r)
return out
def _key(v):
"""Object keys as JavaScript writes them (true/false/null), so the page can look them up as before."""
if v is True:
return "true"
if v is False:
return "false"
if v is None:
return "null"
if isinstance(v, float) and v.is_integer():
return str(int(v))
return str(v)
def is_rare(goal):
return goal in M["rare_goals"]
# sources other than the community spreadsheet: their own group wherever runs are split by where they came from
SOURCE_LABEL = {"reddit comment": "Reddit comments", "bot": "DragonKind bot", "reported": "Reported on the Atlas"}
def entry_group(r):
"""Where a run came from: the spreadsheet's entry type (Form, Data Import, ...) or the source's own label."""
return SOURCE_LABEL.get(r["source"]) or r["entry"]
def stratum(r):
if r["source"] in SOURCE_LABEL:
return SOURCE_LABEL[r["source"]]
if r["entry"] != "Form":
return "Bulk logs (imports)"
g = r["goal"]
return {"Blue": "Form: chasing Blue", "Black": "Form: chasing Black", "Blue or Black": "Form: chasing Blue or Black"}.get(g, "Form: other/blank goal")
def goal2(r):
return r["goal"] if is_rare(r["goal"]) else "Blank/not asked" if r["goal"] == "(not asked/blank)" else "Other colour/any"
def apply_filters(runs, f):
def ok(r):
sheet = r["source"] == "reddit spreadsheet"
if f.get("source") == "form" and not (sheet and r["entry"] == "Form"):
return False
if f.get("source") == "bulk" and not (sheet and r["entry"] != "Form"):
return False
if f.get("source") == "reddit" and r["source"] != "reddit comment":
return False
if f.get("source") == "bot" and r["source"] != "bot":
return False
if f.get("goal") == "exclude_rare" and is_rare(r["goal"]):
return False
if f.get("goal") == "only_rare" and not is_rare(r["goal"]):
return False
if f.get("excludeFlagged") and r["data_flags"]:
return False
return True
return [r for r in runs if ok(r)]
# ---------------------------------------------------------------- tables and models
def odds(a, b, c, d):
or_, lo, hi, p, corr = odds_ratio(a, b, c, d)
return {"or": or_, "or_lo": lo, "or_hi": hi, "fisher_p": p, "corrected": corr}
def factor_table(runs, key, min_n=5):
O = OUTCOMES[key]
D = [r for r in runs if match(r, O["denom"])]
ev = lambda r: match(r, O["event"]) # noqa: E731
rows, missing = [], {}
for f in M["factors"]:
known = [r for r in D if not is_missing(r.get(f))]
missing[f] = len(D) - len(known)
groups = group_by(known, f)
nf, ef = len(known), sum(1 for r in known if ev(r))
for lv in sorted(groups):
g = groups[lv]
k, n = sum(1 for r in g if ev(r)), len(g)
row = {"factor": f, "level": lv, **W(k, n)}
rk, rn = ef - k, nf - n
if n >= min_n and rn >= min_n and 0 < ef < nf:
row.update(odds(k, n - k, rk, rn - rk))
rows.append(row)
tested = [r for r in rows if "fisher_p" in r]
for r, q in zip(tested, bh([r["fisher_p"] for r in tested])):
r["q"] = q
return {"outcome": key, "label": O["label"], "overall": W(sum(1 for r in D if ev(r)), len(D)), "rows": rows, "missing": missing}
def fit_model(runs, key, factors, lam=1.0):
import numpy as np
O = OUTCOMES[key]
D = []
for r in runs:
if not match(r, O["denom"]):
continue
o = {"y": 1 if match(r, O["event"]) else 0, **{name: fn(r) for name, fn in factors.items()}}
if all(o[f] is not None for f in factors):
D.append(o)
if len(D) < 20:
return None
cols, names, refs = [np.ones(len(D))], [], {}
for f in factors:
counts = group_by(D, f)
levels = sorted(counts, key=lambda lv: (-len(counts[lv]), lv))
refs[f] = levels[0]
for lv in levels[1:]:
cols.append(np.array([1.0 if _key(o[f]) == lv else 0.0 for o in D]))
names.append((f, lv, len(counts[lv])))
y = np.array([o["y"] for o in D])
b, se = logistic_ridge(np.column_stack(cols), y, lam)
terms = [{"term": f"{f} = {lv}", "factor": f, "level": lv, "n": n, "or": math.exp(b[i + 1]),
"lo": math.exp(b[i + 1] - Z95 * se[i + 1]), "hi": math.exp(b[i + 1] + Z95 * se[i + 1]),
"p": float(p_from_z(b[i + 1] / se[i + 1]))} for i, (f, lv, n) in enumerate(names)]
return {"outcome": key, "label": O["label"], "n": len(D), "events": int(y.sum()), "refs": refs, "lambda": lam, "terms": terms}
def route_key(r):
return [r["company"], r["direction"], r["first_sighting"], r["scream"], r["resolution"], r.get("post_path") or "—", r.get("hour") or "—"]
def analyse(all_runs, filters):
runs = apply_filters(all_runs, filters)
res = {"filters": filters, "n": len(runs)}
flag_counts = {}
for r in runs:
for f in (r["data_flags"].split("; ") if r["data_flags"] else []):
k = "first sighting outside documented pool" if f.startswith("first sighting") else f
flag_counts[k] = flag_counts.get(k, 0) + 1
dates = sorted(r["date"] for r in runs if r.get("date"))
outs = group_by(runs, lambda r: r["outcome"] + (" " + r["outcome_color"] if r.get("outcome_color") else ""))
res["meta"] = {
"sources": {k: len(v) for k, v in group_by(runs, entry_group).items()},
"date_min": dates[0] if dates else None, "date_max": dates[-1] if dates else None,
"n_flagged": sum(1 for r in runs if r["data_flags"]), "flag_counts": flag_counts,
"outcomes": {k: len(v) for k, v in outs.items()},
}
res["chance"] = {}
for key in ("blue_meadow", "shadow"):
O = OUTCOMES[key]
D = [r for r in runs if match(r, O["denom"])]
ev = lambda r, O=O: match(r, O["event"]) # noqa: E731
by = lambda k: {s: rate(g, ev) for s, g in sorted(group_by(D, k).items())} # noqa: E731
form = [r for r in D if r["entry"] == "Form"]
res["chance"][key] = {"label": O["label"], "overall": rate(D, ev), "by_stratum": by(stratum), "by_goal": by("goal"),
"by_attempt": by("attempt_group"), "form_overall": rate(form, ev)}
fs = group_by([r for r in runs if not is_missing(r["direction"]) and not is_missing(r["company"])], lambda r: r["company"] + "|" + r["direction"])
res["first_sighting"] = [{"key": k, "n": len(g), "colors": {c: len(v) for c, v in sorted(group_by(g, "first_sighting").items())}}
for k, g in sorted(fs.items())]
res["encounters"] = []
for e in M["encounters"]:
col, colour = e["column"], e["colour"]
# a rider who chose to burn here (chasing another colour) did not fail this encounter
A = [r for r in runs if match(r, e["where"]) and r.get(col) in ("Bonded!", "Incinerated") and not r.get("_deliberate")]
win = lambda r, col=col: r.get(col) == "Bonded!" # noqa: E731
intent = [r for r in A if r["goal"] == colour or r["goal"].startswith("Any") or (r["goal"] == "Blue or Black" and colour in ("Blue", "Black"))]
res["encounters"].append({"context": e["context"], "color": colour, "overall": rate(A, win), "intent_matched": rate(intent, win),
"by_stratum": {s: rate(g, win) for s, g in sorted(group_by(A, stratum).items())}})
greens_where = next(e["where"] for e in M["encounters"] if e["colour"] == "Green")
greens = [r for r in runs if match(r, greens_where) and r.get("enc_Green") and not r.get("_deliberate")]
g = {"green_encounters": len(greens),
"bonded_green": sum(1 for r in greens if r["enc_Green"] == "Bonded!"),
"rescued_orange": sum(1 for r in greens if r["rescued"] and r.get("outcome_color") == "Orange"),
"rescued_brown": sum(1 for r in greens if r["rescued"] and r.get("outcome_color") == "Brown"),
"incinerated": sum(1 for r in greens if r["outcome"] == "Incinerated")}
g["no_outcome"] = g["green_encounters"] - g["bonded_green"] - g["rescued_orange"] - g["rescued_brown"] - g["incinerated"]
g["rates"] = {k: W(g[k], g["green_encounters"]) for k in ("bonded_green", "rescued_orange", "rescued_brown", "incinerated", "no_outcome")}
res["green_rescue"] = g
res["factors"] = {k: factor_table(runs, k) for k in OUTCOMES}
first2 = lambda r: r["first_sighting"] if r["first_sighting"] in ("Red", "Brown", "Orange", "Green") else None # noqa: E731
known = lambda f: (lambda r: None if is_missing(r.get(f)) else r[f]) # noqa: E731
src2 = lambda r: SOURCE_LABEL.get(r["source"]) or ("Form submission" if r["entry"] == "Form" else "Bulk log (import)") # noqa: E731
res["models"] = {
"blue_meadow": fit_model(runs, "blue_meadow", {"source": src2, "goal": goal2, "direction": known("direction"),
"first_sighting": first2, "resolution": known("resolution"), "company": known("company")}),
"shadow": fit_model(runs, "shadow", {"source": src2, "goal": goal2, "attempt": known("attempt_group"), "direction": known("direction")}),
"bond_blue_meadow": fit_model(runs, "bond_blue_meadow", {"source": src2, "goal": goal2, "direction": known("direction"), "resolution": known("resolution")}),
}
res["daily"] = []
for d, gr in sorted(group_by([r for r in runs if r.get("date")], "date").items()):
m = [r for r in gr if match(r, OUTCOMES["blue_meadow"]["denom"])]
t = [r for r in gr if match(r, OUTCOMES["shadow"]["denom"])]
bk, sk = sum(1 for r in m if r["saw_blue_meadow"]), sum(1 for r in t if r["saw_shadow"])
res["daily"].append({"date": d, "runs": len(gr), "meadow_n": len(m), "blue_k": bk, "tree_n": len(t), "shadow_k": sk,
"blue": W(bk, len(m)), "shadow": W(sk, len(t))})
res["source_rates"] = {key: {s: rate([r for r in g if match(r, OUTCOMES[key]["denom"])], lambda r, k=key: match(r, OUTCOMES[k]["event"]))
for s, g in group_by(runs, entry_group).items()} for key in ("blue_meadow", "shadow")}
return res
# ---------------------------------------------------------------- route map and route rankings
def stage_value(r, key):
if key in ("company", "direction", "post_path"):
return None if is_missing(r.get(key)) else r[key]
if key == "first_sighting":
return None if r["scream"] == "DIED at first sighting" or is_missing(r["first_sighting"]) else r["first_sighting"]
if key == "cadet":
if r["scream"].startswith("DIED") or is_missing(r["scream"]) or is_missing(r["resolution"]):
return None
return "Help → " + r["resolution"] if r["scream"] == "Investigate & help" else r["scream"] + " → " + r["resolution"]
if key == "hour":
return "Climb the tree" if r.get("hour") == "Tree" else "Valley centre" if r.get("hour") == "Valley" else None
if key == "encounter":
if not r.get("branch"):
return None
if r["hour"] == "Tree":
if r["branch"] == "Shadow (rare)":
return "Tree: shadow → Blue/Black"
if r["branch"] == "Descend quickly: Red":
return "Tree: Red below"
return "Tree: two Greens"
if r["branch"] == "Valley Red":
return "Valley: bond the Red"
for cond, label in ((r.get("enc_Blue"), "Meadow: Blue"), (r.get("enc_Orange") and r.get("enc_Brown"), "Meadow: Orange/Brown"),
(r.get("enc_Orange"), "Meadow: Orange"), (r.get("enc_Brown"), "Meadow: Brown")):
if cond:
return label
return "Meadow (no attempt recorded)"
if key == "end":
if r["outcome"] == "Bonded":
return "Bonded " + r["outcome_color"]
if r["outcome"] in ("Incinerated", "Died before the hour"):
return r["outcome"]
return None
def route_map(runs):
"""The route map: nodes and links with run counts, and for every target the hits and their Wilson interval."""
nodes, links = {}, {}
hits = {t: {} for t in TARGETS}
tries = {t: {} for t in TARGETS} # runs that count for the target (no deliberate deaths for a bond)
for r in runs:
prev = None
hit = {t: match(r, T["event"]) for t, T in TARGETS.items()}
counts = {t: not (r.get("_deliberate") and t.startswith("bond_")) for t in TARGETS}
for si, s in enumerate(STAGES):
v = stage_value(r, s["key"])
if v is None:
continue
nid = f"{si}|{v}"
nd = nodes.setdefault(nid, {"id": nid, "stage": si, "stageKey": s["key"], "value": v, "n": 0})
nd["n"] += 1
for t in TARGETS:
hits[t][nid] = hits[t].get(nid, 0) + hit[t]
tries[t][nid] = tries[t].get(nid, 0) + counts[t]
if prev:
lid = prev + "→" + nid
ln = links.setdefault(lid, {"id": lid, "source": prev, "target": nid, "n": 0, "chance": bool(s.get("chance"))})
ln["n"] += 1
for t in TARGETS:
hits[t][lid] = hits[t].get(lid, 0) + hit[t]
tries[t][lid] = tries[t].get(lid, 0) + counts[t]
prev = nid
nl, ll = list(nodes.values()), list(links.values())
by_target = {t: {"nodes": [W(hits[t][n["id"]], tries[t][n["id"]]) for n in nl],
"links": [W(hits[t][l["id"]], tries[t][l["id"]]) for l in ll]} for t in TARGETS}
return {"nodes": nl, "links": ll, "by_target": by_target}
def rank_routes(runs, target, thresholds):
"""Full routes (every choice up to the hour), ranked for one target. Each route with at least min_n runs is
compared with all other runs that could reach the target (Fisher), and within data-source / chase-goal strata
(Mantel-Haenszel). q-values are corrected within each threshold. Ranked by the lower end of the 95% interval."""
T = TARGETS[target]
ev = lambda r: match(r, T["event"]) # noqa: E731
key_stages = [s["key"] for s in STAGES[:6]]
pool = [r for r in runs if match(r, T["pool"])]
tot_k, tot_n = sum(1 for r in pool if ev(r)), len(pool)
full = [r for r in pool if all(stage_value(r, k) or (k == "post_path" and r.get("post_path") is None) for k in key_stages)]
groups = group_by(full, lambda r: " › ".join(stage_value(r, k) or "—" for k in key_stages))
strata = list(group_by(pool, stratum).values())
rows = []
for key, g in groups.items():
if len(g) < min(thresholds):
continue
k, n = sum(1 for r in g if ev(r)), len(g)
o = {"route": key, "values": [stage_value(g[0], s) for s in key_stages], **W(k, n)}
o.update(odds(k, n - k, tot_k - k, tot_n - n - (tot_k - k)))
ids = {id(r) for r in g}
tabs = []
for sg in strata:
a = sum(1 for r in sg if id(r) in ids and ev(r))
nin = sum(1 for r in sg if id(r) in ids)
c = sum(1 for r in sg if id(r) not in ids and ev(r))
tabs.append([a, nin - a, c, len(sg) - nin - c])
mh = mantel_haenszel(tabs)
o.update(dict(zip(("mh_or", "mh_lo", "mh_hi", "mh_p"), mh)) if mh else {"mh_or": None, "mh_lo": None, "mh_hi": None, "mh_p": None})
rows.append(o)
rows.sort(key=lambda r: (-r["lo"], -r["n"]))
out = {"target": target, "overall": W(tot_k, tot_n), "rows": rows, "by_threshold": {}}
for m in thresholds:
idx = [i for i, r in enumerate(rows) if r["n"] >= m]
q = bh([rows[i]["fisher_p"] for i in idx])
mi = [i for i in idx if rows[i]["mh_p"] is not None]
mq = bh([rows[i]["mh_p"] for i in mi])
out["by_threshold"][str(m)] = {"n_routes_tested": len(idx), "q": {str(i): v for i, v in zip(idx, q)},
"mh_q": {str(i): v for i, v in zip(mi, mq)}}
return out
# ---------------------------------------------------------------- the valley map and the summary cards
def place_ids():
"""The places drawn on the valley map (the keys of L in site/atlas/valley.js)."""
src = (ROOT / "site/atlas/valley.js").read_text(encoding="utf-8")
body = src[src.index("const L = {"):]
body = body[:body.index("\n };")]
return set(re.findall(r'^\s{4}"?([^":{}\n]+(?::[^":{}\n]+)*)"?:\s*\{\s*x:', body, re.M))
PLACES = None
def place_path(r):
"""The places a run passed through, as drawn on the valley map (mirrors path() in valley.js)."""
L = PLACES
s = []
if "comp:" + str(r["company"]) in L:
s.append("comp:" + r["company"])
if r["direction"] not in ("East", "West", "Center"):
return s
s.append("dir:" + r["direction"])
sight = f"see:{r['direction']}:{r['first_sighting']}"
if sight in L:
s.append(sight)
if r["scream"] in ("DIED at first sighting", "Unknown"):
return s
s.append("scream")
if r["scream"] == "Investigate & help":
s.append("bully")
if "res:" + r["resolution"] in L:
s.append("res:" + r["resolution"])
if (r.get("resolution") or "").startswith("DIED"):
return s
if r.get("post_path") in ("Forest", "West", "East"):
s.append("fork:" + r["post_path"])
elif r["scream"] == "Ignore the scream":
s.append("wounded")
if "res:" + r["resolution"] in L:
s.append("res:" + r["resolution"])
elif r["scream"] == "Investigate & leave":
s.append("away")
if r.get("hour") not in ("Tree", "Valley"):
return s
s.append("hour")
b = r.get("branch")
if r["hour"] == "Tree":
s.append("tree")
if b == "Descend quickly: Red":
s.append("treeRed")
elif b == "Descend quickly: let the Red pass":
s += ["treeRed", "greens"]
elif b == "Descend slowly: two Greens":
s.append("greens")
elif b == "Shadow (rare)":
s.append("stream")
if r.get("enc_Blue") or r.get("enc_Black"):
s.append("redwood")
if r.get("enc_Black"):
s.append("black")
else:
s.append("valley")
if b == "Valley Red":
s.append("valleyRed")
elif b == "Passed the Red: meadow":
s.append("meadow")
oc = r.get("outcome_color")
tried = "Blue" if r.get("enc_Blue") else oc if oc in ("Orange", "Brown") else "Orange" if r.get("enc_Orange") else "Brown" if r.get("enc_Brown") else None
if tried:
s.append("meadow:" + tried)
return s
def valley(runs):
counts, edges = {}, {}
for r in runs:
seq = place_path(r)
for i, p in enumerate(seq):
counts[p] = counts.get(p, 0) + 1
if i:
k = seq[i - 1] + ">" + p
edges[k] = edges.get(k, 0) + 1
return {"counts": counts, "edges": [[*k.split(">"), n] for k, n in sorted(edges.items(), key=lambda kv: -kv[1])]}
DEATHS = ("Incinerated", "Died before the hour")
def deliberate(r):
"""A deliberate death: burned at a bonding encounter the rider did not genuinely attempt (chasing another colour,
choosing to die and try again). The route model's own rule."""
if r["outcome"] != "Incinerated":
return False
try:
v = model.prepare_run(r)
return bool(v.get("encCtx")) and not model.genuine_attempt(r, v["encCtx"])
except (KeyError, TypeError, ValueError): # a source coded from other questions (e.g. the bot): not known
return False
def path_layers(runs):
"""'Show the paths of all riders' on the valley map: for every layer in measures.yaml path_layers (one source or
more), how many runs went from each place to the next (like valley()), split by how the runs ended: `lived` (did
not die), `chose` (a deliberate death) and `failed` (every other death), plus where the deaths happened. Every
source is used here, not only the statistics' sources. Sources that record no route are counted only."""
out = []
for layer in M["path_layers"]:
sel = [r for r in runs if r["source"] in layer["sources"]]
kinds = {"lived": [], "chose": [], "failed": []}
for r in sel:
kinds["lived" if r["outcome"] not in DEATHS else "chose" if deliberate(r) else "failed"].append(r)
ends = {}
for k in ("chose", "failed"):
e = {}
for r in kinds[k]:
seq = place_path(r)
if seq:
e[seq[-1]] = e.get(seq[-1], 0) + 1
ends[k] = dict(sorted(e.items(), key=lambda kv: -kv[1]))
out.append({"key": layer["key"], "label": layer["label"], "runs": len(sel),
"counts": {k: len(v) for k, v in kinds.items()},
"lines": {k: valley(v)["edges"] for k, v in kinds.items()}, "ends": ends})
unrouted = {s: sum(1 for r in runs if r["source"] == s) for s in M.get("path_unrouted", [])}
return {"layers": out, "unrouted": unrouted}
def overview(runs):
"""The Odds' summary cards: cadets who went in and either bonded or died. Deliberate deaths (burned at an
encounter the rider did not genuinely attempt) and runs with no recorded outcome are left out."""
ended = [r for r in runs if r["outcome"] in ("Bonded", "Incinerated", "Died before the hour")]
cadets = [r for r in ended if not deliberate(r)]
bonded = sum(1 for r in cadets if r["outcome"] == "Bonded")
return {"cadets": len(cadets), "bonded": bonded, "dead": len(cadets) - bonded, "deliberate": len(ended) - len(cadets),
"early_deaths": sum(1 for r in cadets if r["outcome"] == "Died before the hour")}
# ---------------------------------------------------------------- everything
def clean(o):
if isinstance(o, dict):
return {str(k): clean(v) for k, v in o.items()}
if isinstance(o, (list, tuple)):
return [clean(v) for v in o]
if hasattr(o, "item"):
o = o.item()
if isinstance(o, float) and (math.isinf(o) or math.isnan(o)):
return None
return o
def build(runs, rows, exported=None, every_run=None):
global PLACES
PLACES = place_ids()
th = M["route_thresholds"]
out = {
"schema": 1,
"data": {"form_rows": rows, "runs": len(runs), "exported": exported},
"meta": {
"presets": {k: v["label"] for k, v in M["presets"].items()}, "route_thresholds": th,
"targets": {k: {"label": t["label"], "color": t["colour"]} for k, t in TARGETS.items()},
"outcomes": {k: {"label": o["label"]} for k, o in OUTCOMES.items()},
"factors": M["factors"], "stages": [{"key": s["key"], "label": s["label"], "chance": bool(s.get("chance"))} for s in STAGES],
"missing_values": sorted(coding.MISSING),
},
"presets": {},
}
for pid, p in M["presets"].items():
sub = apply_filters(runs, p["filters"])
out["presets"][pid] = {
"label": p["label"], "analyse": analyse(runs, p["filters"]),
"routes": {t: rank_routes(sub, t, th) for t in TARGETS},
"map": route_map(sub), "valley": valley(sub), "overview": overview(sub),
}
if every_run is not None:
out["paths"] = path_layers(every_run)
return clean(out)
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--data", default=str(runs_io.PATH))
ap.add_argument("--out", default=str(ROOT / "data/public/stats.json"))
a = ap.parse_args(argv)
# a run that cannot have happened as recorded (core/spec/form.yaml impossible_flags) is left out, as in the model
runs = [r for r in runs_io.read(a.data, sources=M["sources"]) if not runs_io.impossible(r)]
for r in runs:
r["_deliberate"] = deliberate(r)
every = [r for r in runs_io.read(a.data) if not runs_io.impossible(r)]
out = build(runs, len(runs), exported(), every_run=every)
Path(a.out).write_text(json.dumps(out, separators=(",", ":"), ensure_ascii=False), encoding="utf-8")
print(f"{a.out}: {len(runs)} runs, {len(out['presets'])} presets, {Path(a.out).stat().st_size / 1e6:.1f} MB")
def exported():
"""When the community spreadsheet last changed (data/manifest.json), else the date in its public file."""
m = ROOT / "data/manifest.json"
entry = json.loads(m.read_text(encoding="utf-8")).get("reddit_spreadsheet") if m.exists() else None
if entry:
return entry["changed"]
return json.loads((ROOT / "data/public/form_responses.json").read_text(encoding="utf-8")).get("exported")
if __name__ == "__main__":
main()
+99
View File
@@ -0,0 +1,99 @@
"""The statistics behind every number on the page: Wilson intervals, odds ratios with Fisher's test, Benjamini-Hochberg
q-values, Mantel-Haenszel, ridge logistic regression. Tested against known answers in tests/core/test_stats.py."""
import math
import numpy as np
from scipy.stats import fisher_exact, norm
Z95 = 1.959963984540054
def wilson(k, n, z=Z95):
"""Wilson score 95% interval for a proportion k/n. Returns (p, lo, hi); None when n == 0."""
if n == 0:
return (None, None, None)
p = k / n
d = 1 + z * z / n
c = (p + z * z / (2 * n)) / d
h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
return (p, max(0.0, c - h), min(1.0, c + h))
def odds_ratio(a, b, c, d):
"""2x2 table [[a, b], [c, d]] = [[exposed&event, exposed&no-event], [unexposed&event, unexposed&no-event]].
Returns OR, 95% CI (Woolf log method with Haldane-Anscombe +0.5 correction when any cell is 0),
and the two-sided Fisher exact p-value (computed on the uncorrected table)."""
corr = 0.5 if 0 in (a, b, c, d) else 0.0
A, B, C, D = a + corr, b + corr, c + corr, d + corr
or_ = (A * D) / (B * C)
se = math.sqrt(1 / A + 1 / B + 1 / C + 1 / D)
lo, hi = math.exp(math.log(or_) - Z95 * se), math.exp(math.log(or_) + Z95 * se)
p = fisher_exact([[a, b], [c, d]], alternative="two-sided")[1]
return or_, lo, hi, p, corr > 0
def bh(pvals):
"""Benjamini-Hochberg false-discovery-rate adjusted q-values."""
m = len(pvals)
if m == 0:
return []
order = sorted(range(m), key=lambda i: pvals[i])
q = [0.0] * m
prev = 1.0
for rank in range(m, 0, -1):
i = order[rank - 1]
prev = min(prev, pvals[i] * m / rank)
q[i] = prev
return q
def logistic_ridge(X, y, lam=1.0, iters=100, tol=1e-10):
"""Logistic regression fitted by Newton-Raphson (IRLS) with a small L2 penalty `lam`
on every coefficient except the intercept (column 0). The penalty keeps estimates finite
when a category has 0 events (complete separation). Returns beta and Wald standard errors
from the penalised information matrix."""
X = np.asarray(X, float)
y = np.asarray(y, float)
n, k = X.shape
beta = np.zeros(k)
P = np.eye(k) * lam
P[0, 0] = 0
for _ in range(iters):
eta = X @ beta
mu = 1 / (1 + np.exp(-eta))
W = mu * (1 - mu)
grad = X.T @ (y - mu) - P @ beta
H = (X * W[:, None]).T @ X + P
step = np.linalg.solve(H, grad)
beta += step
if np.max(np.abs(step)) < tol:
break
mu = 1 / (1 + np.exp(-(X @ beta)))
H = (X * (mu * (1 - mu))[:, None]).T @ X + P
se = np.sqrt(np.diag(np.linalg.inv(H)))
return beta, se
def p_from_z(z):
return 2 * (1 - norm.cdf(abs(z)))
def mantel_haenszel(tables):
"""Mantel-Haenszel common OR over 2x2 tables [a, b, c, d], Robins-Breslow-Greenland 95% CI,
and the CMH chi-square p-value (1 df, continuity-corrected)."""
R = S = PR = PSQR = QS = sA = sE = sV = 0.0
for a, b, c, d in tables:
n = a + b + c + d
if n < 2:
continue
r, s_ = a * d / n, b * c / n
P, Q = (a + d) / n, (b + c) / n
R += r; S += s_; PR += P * r; PSQR += P * s_ + Q * r; QS += Q * s_
m1, m0, n1, n0 = a + b, c + d, a + c, b + d
sA += a; sE += m1 * n1 / n; sV += m1 * m0 * n1 * n0 / (n * n * (n - 1))
if not R or not S:
return None
or_ = R / S
se = math.sqrt(PR / (2 * R * R) + PSQR / (2 * R * S) + QS / (2 * S * S))
chi = max(0.0, abs(sA - sE) - 0.5) ** 2 / sV if sV > 0 else 0.0
return or_, math.exp(math.log(or_) - Z95 * se), math.exp(math.log(or_) + Z95 * se), p_from_z(math.sqrt(chi))