site built from 6b4b1344
This commit is contained in:
@@ -0,0 +1,323 @@
|
||||
"""Community form rows -> one coded run per row.
|
||||
|
||||
The tables (how each question is found, what every answer means, which answers count as missing) live in
|
||||
core/spec/form.yaml. This module holds the logic that combines answers into a run: which way the rider went,
|
||||
which branch they took after the hour, how the run ended, and which rows look inconsistent (data_flags).
|
||||
|
||||
python -m atlas.coding [form_responses.json|.xlsx|.csv] [out.csv]
|
||||
|
||||
Prints a summary of the coded export (default: data/public/form_responses.json), and writes the coded runs to
|
||||
out.csv when given. The published runs (every source) are written by `python -m collection build`.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from . import spec
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
F = spec.form()
|
||||
A = F["answers"]
|
||||
COLORS = F["colours"]
|
||||
MISSING = frozenset(F["missing_values"])
|
||||
ENC = F["encounter_answers"]
|
||||
_EMAIL = re.compile(F["redact"]["pattern"])
|
||||
|
||||
|
||||
def is_missing(v):
|
||||
"""True for answers that carry no information about the choice (blank, "Unsure", not asked...)."""
|
||||
return v is None or v == "" or v in MISSING or (isinstance(v, float) and v != v)
|
||||
|
||||
|
||||
def redact(v):
|
||||
"""Replace email addresses in free text."""
|
||||
return v if v is None else _EMAIL.sub(F["redact"]["replacement"], str(v))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- reading cells
|
||||
|
||||
def find_columns(cols):
|
||||
"""Map each question key in form.yaml to the export's column header. Returns (columns, missing keys)."""
|
||||
out = {}
|
||||
low = {c: str(c).lower() for c in cols}
|
||||
for key, col in F["columns"].items():
|
||||
words = col["match"]
|
||||
hits = [c for c in cols if all((w[1:] not in low[c]) if w[0] == "!" else (w in low[c]) for w in words)]
|
||||
if hits:
|
||||
out[key] = hits[0]
|
||||
missing = [k for k, col in F["columns"].items()
|
||||
if k not in out and not col.get("optional") and col.get("replaced_by") not in out]
|
||||
return out, missing
|
||||
|
||||
|
||||
def blank(v):
|
||||
if v is None or v is pd.NaT:
|
||||
return True
|
||||
if not isinstance(v, str):
|
||||
try:
|
||||
if pd.isna(v):
|
||||
return True
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return str(v).strip() in ("", "nan", "NaN", "NaT")
|
||||
|
||||
|
||||
def text(v):
|
||||
return None if blank(v) else str(v).strip()
|
||||
|
||||
|
||||
def parse_timestamp(v):
|
||||
"""Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'.
|
||||
Returns 'YYYY-MM-DD' or None."""
|
||||
if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v):
|
||||
return pd.Timestamp(v).strftime("%Y-%m-%d")
|
||||
if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date
|
||||
return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d")
|
||||
t = text(v)
|
||||
if not t:
|
||||
return None
|
||||
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
|
||||
m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY
|
||||
if m:
|
||||
return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
|
||||
return None
|
||||
|
||||
|
||||
def parse_attempt(v):
|
||||
v = text(v)
|
||||
if v is None:
|
||||
return None
|
||||
t = v.lower()
|
||||
if "lost count" in t:
|
||||
return None
|
||||
for w, n in F["attempt_words"].items():
|
||||
if t.startswith(w):
|
||||
return n
|
||||
m = re.search(r"\d+", t)
|
||||
return int(m.group()) if m else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- one row -> one run
|
||||
|
||||
def code_row(r, C):
|
||||
"""Code one export row. r: mapping header -> cell; C: from find_columns()."""
|
||||
g = lambda k: text(r[C[k]]) if k in C else None # noqa: E731
|
||||
flags = []
|
||||
ts = r[C["timestamp"]] if "timestamp" in C else None
|
||||
d = parse_timestamp(ts)
|
||||
source, date = ("Form", d) if d else ((text(ts) or "Unknown import"), None)
|
||||
|
||||
wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)]
|
||||
wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown")
|
||||
section = g("wing_" + wings[0]) if len(wings) == 1 else None
|
||||
if len(wings) > 1:
|
||||
flags.append("multiple wings")
|
||||
|
||||
if "first_dir" in C:
|
||||
# Redesigned form: one direction question and one colour question. The older per-direction colour
|
||||
# columns are still in the sheet for some rows and are used only to fill gaps and to cross-check.
|
||||
direction = A["direction"].get(g("first_dir"))
|
||||
first = g("first_colour")
|
||||
legacy = [d for d in ("West", "Center") if g("leg_" + d)]
|
||||
if not direction and len(legacy) == 1:
|
||||
direction = legacy[0]
|
||||
flags.append("direction taken from the older sighting column")
|
||||
leg = g("leg_" + direction) if direction else None
|
||||
if not first and leg:
|
||||
first = leg
|
||||
elif first and leg and first != leg and first != "Unsure" and leg != "Unsure":
|
||||
flags.append("first sighting differs between the old and new columns")
|
||||
if not direction:
|
||||
direction = "Unknown"
|
||||
flags.append("direction missing")
|
||||
else:
|
||||
dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)]
|
||||
if len(dirs) == 1:
|
||||
direction, first = dirs[0], g("dir_" + dirs[0])
|
||||
else:
|
||||
direction, first = ("Multiple" if dirs else "Unknown"), None
|
||||
flags.append("direction " + ("multiple" if dirs else "missing"))
|
||||
|
||||
company_raw = g("company") or "Unknown"
|
||||
suffix = F["company_left_suffix"]
|
||||
left_squad = company_raw.endswith(suffix)
|
||||
company = company_raw[: -len(suffix)] if left_squad else company_raw
|
||||
if not left_squad and company in ("1 Squadmate", "2 Squadmates") and not (date and date >= F["left_squadmates_asked_from"]):
|
||||
left_squad = None # the form did not offer "left them" yet: unknown, not "stayed together"
|
||||
alias = F.get("first_sighting_alias") or {}
|
||||
if first in alias:
|
||||
flags.append(f"first sighting answered as {first!r}")
|
||||
first = alias[first]
|
||||
pools = F["sighting_pool"]
|
||||
pool = list(pools.get(company, {}).get(direction, [])) + (list(pools["Alone"].get(direction, [])) if left_squad else [])
|
||||
if first and first != "Unsure" and pool and first not in pool:
|
||||
flags.append(f"first sighting {first} not in documented pool for {company}/{direction}")
|
||||
|
||||
cadet_raw = g("cadet")
|
||||
combat = combat2 = None
|
||||
if "cadet_actions" in C:
|
||||
act = g("cadet_actions")
|
||||
a = A["cadet_actions"].get(act, ["Unknown", None, None, "Unknown"])
|
||||
if act and act not in A["cadet_actions"]:
|
||||
flags.append("unrecognised cadet answer")
|
||||
scream, combat, combat2 = a[0], a[1], a[2]
|
||||
out = A["cadet_outcome"].get(cadet_raw)
|
||||
if a[3] is not None:
|
||||
resolution = a[3]
|
||||
if out == "DIED in combat" and not resolution.startswith("DIED"):
|
||||
flags.append("died in combat on a path without combat")
|
||||
else:
|
||||
resolution = out or "Unknown"
|
||||
else:
|
||||
scream, resolution = A["cadet_legacy"].get(cadet_raw, ["Unknown", "Unknown"])
|
||||
if cadet_raw and cadet_raw not in A["cadet_legacy"]:
|
||||
flags.append("unrecognised cadet answer")
|
||||
combat = A["combat_legacy"].get(g("combat")) if g("combat") else None
|
||||
|
||||
post_raw = g("post")
|
||||
if scream == "Investigate & help" and not resolution.startswith("DIED"):
|
||||
post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown"
|
||||
elif scream in ("Ignore the scream", "Investigate & leave"):
|
||||
post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch
|
||||
else:
|
||||
post = None
|
||||
|
||||
hour = A["hour"].get(g("hour"))
|
||||
died_early = scream.startswith("DIED") or resolution.startswith("DIED")
|
||||
if died_early:
|
||||
hour = None
|
||||
elif hour is None:
|
||||
hour = "Unknown"
|
||||
|
||||
enc = {c: g("enc_" + c) for c in COLORS}
|
||||
rare = A["rare"].get(g("rare"))
|
||||
bonded = [c for c in COLORS if enc[c] == ENC["bonded"]]
|
||||
burned = [c for c in COLORS if enc[c] == ENC["burned"]]
|
||||
|
||||
# Which encounter branch did the run take after the hour?
|
||||
branch = None
|
||||
if hour == "Tree":
|
||||
if enc["Blue"] or enc["Black"] or rare == "Shadow":
|
||||
branch = "Shadow (rare)"
|
||||
elif enc["Red"] == ENC["passed"] and enc["Green"]:
|
||||
# only climbing down quickly puts you above the Red: passing on it and meeting the Greens means the
|
||||
# rider let it pass
|
||||
branch = "Descend quickly: let the Red pass"
|
||||
elif enc["Green"]:
|
||||
branch = "Descend slowly: two Greens"
|
||||
if enc["Red"]:
|
||||
flags.append("tree run with both the Red and the two Greens attempted")
|
||||
elif enc["Red"]:
|
||||
branch = "Descend quickly: Red"
|
||||
elif enc["Orange"] or enc["Brown"]:
|
||||
branch = "Descend slowly: two Greens"
|
||||
flags.append("tree run with Orange/Brown but no Green recorded")
|
||||
elif hour == "Valley":
|
||||
if enc["Green"]:
|
||||
flags.append("Green recorded on a Valley run (no Greens in the valley)")
|
||||
meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]] or rare == "Blue (valley)"
|
||||
if meadow or enc["Red"] == ENC["passed"]:
|
||||
branch = "Passed the Red: meadow"
|
||||
elif enc["Red"]:
|
||||
branch = "Valley Red"
|
||||
|
||||
# How the run ended
|
||||
outcome, outcome_color = None, None
|
||||
if died_early:
|
||||
outcome = "Died before the hour"
|
||||
elif len(bonded) == 1:
|
||||
outcome, outcome_color = "Bonded", bonded[0]
|
||||
elif len(bonded) > 1:
|
||||
outcome = "Ambiguous"
|
||||
flags.append("more than one dragon marked Bonded")
|
||||
elif burned:
|
||||
order = F["burn_order"]["valley" if hour == "Valley" else "tree"]
|
||||
outcome, outcome_color = "Incinerated", [c for c in order if c in burned][-1]
|
||||
if len(burned) > 1:
|
||||
flags.append("more than one dragon marked Incinerated")
|
||||
elif hour == "DIED before the hour":
|
||||
outcome = "Died before the hour"
|
||||
else:
|
||||
outcome = "No outcome recorded"
|
||||
|
||||
rescued = enc["Green"] == ENC["rescued"] and outcome == "Bonded" and outcome_color in ("Orange", "Brown")
|
||||
attempt = parse_attempt(g("attempt"))
|
||||
if (rare == "Blue (valley)" and hour == "Tree") or (rare == "Shadow" and hour == "Valley"):
|
||||
flags.append("rare-dragon answer does not match the path")
|
||||
goal_raw = g("goal_v2")
|
||||
goal = g("goal") or (A["goal_v2"].get(goal_raw, goal_raw) if goal_raw and goal_raw != "Unknown" else None) or "(not asked/blank)"
|
||||
return {
|
||||
"source": source, "date": date, "wing": wing, "section": section,
|
||||
"squad": A["squad_any_case"].get((g("squad") or "").strip().lower(), g("squad") or "Unknown"), "attempt": attempt,
|
||||
# "I've lost count" means many attempts, so it is a later attempt, not missing
|
||||
"attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if (attempt and attempt > 1) or "lost count" in (g("attempt") or "").lower() else "Unknown"),
|
||||
"goal": goal, "company": company, "left_squadmates": left_squad,
|
||||
"direction": direction, "first_sighting": first or "Unknown",
|
||||
"sighting_action": (A["sighting_action"].get(g("sighting_action")) or g("sighting_action") or "Unknown") if "sighting_action" in C else "(not asked)",
|
||||
"scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)",
|
||||
"combat_second": combat2 or "(not recorded)",
|
||||
"post_path": post, "hour": hour, "branch": branch,
|
||||
**{"enc_" + c: enc[c] for c in COLORS},
|
||||
"outcome": outcome, "outcome_color": outcome_color, "rescued": rescued,
|
||||
"saw_blue_meadow": bool(hour == "Valley" and (enc["Blue"] or rare == "Blue (valley)")),
|
||||
"saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"] or rare == "Shadow")),
|
||||
"rare_reported": (rare or "Unknown") if "rare" in C else "(not asked)",
|
||||
"malek_intent": (g("malek") or "Unknown") if "malek" in C else "(not asked)",
|
||||
"tail": g("tail"),
|
||||
"data_flags": "; ".join(flags),
|
||||
"death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- files
|
||||
|
||||
def read_export(path):
|
||||
"""An export as a DataFrame: the bundled JSON (data/public/form_responses.json), a CSV or the .xlsx."""
|
||||
p = str(path).lower()
|
||||
if p.endswith(".json"):
|
||||
with open(path, encoding="utf-8") as f:
|
||||
d = json.load(f)
|
||||
return pd.DataFrame(d["rows"], columns=d["headers"])
|
||||
if p.endswith(".csv"):
|
||||
return pd.read_csv(path)
|
||||
xl = pd.ExcelFile(path)
|
||||
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
|
||||
return pd.read_excel(xl, sheet)
|
||||
|
||||
|
||||
def code_frame(raw):
|
||||
"""Code every row of an export. Returns (runs DataFrame with run_id, missing question keys)."""
|
||||
C, missing = find_columns(raw.columns)
|
||||
df = pd.DataFrame([code_row(r, C) for _, r in raw.iterrows()])
|
||||
df.insert(0, "run_id", range(1, len(df) + 1))
|
||||
return df, missing
|
||||
|
||||
|
||||
def load(path):
|
||||
return code_frame(read_export(path))
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = sys.argv[1:] if argv is None else argv
|
||||
src = argv[0] if argv else ROOT / "data/public/form_responses.json"
|
||||
out = argv[1] if len(argv) > 1 else None
|
||||
df, missing = load(src)
|
||||
if missing:
|
||||
print("WARNING: columns not found:", missing)
|
||||
# a whole-number column with gaps would otherwise be written as 1.0, 2.0, ... (data/dictionary.yaml: integer)
|
||||
df["attempt"] = df["attempt"].astype("Int64")
|
||||
if out:
|
||||
df.to_csv(out, index=False)
|
||||
print(f"{len(df)} runs" + (f" -> {out}" if out else ""))
|
||||
print(df.outcome.value_counts().to_string())
|
||||
print("rows with flags:", (df["data_flags"] != "").sum())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,193 @@
|
||||
# The community "Threshing Routes" form: how each question is found in an export and how every answer is coded.
|
||||
#
|
||||
# This file is the single source of truth for the coding rules' tables. core/atlas/coding.py reads it; the logic
|
||||
# that combines answers (which branch a run took, how it ended) lives there and is described on the page's
|
||||
# Methods tab. To support a new or renamed answer, add it here and add a case to tests/core/test_coding.py.
|
||||
version: 1
|
||||
|
||||
colours: [Red, Green, Orange, Brown, Blue, Black]
|
||||
|
||||
# Answers that carry no information about the choice. They are kept in the coded data for transparency, but every
|
||||
# analysis treats them as missing: never shown or compared as if they were a choice.
|
||||
missing_values: ["Unknown", "Unsure", "Unsure/unknown", "Multiple", "(not recorded)", "(not asked)",
|
||||
"(not asked/blank)", "Unknown/lost count", "Blank/not asked"]
|
||||
|
||||
# Free-text answers sometimes contain an email address; it is replaced before anything is displayed or exported.
|
||||
redact:
|
||||
pattern: '[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}'
|
||||
replacement: "[email removed]"
|
||||
|
||||
# Columns are found by words in the question text (case-insensitive), so the coding keeps working when the form
|
||||
# owner reorders, renames slightly or adds questions. A word starting with "!" must NOT appear in the header.
|
||||
columns:
|
||||
timestamp: { match: ["timestamp"] }
|
||||
wing_First: { match: ["wing and section", "[first]"] }
|
||||
wing_Second: { match: ["wing and section", "[second]"] }
|
||||
wing_Third: { match: ["wing and section", "[third]"] }
|
||||
wing_Fourth: { match: ["wing and section", "[fourth]"] }
|
||||
squad: { match: ["what squad"] }
|
||||
attempt: { match: ["attempt number"] }
|
||||
company: { match: ["who accompanied"] }
|
||||
dir_East: { match: ["initial direction", "[east"], replaced_by: first_dir }
|
||||
dir_West: { match: ["initial direction", "[west"], replaced_by: first_dir }
|
||||
dir_Center: { match: ["initial direction", "[forest center"], replaced_by: first_dir }
|
||||
cadet: { match: ["outcome of your cadet encounter"] }
|
||||
post: { match: ["path did you take after the bully"] }
|
||||
hour: { match: ["after walking an hour"] }
|
||||
enc_Red: { match: ["attempt to bond", "[red"] }
|
||||
enc_Green: { match: ["attempt to bond", "[green"] }
|
||||
enc_Orange: { match: ["attempt to bond", "[orange"] }
|
||||
enc_Brown: { match: ["attempt to bond", "[brown"] }
|
||||
enc_Blue: { match: ["attempt to bond", "[blue"] }
|
||||
enc_Black: { match: ["attempt to bond", "[black"] }
|
||||
death_note: { match: ["those who died"] }
|
||||
bond_note: { match: ["those who bonded"] }
|
||||
goal: { match: ["colour dragon are you trying"], replaced_by: goal_v2 }
|
||||
combat: { match: ["combat choice"], replaced_by: cadet_actions }
|
||||
# Questions added in the redesigned form (late October 2026)
|
||||
first_dir: { match: ["what was your first direction"], replaced_by: dir_Center }
|
||||
first_colour: { match: ["colour was your initial dragon sighting", "!["], replaced_by: dir_Center }
|
||||
leg_West: { match: ["colour was your initial dragon sighting", "[west"], optional: true }
|
||||
leg_Center: { match: ["colour was your initial dragon sighting", "[forest center"], optional: true }
|
||||
sighting_action: { match: ["action did you take during the first dragon sighting"], optional: true }
|
||||
cadet_actions: { match: ["during your cadet encounter"], replaced_by: combat }
|
||||
goal_v2: { match: ["why are you doing this run"], replaced_by: goal }
|
||||
malek: { match: ["seek malek"], optional: true }
|
||||
rare: { match: ["encounter a rare dragon"], optional: true }
|
||||
tail: { match: ["what tail does your dragon have"], optional: true }
|
||||
|
||||
# Answer → coded value, one table per question.
|
||||
answers:
|
||||
# Older form: one answer for the whole cadet encounter → [scream, resolution]
|
||||
cadet_legacy:
|
||||
"Bullies > You dealt with them": ["Investigate & help", "Killed the bullies yourself"]
|
||||
"Bullies > Squadmate's revenge": ["Investigate & help", "Squadmate's revenge"]
|
||||
"Bullies > Left to the Dragons": ["Investigate & help", "Left bullies to the dragons"]
|
||||
"Bullies > Assessed the situation": ["Investigate & help", "Unknown"]
|
||||
"Injured Cadet > Tourniquet": ["Ignore the scream", "Tourniquet"]
|
||||
"Injured Cadet > Left them there": ["Ignore the scream", "Left injured cadet"]
|
||||
"Injured Cadet > Took their weapons": ["Ignore the scream", "Took their weapons"]
|
||||
"Avoided combat (Investigated scream > Left them to it)": ["Investigate & leave", "Left them to it"]
|
||||
"You died in combat": ["Investigate & help", "DIED in combat"]
|
||||
"N/A - You died with initial dragon sighting": ["DIED at first sighting", "DIED at first sighting"]
|
||||
# Older form: the first move in the fight
|
||||
combat_legacy:
|
||||
"Bullies > Assessed the situation": "Assess the situation"
|
||||
"Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)"
|
||||
"Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)": "Put yourself in the middle (bigger cadet)"
|
||||
# Redesigned form: the encounter as a path of actions → [scream, first move, second move, resolution or null]
|
||||
# (null resolution: the separate outcome question decides it, see cadet_outcome)
|
||||
cadet_actions:
|
||||
"Bullies > Assess > Unknown": ["Investigate & help", "Assess the situation", null, null]
|
||||
"Bullies > Assess > Smaller Cadet": ["Investigate & help", "Assess the situation", "Engage the smaller cadet", null]
|
||||
"Bullies > Assess > Daggers or not": ["Investigate & help", "Assess the situation", "Daggers or not (protect squadmate)", null]
|
||||
"Bullies > Unknown": ["Investigate & help", null, null, null]
|
||||
"Bullies > Smaller Cadet (Did not assess)": ["Investigate & help", "Even the numbers (smaller cadet)", null, null]
|
||||
"Bullies > Bigger Cadet (Did not assess)": ["Investigate & help", "Put yourself in the middle (bigger cadet)", null, null]
|
||||
"Injured Cadet > Tourniquet": ["Ignore the scream", null, null, "Tourniquet"]
|
||||
"Injured Cadet > Left them there": ["Ignore the scream", null, null, "Left injured cadet"]
|
||||
"Injured Cadet > Took their weapons": ["Ignore the scream", null, null, "Took their weapons"]
|
||||
"Avoided combat (Investigated scream > Left them to it)": ["Investigate & leave", null, null, "Left them to it"]
|
||||
"N/A - You died with initial dragon sighting": ["DIED at first sighting", null, null, "DIED at first sighting"]
|
||||
"Unsure": ["Unknown", null, null, "Unknown"]
|
||||
cadet_outcome:
|
||||
"Squadmate's revenge": "Squadmate's revenge"
|
||||
"You ended the bullies yourself": "Killed the bullies yourself"
|
||||
"Left to the Dragons": "Left bullies to the dragons"
|
||||
"You died in combat": "DIED in combat"
|
||||
direction:
|
||||
"Forest Center": Center
|
||||
"West / Scent": West
|
||||
"East / Roar": East
|
||||
goal_v2:
|
||||
"Hoping for a Rare Dragon (Blue or Black)": "Blue or Black"
|
||||
"Trying to bond the Red dragon under the tree": "Red (tree)"
|
||||
"Daily Run Assignment data": "Daily run data"
|
||||
"Just here for fun, and maybe a dragon": "Any (for fun)"
|
||||
# Squad: matched ignoring case and spaces (riders type it); an answer not listed still stops the build
|
||||
squad_any_case:
|
||||
"first": "First"
|
||||
"second": "Second"
|
||||
"third": "Third"
|
||||
"don't know": "Don't know"
|
||||
sighting_action:
|
||||
"Hold Ground / Stay Still": "Hold still"
|
||||
"Respect / Face it": "Respect / face it"
|
||||
"Retreat / Run / Back Away": "Retreat / run"
|
||||
"Hide / Duck": "Hide"
|
||||
"Protect / Help Squadmate": "Protect squadmate"
|
||||
"Draw / Drop Weapon": "Weapon"
|
||||
"Meet Gaze": "Meet its gaze"
|
||||
"Protect / Help Sqaudmate": "Protect squadmate"
|
||||
"Approach": "Approach"
|
||||
"Stand on your own": "Stand alone"
|
||||
"Unsure": "Unknown"
|
||||
"Unsure - You don't remember": "Unknown"
|
||||
rare:
|
||||
"Yes - A blue dragon in the valley": "Blue (valley)"
|
||||
"Yes - Shadow!!!": "Shadow"
|
||||
"No": "No"
|
||||
hour:
|
||||
"Continued to Valley Centre": Valley
|
||||
"Climbed the tallest tree": Tree
|
||||
"N/A - You're dead": "DIED before the hour"
|
||||
|
||||
# Suffix on the company answer that means the rider later left their squadmates. The answer was only offered from
|
||||
# this date (none of the 504 squad runs before it chose it; 40 of 144 after it did): earlier and undated squad runs
|
||||
# are coded as unknown (empty), not as "stayed together".
|
||||
company_left_suffix: " > Left them"
|
||||
left_squadmates_asked_from: "2026-10-27"
|
||||
|
||||
# The encounter answers in the "attempt to bond" grid
|
||||
encounter_answers: { bonded: "Bonded!", burned: "Incinerated", passed: "Changed my mind", rescued: "Saved by another" }
|
||||
|
||||
# When more than one dragon is marked Incinerated, the run ended on the last one met, in this order.
|
||||
burn_order:
|
||||
tree: [Red, Green, Orange, Brown, Blue, Black]
|
||||
valley: [Red, Orange, Brown, Blue, Green, Black]
|
||||
|
||||
# First-dragon pools per company and direction, from the community PDF. Only used to FLAG rows that report a
|
||||
# dragon outside the documented pool. Leaving your squadmates sends you into the solo pools too.
|
||||
sighting_pool:
|
||||
Alone: { East: [Red, Orange], West: [Red, Brown, Green], Center: [Red, Brown] }
|
||||
1 Squadmate: { East: [Green, Orange, Red], West: [Red, Brown, Green], Center: [Red, Brown] }
|
||||
2 Squadmates: { East: [Red, Orange], West: [Green, Brown, Red], Center: [Red, Brown] }
|
||||
|
||||
# The dragons each bonding encounter can end with (bond or burn), from the community PDF. Only used to FLAG runs
|
||||
# answered through "Report a run" (reported, reddit comment) that end with a dragon the encounter does not have.
|
||||
# Orange and Brown after the two Greens: a Green's rescue, then the Orange or Brown.
|
||||
# Flags (data_flags, matched as text inside a flag) that mean a run cannot have happened as recorded: the game's own
|
||||
# rules, from the form coder (core/atlas/coding.py) and the checks of collection/etl/runs.py rule_flags. A run with one
|
||||
# is kept in runs_clean.csv, flagged, and left out of the route model and the site statistics (atlas.runs.impossible).
|
||||
# Everything else, however partial, is used. Not here on purpose: a first sighting outside the documented pool (the
|
||||
# pool is what riders have seen, not a rule), a missing direction, several wings, a missing Green before a rescue.
|
||||
impossible_flags:
|
||||
- "Green recorded on a Valley run"
|
||||
- "Black recorded on a Valley run"
|
||||
- "encounter, which has only"
|
||||
- "more than one dragon marked Bonded"
|
||||
- "more than one dragon marked Incinerated"
|
||||
- "tree run with both the Red and the two Greens attempted"
|
||||
- "died in combat on a path without combat"
|
||||
- "rare-dragon answer does not match the path"
|
||||
- "does not follow"
|
||||
- "fight answers on a run that did not help"
|
||||
- "a path after the fight on a run without a fight"
|
||||
- "a second fight move without pausing to assess"
|
||||
- "died before the hour, yet reached"
|
||||
|
||||
encounter_colours:
|
||||
"Shadow (rare)": [Blue, Black]
|
||||
"Descend quickly: Red": [Red]
|
||||
"Descend quickly: let the Red pass": [Green]
|
||||
"Descend slowly: two Greens": [Green, Orange, Brown]
|
||||
"Valley Red": [Red]
|
||||
"Passed the Red: meadow": [Orange, Brown, Blue]
|
||||
|
||||
# First-sighting answers that are not a single colour: coded as this colour, and the row is flagged with the
|
||||
# original answer (data_flags), so nothing is lost.
|
||||
first_sighting_alias:
|
||||
"2 Greens": Green
|
||||
|
||||
# How a bulk-pasted row's attempt answer is read
|
||||
attempt_words: { first: 1, second: 2, third: 3, fourth: 4, fifth: 5 }
|
||||
@@ -0,0 +1,108 @@
|
||||
# What the site measures: outcomes and their denominators, route targets and their comparison groups, the
|
||||
# bonding-encounter contexts, the stages of the route map, the filter presets and the minimum-run thresholds.
|
||||
# analysis/stats/site_stats.py reads this file and writes data/public/stats.json, which the page only displays.
|
||||
#
|
||||
# Conditions: {field: value} means equal; {field: {in: [...]}}, {field: {not: value}}, {field: {exists: true}}
|
||||
# (not blank), {any: [condition, ...]} (at least one). Several fields in one condition must all hold.
|
||||
version: 1
|
||||
|
||||
# Which runs of data/public/runs_clean.csv the statistics use (its `source` column): every source that records the
|
||||
# route (the community form, "Report a run", the Reddit megathread comments, the DragonKind bot). The Reddit result
|
||||
# pages and the register record only the end, so they stay out. The bot plays to a strategy and burns on purpose at
|
||||
# encounters it is not chasing: like every rider's deliberate death (`_deliberate`, the route model's rule), those
|
||||
# burns are left out of the bond rates, never counted as failed bonds.
|
||||
sources: [reddit spreadsheet, reported, reddit comment, bot]
|
||||
|
||||
colours: [Red, Green, Orange, Brown, Blue, Black]
|
||||
|
||||
# Outcomes for the factor tables and models: who could have had the outcome (denom), and the outcome (event).
|
||||
outcomes:
|
||||
blue_meadow:
|
||||
label: Blue appears in the meadow
|
||||
denom: { branch: "Passed the Red: meadow" }
|
||||
event: { saw_blue_meadow: true }
|
||||
shadow:
|
||||
label: Shadow option appears at the tree
|
||||
denom: { hour: Tree, branch: { exists: true } }
|
||||
event: { saw_shadow: true }
|
||||
bond_blue_meadow:
|
||||
label: Bond, given you approached the meadow Blue
|
||||
denom: { branch: "Passed the Red: meadow", enc_Blue: { in: ["Bonded!", "Incinerated"] }, _deliberate: { not: true } }
|
||||
event: { enc_Blue: "Bonded!" }
|
||||
reach_hour:
|
||||
label: Survive to the hour (tree/valley choice)
|
||||
denom: {}
|
||||
event: { hour: { in: [Tree, Valley] } }
|
||||
# one more outcome per colour: "Run ends bonded to <colour>", among runs that reached the hour
|
||||
bonded_outcome:
|
||||
label: Run ends bonded to {colour}
|
||||
denom: { hour: { in: [Tree, Valley] }, _deliberate: { not: true } }
|
||||
event: { outcome: Bonded, outcome_color: "{colour}" }
|
||||
|
||||
# Factors compared in the factor tables (each answer against the other answers to the same question)
|
||||
factors: [company, direction, first_sighting, scream, resolution, combat_first, post_path, hour, wing, squad,
|
||||
attempt_group, entry, goal, sighting_action, combat_second, malek_intent]
|
||||
|
||||
# Route targets: the outcome (event) and which runs could have reached it at all (pool)
|
||||
targets:
|
||||
bond_Blue: { label: Bond Blue, colour: Blue, event: { outcome: Bonded, outcome_color: Blue }, pool: { hour: Valley, _deliberate: { not: true } } }
|
||||
bond_Black: { label: Bond Black, colour: Black, event: { outcome: Bonded, outcome_color: Black }, pool: { hour: Tree, _deliberate: { not: true } } }
|
||||
see_Blue: { label: Meet a Blue (meadow), colour: Blue, event: { saw_blue_meadow: true }, pool: { branch: "Passed the Red: meadow" } }
|
||||
see_shadow: { label: Get the tree shadow, colour: Black, event: { saw_shadow: true }, pool: { hour: Tree, branch: { exists: true } } }
|
||||
bond_Red: { label: Bond Red, colour: Red, event: { outcome: Bonded, outcome_color: Red }, pool: { hour: { in: [Tree, Valley] }, _deliberate: { not: true } } }
|
||||
bond_Green: { label: Bond Green, colour: Green, event: { outcome: Bonded, outcome_color: Green }, pool: { hour: Tree, _deliberate: { not: true } } }
|
||||
bond_Orange: { label: Bond Orange, colour: Orange, event: { outcome: Bonded, outcome_color: Orange }, pool: { hour: { in: [Tree, Valley] }, _deliberate: { not: true } } }
|
||||
bond_Brown: { label: Bond Brown, colour: Brown, event: { outcome: Bonded, outcome_color: Brown }, pool: { hour: { in: [Tree, Valley] }, _deliberate: { not: true } } }
|
||||
bond_any: { label: Bond any dragon, colour: null, event: { outcome: Bonded }, pool: { hour: { in: [Tree, Valley] }, _deliberate: { not: true } } }
|
||||
|
||||
# Bonding encounters: which recorded attempt (column) belongs to which in-game encounter (where)
|
||||
encounters:
|
||||
- { context: "Red (tree, descend quickly)", colour: Red, column: enc_Red, where: { hour: Tree, branch: "Descend quickly: Red" } }
|
||||
- { context: "Green (two Greens)", colour: Green, column: enc_Green, where: { hour: Tree, any: [ { branch: "Descend slowly: two Greens" }, { branch: "Descend quickly: let the Red pass" } ] } }
|
||||
- { context: "Blue (shadow)", colour: Blue, column: enc_Blue, where: { hour: Tree, branch: "Shadow (rare)" } }
|
||||
- { context: "Black (shadow)", colour: Black, column: enc_Black, where: { hour: Tree, branch: "Shadow (rare)" } }
|
||||
- { context: "Red (valley)", colour: Red, column: enc_Red, where: { hour: Valley, branch: Valley Red } }
|
||||
- { context: "Orange (meadow)", colour: Orange, column: enc_Orange, where: { hour: Valley, branch: "Passed the Red: meadow" } }
|
||||
- { context: "Brown (meadow)", colour: Brown, column: enc_Brown, where: { hour: Valley, branch: "Passed the Red: meadow" } }
|
||||
- { context: "Blue (meadow)", colour: Blue, column: enc_Blue, where: { hour: Valley, branch: "Passed the Red: meadow" } }
|
||||
|
||||
# The route map's columns, in order. "chance": steps the game rolls, not the rider.
|
||||
stages:
|
||||
- { key: company, label: Company }
|
||||
- { key: direction, label: Direction }
|
||||
- { key: first_sighting, label: First sighting, chance: true }
|
||||
- { key: cadet, label: Cadet encounter }
|
||||
- { key: post_path, label: Fork in the road }
|
||||
- { key: hour, label: After an hour }
|
||||
- { key: encounter, label: Encounter, chance: true }
|
||||
- { key: end, label: Outcome }
|
||||
|
||||
# Filter presets: the page shows one at a time (no statistics are computed in the browser)
|
||||
presets:
|
||||
all: { label: All runs, filters: { source: all, goal: all, excludeFlagged: false } }
|
||||
form: { label: Form submissions only, filters: { source: form, goal: all, excludeFlagged: false } }
|
||||
bulk: { label: Bulk logs only (imports), filters: { source: bulk, goal: all, excludeFlagged: false } }
|
||||
reddit: { label: Reddit comments only, filters: { source: reddit, goal: all, excludeFlagged: false } }
|
||||
bot: { label: DragonKind bot only, filters: { source: bot, goal: all, excludeFlagged: false } }
|
||||
not_chasing: { label: Not chasing Blue/Black, filters: { source: all, goal: exclude_rare, excludeFlagged: false } }
|
||||
chasing: { label: Chasing Blue/Black, filters: { source: all, goal: only_rare, excludeFlagged: false } }
|
||||
unflagged: { label: Rows without data-quality flags, filters: { source: all, goal: all, excludeFlagged: true } }
|
||||
|
||||
# Minimum runs a full route needs before it is ranked (q-values are corrected within each threshold)
|
||||
route_thresholds: [5, 10, 20, 30]
|
||||
|
||||
rare_goals: [Blue, Black, Blue or Black]
|
||||
|
||||
# "Show the paths of all riders" on the valley map (site/atlas/explorer.js): one layer per source that records a
|
||||
# route, each with its own switch. Unlike everything above, the layers use every source of runs_clean.csv. Within a
|
||||
# layer the runs are split by how they ended, drawn as line styles in the layer's colour: solid (did not die), dashed
|
||||
# (chose to die: burned at an encounter the rider did not genuinely attempt, the route model's rule) and dotted
|
||||
# (failed: every other death). Discord has no runs yet: its layer is ready for when it is imported.
|
||||
path_layers:
|
||||
- { key: sheet, label: "Google Sheet", sources: [reddit spreadsheet] }
|
||||
- { key: reported, label: "Reported on the Atlas", sources: [reported] }
|
||||
- { key: reddit, label: "Reddit megathread comments", sources: [reddit comment] }
|
||||
- { key: bot, label: "DragonKind bot", sources: [bot] }
|
||||
- { key: discord, label: "Discord", sources: [discord] }
|
||||
# sources that record which dragon bonded but not the route: counted on the map, not drawn
|
||||
path_unrouted: [reddit thread, register]
|
||||
@@ -0,0 +1,595 @@
|
||||
"""Every number the Atlas page shows, precomputed: data/public/stats.json.
|
||||
|
||||
The page only displays these; it computes no statistics itself. What is measured (outcomes, route targets,
|
||||
encounters, the route map's stages, the filter presets and route thresholds) is declared in measures.yaml.
|
||||
|
||||
python analysis/stats/site_stats.py [--out data/public/stats.json]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parents[1]
|
||||
sys.path.insert(0, str(ROOT / "core"))
|
||||
sys.path.insert(0, str(HERE))
|
||||
from atlas import coding, model # noqa: E402
|
||||
from atlas import runs as runs_io # noqa: E402
|
||||
from stats_core import Z95, bh, logistic_ridge, mantel_haenszel, odds_ratio, p_from_z # noqa: E402
|
||||
from stats_core import wilson as _wilson # noqa: E402
|
||||
|
||||
M = yaml.safe_load(open(HERE / "measures.yaml", encoding="utf-8"))
|
||||
COLOURS = M["colours"]
|
||||
is_missing = coding.is_missing
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- conditions from measures.yaml
|
||||
|
||||
def match(r, cond):
|
||||
for k, want in cond.items():
|
||||
if k == "any":
|
||||
if not any(match(r, c) for c in want):
|
||||
return False
|
||||
continue
|
||||
v = r.get(k)
|
||||
if isinstance(want, dict):
|
||||
if "in" in want and v not in want["in"]:
|
||||
return False
|
||||
if "not" in want and v == want["not"]:
|
||||
return False
|
||||
if "exists" in want and (v is not None and v != "") != want["exists"]:
|
||||
return False
|
||||
elif v != want:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _fill(cond, colour):
|
||||
return json.loads(json.dumps(cond).replace("{colour}", colour))
|
||||
|
||||
|
||||
OUTCOMES = {k: v for k, v in M["outcomes"].items()}
|
||||
for c in COLOURS:
|
||||
b = M["bonded_outcome"]
|
||||
OUTCOMES["bonded_" + c] = {"label": b["label"].replace("{colour}", c), "denom": b["denom"], "event": _fill(b["event"], c)}
|
||||
TARGETS = M["targets"]
|
||||
STAGES = M["stages"]
|
||||
|
||||
|
||||
def W(k, n):
|
||||
p, lo, hi = _wilson(k, n)
|
||||
return {"k": int(k), "n": int(n), "p": p, "lo": lo, "hi": hi}
|
||||
|
||||
|
||||
def rate(runs, pred):
|
||||
return W(sum(1 for r in runs if pred(r)), len(runs))
|
||||
|
||||
|
||||
def group_by(runs, key):
|
||||
out = {}
|
||||
for r in runs:
|
||||
k = key(r) if callable(key) else r.get(key)
|
||||
out.setdefault(_key(k), []).append(r)
|
||||
return out
|
||||
|
||||
|
||||
def _key(v):
|
||||
"""Object keys as JavaScript writes them (true/false/null), so the page can look them up as before."""
|
||||
if v is True:
|
||||
return "true"
|
||||
if v is False:
|
||||
return "false"
|
||||
if v is None:
|
||||
return "null"
|
||||
if isinstance(v, float) and v.is_integer():
|
||||
return str(int(v))
|
||||
return str(v)
|
||||
|
||||
|
||||
def is_rare(goal):
|
||||
return goal in M["rare_goals"]
|
||||
|
||||
|
||||
# sources other than the community spreadsheet: their own group wherever runs are split by where they came from
|
||||
SOURCE_LABEL = {"reddit comment": "Reddit comments", "bot": "DragonKind bot", "reported": "Reported on the Atlas"}
|
||||
|
||||
|
||||
def entry_group(r):
|
||||
"""Where a run came from: the spreadsheet's entry type (Form, Data Import, ...) or the source's own label."""
|
||||
return SOURCE_LABEL.get(r["source"]) or r["entry"]
|
||||
|
||||
|
||||
def stratum(r):
|
||||
if r["source"] in SOURCE_LABEL:
|
||||
return SOURCE_LABEL[r["source"]]
|
||||
if r["entry"] != "Form":
|
||||
return "Bulk logs (imports)"
|
||||
g = r["goal"]
|
||||
return {"Blue": "Form: chasing Blue", "Black": "Form: chasing Black", "Blue or Black": "Form: chasing Blue or Black"}.get(g, "Form: other/blank goal")
|
||||
|
||||
|
||||
def goal2(r):
|
||||
return r["goal"] if is_rare(r["goal"]) else "Blank/not asked" if r["goal"] == "(not asked/blank)" else "Other colour/any"
|
||||
|
||||
|
||||
def apply_filters(runs, f):
|
||||
def ok(r):
|
||||
sheet = r["source"] == "reddit spreadsheet"
|
||||
if f.get("source") == "form" and not (sheet and r["entry"] == "Form"):
|
||||
return False
|
||||
if f.get("source") == "bulk" and not (sheet and r["entry"] != "Form"):
|
||||
return False
|
||||
if f.get("source") == "reddit" and r["source"] != "reddit comment":
|
||||
return False
|
||||
if f.get("source") == "bot" and r["source"] != "bot":
|
||||
return False
|
||||
if f.get("goal") == "exclude_rare" and is_rare(r["goal"]):
|
||||
return False
|
||||
if f.get("goal") == "only_rare" and not is_rare(r["goal"]):
|
||||
return False
|
||||
if f.get("excludeFlagged") and r["data_flags"]:
|
||||
return False
|
||||
return True
|
||||
return [r for r in runs if ok(r)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- tables and models
|
||||
|
||||
def odds(a, b, c, d):
|
||||
or_, lo, hi, p, corr = odds_ratio(a, b, c, d)
|
||||
return {"or": or_, "or_lo": lo, "or_hi": hi, "fisher_p": p, "corrected": corr}
|
||||
|
||||
|
||||
def factor_table(runs, key, min_n=5):
|
||||
O = OUTCOMES[key]
|
||||
D = [r for r in runs if match(r, O["denom"])]
|
||||
ev = lambda r: match(r, O["event"]) # noqa: E731
|
||||
rows, missing = [], {}
|
||||
for f in M["factors"]:
|
||||
known = [r for r in D if not is_missing(r.get(f))]
|
||||
missing[f] = len(D) - len(known)
|
||||
groups = group_by(known, f)
|
||||
nf, ef = len(known), sum(1 for r in known if ev(r))
|
||||
for lv in sorted(groups):
|
||||
g = groups[lv]
|
||||
k, n = sum(1 for r in g if ev(r)), len(g)
|
||||
row = {"factor": f, "level": lv, **W(k, n)}
|
||||
rk, rn = ef - k, nf - n
|
||||
if n >= min_n and rn >= min_n and 0 < ef < nf:
|
||||
row.update(odds(k, n - k, rk, rn - rk))
|
||||
rows.append(row)
|
||||
tested = [r for r in rows if "fisher_p" in r]
|
||||
for r, q in zip(tested, bh([r["fisher_p"] for r in tested])):
|
||||
r["q"] = q
|
||||
return {"outcome": key, "label": O["label"], "overall": W(sum(1 for r in D if ev(r)), len(D)), "rows": rows, "missing": missing}
|
||||
|
||||
|
||||
def fit_model(runs, key, factors, lam=1.0):
|
||||
import numpy as np
|
||||
O = OUTCOMES[key]
|
||||
D = []
|
||||
for r in runs:
|
||||
if not match(r, O["denom"]):
|
||||
continue
|
||||
o = {"y": 1 if match(r, O["event"]) else 0, **{name: fn(r) for name, fn in factors.items()}}
|
||||
if all(o[f] is not None for f in factors):
|
||||
D.append(o)
|
||||
if len(D) < 20:
|
||||
return None
|
||||
cols, names, refs = [np.ones(len(D))], [], {}
|
||||
for f in factors:
|
||||
counts = group_by(D, f)
|
||||
levels = sorted(counts, key=lambda lv: (-len(counts[lv]), lv))
|
||||
refs[f] = levels[0]
|
||||
for lv in levels[1:]:
|
||||
cols.append(np.array([1.0 if _key(o[f]) == lv else 0.0 for o in D]))
|
||||
names.append((f, lv, len(counts[lv])))
|
||||
y = np.array([o["y"] for o in D])
|
||||
b, se = logistic_ridge(np.column_stack(cols), y, lam)
|
||||
terms = [{"term": f"{f} = {lv}", "factor": f, "level": lv, "n": n, "or": math.exp(b[i + 1]),
|
||||
"lo": math.exp(b[i + 1] - Z95 * se[i + 1]), "hi": math.exp(b[i + 1] + Z95 * se[i + 1]),
|
||||
"p": float(p_from_z(b[i + 1] / se[i + 1]))} for i, (f, lv, n) in enumerate(names)]
|
||||
return {"outcome": key, "label": O["label"], "n": len(D), "events": int(y.sum()), "refs": refs, "lambda": lam, "terms": terms}
|
||||
|
||||
|
||||
def route_key(r):
|
||||
return [r["company"], r["direction"], r["first_sighting"], r["scream"], r["resolution"], r.get("post_path") or "—", r.get("hour") or "—"]
|
||||
|
||||
|
||||
def analyse(all_runs, filters):
|
||||
runs = apply_filters(all_runs, filters)
|
||||
res = {"filters": filters, "n": len(runs)}
|
||||
|
||||
flag_counts = {}
|
||||
for r in runs:
|
||||
for f in (r["data_flags"].split("; ") if r["data_flags"] else []):
|
||||
k = "first sighting outside documented pool" if f.startswith("first sighting") else f
|
||||
flag_counts[k] = flag_counts.get(k, 0) + 1
|
||||
dates = sorted(r["date"] for r in runs if r.get("date"))
|
||||
outs = group_by(runs, lambda r: r["outcome"] + (" " + r["outcome_color"] if r.get("outcome_color") else ""))
|
||||
res["meta"] = {
|
||||
"sources": {k: len(v) for k, v in group_by(runs, entry_group).items()},
|
||||
"date_min": dates[0] if dates else None, "date_max": dates[-1] if dates else None,
|
||||
"n_flagged": sum(1 for r in runs if r["data_flags"]), "flag_counts": flag_counts,
|
||||
"outcomes": {k: len(v) for k, v in outs.items()},
|
||||
}
|
||||
|
||||
res["chance"] = {}
|
||||
for key in ("blue_meadow", "shadow"):
|
||||
O = OUTCOMES[key]
|
||||
D = [r for r in runs if match(r, O["denom"])]
|
||||
ev = lambda r, O=O: match(r, O["event"]) # noqa: E731
|
||||
by = lambda k: {s: rate(g, ev) for s, g in sorted(group_by(D, k).items())} # noqa: E731
|
||||
form = [r for r in D if r["entry"] == "Form"]
|
||||
res["chance"][key] = {"label": O["label"], "overall": rate(D, ev), "by_stratum": by(stratum), "by_goal": by("goal"),
|
||||
"by_attempt": by("attempt_group"), "form_overall": rate(form, ev)}
|
||||
|
||||
fs = group_by([r for r in runs if not is_missing(r["direction"]) and not is_missing(r["company"])], lambda r: r["company"] + "|" + r["direction"])
|
||||
res["first_sighting"] = [{"key": k, "n": len(g), "colors": {c: len(v) for c, v in sorted(group_by(g, "first_sighting").items())}}
|
||||
for k, g in sorted(fs.items())]
|
||||
|
||||
res["encounters"] = []
|
||||
for e in M["encounters"]:
|
||||
col, colour = e["column"], e["colour"]
|
||||
# a rider who chose to burn here (chasing another colour) did not fail this encounter
|
||||
A = [r for r in runs if match(r, e["where"]) and r.get(col) in ("Bonded!", "Incinerated") and not r.get("_deliberate")]
|
||||
win = lambda r, col=col: r.get(col) == "Bonded!" # noqa: E731
|
||||
intent = [r for r in A if r["goal"] == colour or r["goal"].startswith("Any") or (r["goal"] == "Blue or Black" and colour in ("Blue", "Black"))]
|
||||
res["encounters"].append({"context": e["context"], "color": colour, "overall": rate(A, win), "intent_matched": rate(intent, win),
|
||||
"by_stratum": {s: rate(g, win) for s, g in sorted(group_by(A, stratum).items())}})
|
||||
greens_where = next(e["where"] for e in M["encounters"] if e["colour"] == "Green")
|
||||
greens = [r for r in runs if match(r, greens_where) and r.get("enc_Green") and not r.get("_deliberate")]
|
||||
g = {"green_encounters": len(greens),
|
||||
"bonded_green": sum(1 for r in greens if r["enc_Green"] == "Bonded!"),
|
||||
"rescued_orange": sum(1 for r in greens if r["rescued"] and r.get("outcome_color") == "Orange"),
|
||||
"rescued_brown": sum(1 for r in greens if r["rescued"] and r.get("outcome_color") == "Brown"),
|
||||
"incinerated": sum(1 for r in greens if r["outcome"] == "Incinerated")}
|
||||
g["no_outcome"] = g["green_encounters"] - g["bonded_green"] - g["rescued_orange"] - g["rescued_brown"] - g["incinerated"]
|
||||
g["rates"] = {k: W(g[k], g["green_encounters"]) for k in ("bonded_green", "rescued_orange", "rescued_brown", "incinerated", "no_outcome")}
|
||||
res["green_rescue"] = g
|
||||
|
||||
res["factors"] = {k: factor_table(runs, k) for k in OUTCOMES}
|
||||
|
||||
first2 = lambda r: r["first_sighting"] if r["first_sighting"] in ("Red", "Brown", "Orange", "Green") else None # noqa: E731
|
||||
known = lambda f: (lambda r: None if is_missing(r.get(f)) else r[f]) # noqa: E731
|
||||
src2 = lambda r: SOURCE_LABEL.get(r["source"]) or ("Form submission" if r["entry"] == "Form" else "Bulk log (import)") # noqa: E731
|
||||
res["models"] = {
|
||||
"blue_meadow": fit_model(runs, "blue_meadow", {"source": src2, "goal": goal2, "direction": known("direction"),
|
||||
"first_sighting": first2, "resolution": known("resolution"), "company": known("company")}),
|
||||
"shadow": fit_model(runs, "shadow", {"source": src2, "goal": goal2, "attempt": known("attempt_group"), "direction": known("direction")}),
|
||||
"bond_blue_meadow": fit_model(runs, "bond_blue_meadow", {"source": src2, "goal": goal2, "direction": known("direction"), "resolution": known("resolution")}),
|
||||
}
|
||||
|
||||
res["daily"] = []
|
||||
for d, gr in sorted(group_by([r for r in runs if r.get("date")], "date").items()):
|
||||
m = [r for r in gr if match(r, OUTCOMES["blue_meadow"]["denom"])]
|
||||
t = [r for r in gr if match(r, OUTCOMES["shadow"]["denom"])]
|
||||
bk, sk = sum(1 for r in m if r["saw_blue_meadow"]), sum(1 for r in t if r["saw_shadow"])
|
||||
res["daily"].append({"date": d, "runs": len(gr), "meadow_n": len(m), "blue_k": bk, "tree_n": len(t), "shadow_k": sk,
|
||||
"blue": W(bk, len(m)), "shadow": W(sk, len(t))})
|
||||
|
||||
res["source_rates"] = {key: {s: rate([r for r in g if match(r, OUTCOMES[key]["denom"])], lambda r, k=key: match(r, OUTCOMES[k]["event"]))
|
||||
for s, g in group_by(runs, entry_group).items()} for key in ("blue_meadow", "shadow")}
|
||||
return res
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- route map and route rankings
|
||||
|
||||
def stage_value(r, key):
|
||||
if key in ("company", "direction", "post_path"):
|
||||
return None if is_missing(r.get(key)) else r[key]
|
||||
if key == "first_sighting":
|
||||
return None if r["scream"] == "DIED at first sighting" or is_missing(r["first_sighting"]) else r["first_sighting"]
|
||||
if key == "cadet":
|
||||
if r["scream"].startswith("DIED") or is_missing(r["scream"]) or is_missing(r["resolution"]):
|
||||
return None
|
||||
return "Help → " + r["resolution"] if r["scream"] == "Investigate & help" else r["scream"] + " → " + r["resolution"]
|
||||
if key == "hour":
|
||||
return "Climb the tree" if r.get("hour") == "Tree" else "Valley centre" if r.get("hour") == "Valley" else None
|
||||
if key == "encounter":
|
||||
if not r.get("branch"):
|
||||
return None
|
||||
if r["hour"] == "Tree":
|
||||
if r["branch"] == "Shadow (rare)":
|
||||
return "Tree: shadow → Blue/Black"
|
||||
if r["branch"] == "Descend quickly: Red":
|
||||
return "Tree: Red below"
|
||||
return "Tree: two Greens"
|
||||
if r["branch"] == "Valley Red":
|
||||
return "Valley: bond the Red"
|
||||
for cond, label in ((r.get("enc_Blue"), "Meadow: Blue"), (r.get("enc_Orange") and r.get("enc_Brown"), "Meadow: Orange/Brown"),
|
||||
(r.get("enc_Orange"), "Meadow: Orange"), (r.get("enc_Brown"), "Meadow: Brown")):
|
||||
if cond:
|
||||
return label
|
||||
return "Meadow (no attempt recorded)"
|
||||
if key == "end":
|
||||
if r["outcome"] == "Bonded":
|
||||
return "Bonded " + r["outcome_color"]
|
||||
if r["outcome"] in ("Incinerated", "Died before the hour"):
|
||||
return r["outcome"]
|
||||
return None
|
||||
|
||||
|
||||
def route_map(runs):
|
||||
"""The route map: nodes and links with run counts, and for every target the hits and their Wilson interval."""
|
||||
nodes, links = {}, {}
|
||||
hits = {t: {} for t in TARGETS}
|
||||
tries = {t: {} for t in TARGETS} # runs that count for the target (no deliberate deaths for a bond)
|
||||
for r in runs:
|
||||
prev = None
|
||||
hit = {t: match(r, T["event"]) for t, T in TARGETS.items()}
|
||||
counts = {t: not (r.get("_deliberate") and t.startswith("bond_")) for t in TARGETS}
|
||||
for si, s in enumerate(STAGES):
|
||||
v = stage_value(r, s["key"])
|
||||
if v is None:
|
||||
continue
|
||||
nid = f"{si}|{v}"
|
||||
nd = nodes.setdefault(nid, {"id": nid, "stage": si, "stageKey": s["key"], "value": v, "n": 0})
|
||||
nd["n"] += 1
|
||||
for t in TARGETS:
|
||||
hits[t][nid] = hits[t].get(nid, 0) + hit[t]
|
||||
tries[t][nid] = tries[t].get(nid, 0) + counts[t]
|
||||
if prev:
|
||||
lid = prev + "→" + nid
|
||||
ln = links.setdefault(lid, {"id": lid, "source": prev, "target": nid, "n": 0, "chance": bool(s.get("chance"))})
|
||||
ln["n"] += 1
|
||||
for t in TARGETS:
|
||||
hits[t][lid] = hits[t].get(lid, 0) + hit[t]
|
||||
tries[t][lid] = tries[t].get(lid, 0) + counts[t]
|
||||
prev = nid
|
||||
nl, ll = list(nodes.values()), list(links.values())
|
||||
by_target = {t: {"nodes": [W(hits[t][n["id"]], tries[t][n["id"]]) for n in nl],
|
||||
"links": [W(hits[t][l["id"]], tries[t][l["id"]]) for l in ll]} for t in TARGETS}
|
||||
return {"nodes": nl, "links": ll, "by_target": by_target}
|
||||
|
||||
|
||||
def rank_routes(runs, target, thresholds):
|
||||
"""Full routes (every choice up to the hour), ranked for one target. Each route with at least min_n runs is
|
||||
compared with all other runs that could reach the target (Fisher), and within data-source / chase-goal strata
|
||||
(Mantel-Haenszel). q-values are corrected within each threshold. Ranked by the lower end of the 95% interval."""
|
||||
T = TARGETS[target]
|
||||
ev = lambda r: match(r, T["event"]) # noqa: E731
|
||||
key_stages = [s["key"] for s in STAGES[:6]]
|
||||
pool = [r for r in runs if match(r, T["pool"])]
|
||||
tot_k, tot_n = sum(1 for r in pool if ev(r)), len(pool)
|
||||
full = [r for r in pool if all(stage_value(r, k) or (k == "post_path" and r.get("post_path") is None) for k in key_stages)]
|
||||
groups = group_by(full, lambda r: " › ".join(stage_value(r, k) or "—" for k in key_stages))
|
||||
strata = list(group_by(pool, stratum).values())
|
||||
rows = []
|
||||
for key, g in groups.items():
|
||||
if len(g) < min(thresholds):
|
||||
continue
|
||||
k, n = sum(1 for r in g if ev(r)), len(g)
|
||||
o = {"route": key, "values": [stage_value(g[0], s) for s in key_stages], **W(k, n)}
|
||||
o.update(odds(k, n - k, tot_k - k, tot_n - n - (tot_k - k)))
|
||||
ids = {id(r) for r in g}
|
||||
tabs = []
|
||||
for sg in strata:
|
||||
a = sum(1 for r in sg if id(r) in ids and ev(r))
|
||||
nin = sum(1 for r in sg if id(r) in ids)
|
||||
c = sum(1 for r in sg if id(r) not in ids and ev(r))
|
||||
tabs.append([a, nin - a, c, len(sg) - nin - c])
|
||||
mh = mantel_haenszel(tabs)
|
||||
o.update(dict(zip(("mh_or", "mh_lo", "mh_hi", "mh_p"), mh)) if mh else {"mh_or": None, "mh_lo": None, "mh_hi": None, "mh_p": None})
|
||||
rows.append(o)
|
||||
rows.sort(key=lambda r: (-r["lo"], -r["n"]))
|
||||
out = {"target": target, "overall": W(tot_k, tot_n), "rows": rows, "by_threshold": {}}
|
||||
for m in thresholds:
|
||||
idx = [i for i, r in enumerate(rows) if r["n"] >= m]
|
||||
q = bh([rows[i]["fisher_p"] for i in idx])
|
||||
mi = [i for i in idx if rows[i]["mh_p"] is not None]
|
||||
mq = bh([rows[i]["mh_p"] for i in mi])
|
||||
out["by_threshold"][str(m)] = {"n_routes_tested": len(idx), "q": {str(i): v for i, v in zip(idx, q)},
|
||||
"mh_q": {str(i): v for i, v in zip(mi, mq)}}
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the valley map and the summary cards
|
||||
|
||||
def place_ids():
|
||||
"""The places drawn on the valley map (the keys of L in site/atlas/valley.js)."""
|
||||
src = (ROOT / "site/atlas/valley.js").read_text(encoding="utf-8")
|
||||
body = src[src.index("const L = {"):]
|
||||
body = body[:body.index("\n };")]
|
||||
return set(re.findall(r'^\s{4}"?([^":{}\n]+(?::[^":{}\n]+)*)"?:\s*\{\s*x:', body, re.M))
|
||||
|
||||
|
||||
PLACES = None
|
||||
|
||||
|
||||
def place_path(r):
|
||||
"""The places a run passed through, as drawn on the valley map (mirrors path() in valley.js)."""
|
||||
L = PLACES
|
||||
s = []
|
||||
if "comp:" + str(r["company"]) in L:
|
||||
s.append("comp:" + r["company"])
|
||||
if r["direction"] not in ("East", "West", "Center"):
|
||||
return s
|
||||
s.append("dir:" + r["direction"])
|
||||
sight = f"see:{r['direction']}:{r['first_sighting']}"
|
||||
if sight in L:
|
||||
s.append(sight)
|
||||
if r["scream"] in ("DIED at first sighting", "Unknown"):
|
||||
return s
|
||||
s.append("scream")
|
||||
if r["scream"] == "Investigate & help":
|
||||
s.append("bully")
|
||||
if "res:" + r["resolution"] in L:
|
||||
s.append("res:" + r["resolution"])
|
||||
if (r.get("resolution") or "").startswith("DIED"):
|
||||
return s
|
||||
if r.get("post_path") in ("Forest", "West", "East"):
|
||||
s.append("fork:" + r["post_path"])
|
||||
elif r["scream"] == "Ignore the scream":
|
||||
s.append("wounded")
|
||||
if "res:" + r["resolution"] in L:
|
||||
s.append("res:" + r["resolution"])
|
||||
elif r["scream"] == "Investigate & leave":
|
||||
s.append("away")
|
||||
if r.get("hour") not in ("Tree", "Valley"):
|
||||
return s
|
||||
s.append("hour")
|
||||
b = r.get("branch")
|
||||
if r["hour"] == "Tree":
|
||||
s.append("tree")
|
||||
if b == "Descend quickly: Red":
|
||||
s.append("treeRed")
|
||||
elif b == "Descend quickly: let the Red pass":
|
||||
s += ["treeRed", "greens"]
|
||||
elif b == "Descend slowly: two Greens":
|
||||
s.append("greens")
|
||||
elif b == "Shadow (rare)":
|
||||
s.append("stream")
|
||||
if r.get("enc_Blue") or r.get("enc_Black"):
|
||||
s.append("redwood")
|
||||
if r.get("enc_Black"):
|
||||
s.append("black")
|
||||
else:
|
||||
s.append("valley")
|
||||
if b == "Valley Red":
|
||||
s.append("valleyRed")
|
||||
elif b == "Passed the Red: meadow":
|
||||
s.append("meadow")
|
||||
oc = r.get("outcome_color")
|
||||
tried = "Blue" if r.get("enc_Blue") else oc if oc in ("Orange", "Brown") else "Orange" if r.get("enc_Orange") else "Brown" if r.get("enc_Brown") else None
|
||||
if tried:
|
||||
s.append("meadow:" + tried)
|
||||
return s
|
||||
|
||||
|
||||
def valley(runs):
|
||||
counts, edges = {}, {}
|
||||
for r in runs:
|
||||
seq = place_path(r)
|
||||
for i, p in enumerate(seq):
|
||||
counts[p] = counts.get(p, 0) + 1
|
||||
if i:
|
||||
k = seq[i - 1] + ">" + p
|
||||
edges[k] = edges.get(k, 0) + 1
|
||||
return {"counts": counts, "edges": [[*k.split(">"), n] for k, n in sorted(edges.items(), key=lambda kv: -kv[1])]}
|
||||
|
||||
|
||||
DEATHS = ("Incinerated", "Died before the hour")
|
||||
|
||||
|
||||
def deliberate(r):
|
||||
"""A deliberate death: burned at a bonding encounter the rider did not genuinely attempt (chasing another colour,
|
||||
choosing to die and try again). The route model's own rule."""
|
||||
if r["outcome"] != "Incinerated":
|
||||
return False
|
||||
try:
|
||||
v = model.prepare_run(r)
|
||||
return bool(v.get("encCtx")) and not model.genuine_attempt(r, v["encCtx"])
|
||||
except (KeyError, TypeError, ValueError): # a source coded from other questions (e.g. the bot): not known
|
||||
return False
|
||||
|
||||
|
||||
def path_layers(runs):
|
||||
"""'Show the paths of all riders' on the valley map: for every layer in measures.yaml path_layers (one source or
|
||||
more), how many runs went from each place to the next (like valley()), split by how the runs ended: `lived` (did
|
||||
not die), `chose` (a deliberate death) and `failed` (every other death), plus where the deaths happened. Every
|
||||
source is used here, not only the statistics' sources. Sources that record no route are counted only."""
|
||||
out = []
|
||||
for layer in M["path_layers"]:
|
||||
sel = [r for r in runs if r["source"] in layer["sources"]]
|
||||
kinds = {"lived": [], "chose": [], "failed": []}
|
||||
for r in sel:
|
||||
kinds["lived" if r["outcome"] not in DEATHS else "chose" if deliberate(r) else "failed"].append(r)
|
||||
ends = {}
|
||||
for k in ("chose", "failed"):
|
||||
e = {}
|
||||
for r in kinds[k]:
|
||||
seq = place_path(r)
|
||||
if seq:
|
||||
e[seq[-1]] = e.get(seq[-1], 0) + 1
|
||||
ends[k] = dict(sorted(e.items(), key=lambda kv: -kv[1]))
|
||||
out.append({"key": layer["key"], "label": layer["label"], "runs": len(sel),
|
||||
"counts": {k: len(v) for k, v in kinds.items()},
|
||||
"lines": {k: valley(v)["edges"] for k, v in kinds.items()}, "ends": ends})
|
||||
unrouted = {s: sum(1 for r in runs if r["source"] == s) for s in M.get("path_unrouted", [])}
|
||||
return {"layers": out, "unrouted": unrouted}
|
||||
|
||||
|
||||
def overview(runs):
|
||||
"""The Odds' summary cards: cadets who went in and either bonded or died. Deliberate deaths (burned at an
|
||||
encounter the rider did not genuinely attempt) and runs with no recorded outcome are left out."""
|
||||
ended = [r for r in runs if r["outcome"] in ("Bonded", "Incinerated", "Died before the hour")]
|
||||
cadets = [r for r in ended if not deliberate(r)]
|
||||
bonded = sum(1 for r in cadets if r["outcome"] == "Bonded")
|
||||
return {"cadets": len(cadets), "bonded": bonded, "dead": len(cadets) - bonded, "deliberate": len(ended) - len(cadets),
|
||||
"early_deaths": sum(1 for r in cadets if r["outcome"] == "Died before the hour")}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- everything
|
||||
|
||||
def clean(o):
|
||||
if isinstance(o, dict):
|
||||
return {str(k): clean(v) for k, v in o.items()}
|
||||
if isinstance(o, (list, tuple)):
|
||||
return [clean(v) for v in o]
|
||||
if hasattr(o, "item"):
|
||||
o = o.item()
|
||||
if isinstance(o, float) and (math.isinf(o) or math.isnan(o)):
|
||||
return None
|
||||
return o
|
||||
|
||||
|
||||
def build(runs, rows, exported=None, every_run=None):
|
||||
global PLACES
|
||||
PLACES = place_ids()
|
||||
th = M["route_thresholds"]
|
||||
out = {
|
||||
"schema": 1,
|
||||
"data": {"form_rows": rows, "runs": len(runs), "exported": exported},
|
||||
"meta": {
|
||||
"presets": {k: v["label"] for k, v in M["presets"].items()}, "route_thresholds": th,
|
||||
"targets": {k: {"label": t["label"], "color": t["colour"]} for k, t in TARGETS.items()},
|
||||
"outcomes": {k: {"label": o["label"]} for k, o in OUTCOMES.items()},
|
||||
"factors": M["factors"], "stages": [{"key": s["key"], "label": s["label"], "chance": bool(s.get("chance"))} for s in STAGES],
|
||||
"missing_values": sorted(coding.MISSING),
|
||||
},
|
||||
"presets": {},
|
||||
}
|
||||
for pid, p in M["presets"].items():
|
||||
sub = apply_filters(runs, p["filters"])
|
||||
out["presets"][pid] = {
|
||||
"label": p["label"], "analyse": analyse(runs, p["filters"]),
|
||||
"routes": {t: rank_routes(sub, t, th) for t in TARGETS},
|
||||
"map": route_map(sub), "valley": valley(sub), "overview": overview(sub),
|
||||
}
|
||||
if every_run is not None:
|
||||
out["paths"] = path_layers(every_run)
|
||||
return clean(out)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--data", default=str(runs_io.PATH))
|
||||
ap.add_argument("--out", default=str(ROOT / "data/public/stats.json"))
|
||||
a = ap.parse_args(argv)
|
||||
# a run that cannot have happened as recorded (core/spec/form.yaml impossible_flags) is left out, as in the model
|
||||
runs = [r for r in runs_io.read(a.data, sources=M["sources"]) if not runs_io.impossible(r)]
|
||||
for r in runs:
|
||||
r["_deliberate"] = deliberate(r)
|
||||
every = [r for r in runs_io.read(a.data) if not runs_io.impossible(r)]
|
||||
out = build(runs, len(runs), exported(), every_run=every)
|
||||
Path(a.out).write_text(json.dumps(out, separators=(",", ":"), ensure_ascii=False), encoding="utf-8")
|
||||
print(f"{a.out}: {len(runs)} runs, {len(out['presets'])} presets, {Path(a.out).stat().st_size / 1e6:.1f} MB")
|
||||
|
||||
|
||||
def exported():
|
||||
"""When the community spreadsheet last changed (data/manifest.json), else the date in its public file."""
|
||||
m = ROOT / "data/manifest.json"
|
||||
entry = json.loads(m.read_text(encoding="utf-8")).get("reddit_spreadsheet") if m.exists() else None
|
||||
if entry:
|
||||
return entry["changed"]
|
||||
return json.loads((ROOT / "data/public/form_responses.json").read_text(encoding="utf-8")).get("exported")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,99 @@
|
||||
"""The statistics behind every number on the page: Wilson intervals, odds ratios with Fisher's test, Benjamini-Hochberg
|
||||
q-values, Mantel-Haenszel, ridge logistic regression. Tested against known answers in tests/core/test_stats.py."""
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
from scipy.stats import fisher_exact, norm
|
||||
|
||||
Z95 = 1.959963984540054
|
||||
|
||||
|
||||
def wilson(k, n, z=Z95):
|
||||
"""Wilson score 95% interval for a proportion k/n. Returns (p, lo, hi); None when n == 0."""
|
||||
if n == 0:
|
||||
return (None, None, None)
|
||||
p = k / n
|
||||
d = 1 + z * z / n
|
||||
c = (p + z * z / (2 * n)) / d
|
||||
h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
|
||||
return (p, max(0.0, c - h), min(1.0, c + h))
|
||||
|
||||
|
||||
def odds_ratio(a, b, c, d):
|
||||
"""2x2 table [[a, b], [c, d]] = [[exposed&event, exposed&no-event], [unexposed&event, unexposed&no-event]].
|
||||
Returns OR, 95% CI (Woolf log method with Haldane-Anscombe +0.5 correction when any cell is 0),
|
||||
and the two-sided Fisher exact p-value (computed on the uncorrected table)."""
|
||||
corr = 0.5 if 0 in (a, b, c, d) else 0.0
|
||||
A, B, C, D = a + corr, b + corr, c + corr, d + corr
|
||||
or_ = (A * D) / (B * C)
|
||||
se = math.sqrt(1 / A + 1 / B + 1 / C + 1 / D)
|
||||
lo, hi = math.exp(math.log(or_) - Z95 * se), math.exp(math.log(or_) + Z95 * se)
|
||||
p = fisher_exact([[a, b], [c, d]], alternative="two-sided")[1]
|
||||
return or_, lo, hi, p, corr > 0
|
||||
|
||||
|
||||
def bh(pvals):
|
||||
"""Benjamini-Hochberg false-discovery-rate adjusted q-values."""
|
||||
m = len(pvals)
|
||||
if m == 0:
|
||||
return []
|
||||
order = sorted(range(m), key=lambda i: pvals[i])
|
||||
q = [0.0] * m
|
||||
prev = 1.0
|
||||
for rank in range(m, 0, -1):
|
||||
i = order[rank - 1]
|
||||
prev = min(prev, pvals[i] * m / rank)
|
||||
q[i] = prev
|
||||
return q
|
||||
|
||||
|
||||
def logistic_ridge(X, y, lam=1.0, iters=100, tol=1e-10):
|
||||
"""Logistic regression fitted by Newton-Raphson (IRLS) with a small L2 penalty `lam`
|
||||
on every coefficient except the intercept (column 0). The penalty keeps estimates finite
|
||||
when a category has 0 events (complete separation). Returns beta and Wald standard errors
|
||||
from the penalised information matrix."""
|
||||
X = np.asarray(X, float)
|
||||
y = np.asarray(y, float)
|
||||
n, k = X.shape
|
||||
beta = np.zeros(k)
|
||||
P = np.eye(k) * lam
|
||||
P[0, 0] = 0
|
||||
for _ in range(iters):
|
||||
eta = X @ beta
|
||||
mu = 1 / (1 + np.exp(-eta))
|
||||
W = mu * (1 - mu)
|
||||
grad = X.T @ (y - mu) - P @ beta
|
||||
H = (X * W[:, None]).T @ X + P
|
||||
step = np.linalg.solve(H, grad)
|
||||
beta += step
|
||||
if np.max(np.abs(step)) < tol:
|
||||
break
|
||||
mu = 1 / (1 + np.exp(-(X @ beta)))
|
||||
H = (X * (mu * (1 - mu))[:, None]).T @ X + P
|
||||
se = np.sqrt(np.diag(np.linalg.inv(H)))
|
||||
return beta, se
|
||||
|
||||
|
||||
def p_from_z(z):
|
||||
return 2 * (1 - norm.cdf(abs(z)))
|
||||
|
||||
|
||||
def mantel_haenszel(tables):
|
||||
"""Mantel-Haenszel common OR over 2x2 tables [a, b, c, d], Robins-Breslow-Greenland 95% CI,
|
||||
and the CMH chi-square p-value (1 df, continuity-corrected)."""
|
||||
R = S = PR = PSQR = QS = sA = sE = sV = 0.0
|
||||
for a, b, c, d in tables:
|
||||
n = a + b + c + d
|
||||
if n < 2:
|
||||
continue
|
||||
r, s_ = a * d / n, b * c / n
|
||||
P, Q = (a + d) / n, (b + c) / n
|
||||
R += r; S += s_; PR += P * r; PSQR += P * s_ + Q * r; QS += Q * s_
|
||||
m1, m0, n1, n0 = a + b, c + d, a + c, b + d
|
||||
sA += a; sE += m1 * n1 / n; sV += m1 * m0 * n1 * n0 / (n * n * (n - 1))
|
||||
if not R or not S:
|
||||
return None
|
||||
or_ = R / S
|
||||
se = math.sqrt(PR / (2 * R * R) + PSQR / (2 * R * S) + QS / (2 * S * S))
|
||||
chi = max(0.0, abs(sA - sE) - 0.5) ** 2 / sV if sV > 0 else 0.0
|
||||
return or_, math.exp(math.log(or_) - Z95 * se), math.exp(math.log(or_) + Z95 * se), p_from_z(math.sqrt(chi))
|
||||
Reference in New Issue
Block a user