Threshing Atlas: monorepo, route graph, bot importer, daily source checks

- apps/atlas: the web page (valley map, route flow, Dragon Register)
- packages/core: coding rules and statistics (engine.js) and the shared route
  graph (graph.js): scene graph, decision-variable graph, bot-to-form crosswalk,
  Beta credible intervals
- packages/ingest: DragonKind bot importer to anonymised trajectories (no prose,
  no account identifiers); data/bot holds Ken's sample
- research/python: independent reference implementation + update_sources.py
- tests: JS/Python parity checks and graph/importer unit tests
- CI, GitHub Pages deploy, daily check of the community sheets and route PDF
- Email addresses in free-text form answers are redacted; the raw spreadsheet
  is not committed
- MIT code licence, CC BY 4.0 data licence

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-01 21:54:54 +00:00
co-authored by Claude Opus 5.5
parent 0052d1a5fd
commit c3d067b88c
48 changed files with 8334 additions and 0 deletions
+134
View File
@@ -0,0 +1,134 @@
"""
analyze.py — Python reference implementation of the statistics shown on the route-map page.
python analyze.py ../../data/derived/runs_clean.csv ../../tests/out/results_py.json
It mirrors analyse() in packages/core/src/engine.js (same denominators, same tests, same model); tests/compare_results.py
checks that both produce the same numbers. Edit either, then re-run the comparison.
"""
import json
import sys
import numpy as np
import pandas as pd
from stats_core import Z95, bh, logistic_ridge, odds_ratio, p_from_z, wilson
COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
FACTORS = ["company", "direction", "first_sighting", "scream", "resolution", "combat_first",
"post_path", "hour", "wing", "squad", "attempt_group", "source", "goal"]
def W(k, n):
p, lo, hi = wilson(k, n)
return {"k": int(k), "n": int(n), "p": p, "lo": lo, "hi": hi}
def outcomes():
o = {
"blue_meadow": (lambda d: d.branch == "Passed the Red: meadow", lambda d: d.saw_blue_meadow),
"shadow": (lambda d: (d.hour == "Tree") & d.branch.notna(), lambda d: d.saw_shadow),
"bond_blue_meadow": (lambda d: (d.branch == "Passed the Red: meadow") & d.enc_Blue.isin(["Bonded!", "Incinerated"]),
lambda d: d.enc_Blue == "Bonded!"),
"reach_hour": (lambda d: pd.Series(True, index=d.index), lambda d: d.hour.isin(["Tree", "Valley"])),
}
for c in COLORS:
o["bonded_" + c] = (lambda d: d.hour.isin(["Tree", "Valley"]),
(lambda c: lambda d: (d.outcome == "Bonded") & (d.outcome_color == c))(c))
return o
OUT = outcomes()
def factor_table(d, key, min_n=5):
den, ev = OUT[key]
D = d[den(d)].copy()
D["_e"] = ev(D).astype(bool)
rows = []
for f in FACTORS:
sub = D[D[f].notna()]
nf, ef = len(sub), int(sub._e.sum())
for lv in sorted(sub[f].astype(str).unique()):
g = sub[sub[f].astype(str) == lv]
k, n = int(g._e.sum()), len(g)
row = {"factor": f, "level": lv, **W(k, n)}
rk, rn = ef - k, nf - n
if n >= min_n and rn >= min_n and 0 < ef < nf:
or_, lo, hi, p, corr = odds_ratio(k, n - k, rk, rn - rk)
row.update({"or": or_, "or_lo": lo, "or_hi": hi, "fisher_p": p, "corrected": corr})
rows.append(row)
tested = [r for r in rows if "fisher_p" in r]
for r, q in zip(tested, bh([r["fisher_p"] for r in tested])):
r["q"] = q
return {"outcome": key, "overall": W(int(D._e.sum()), len(D)), "rows": rows}
def model(d, key, factors, lam=1.0):
den, ev = OUT[key]
D = d[den(d)].copy()
y = ev(D).astype(int).values
F = pd.DataFrame({name: D.apply(fn, axis=1) for name, fn in factors.items()}, index=D.index)
keep = F.notna().all(axis=1).values
F, y = F[keep], y[keep]
cols, names = [np.ones(len(F))], []
refs = {}
for f in factors:
counts = F[f].value_counts()
levels = sorted(counts.index, key=lambda lv: (-counts[lv], lv))
refs[f] = levels[0]
for lv in levels[1:]:
cols.append((F[f] == lv).astype(float).values)
names.append((f, lv, int(counts[lv])))
X = np.column_stack(cols)
b, se = logistic_ridge(X, y, lam)
terms = [{"term": f"{f} = {lv}", "factor": f, "level": lv, "n": n, "or": float(np.exp(b[i + 1])),
"lo": float(np.exp(b[i + 1] - Z95 * se[i + 1])), "hi": float(np.exp(b[i + 1] + Z95 * se[i + 1])),
"p": float(p_from_z(b[i + 1] / se[i + 1]))} for i, (f, lv, n) in enumerate(names)]
return {"outcome": key, "n": int(len(y)), "events": int(y.sum()), "refs": refs, "lambda": lam, "terms": terms}
def goal2(r):
if r.goal in ("Blue", "Black"):
return r.goal
return "Blank/not asked" if r.goal == "(not asked/blank)" else "Other colour/any"
def src2(r):
return "Form submission" if r.source == "Form" else "Bulk log (import)"
def first2(r):
return r.first_sighting if r.first_sighting in ("Red", "Brown", "Orange", "Green") else "Unsure/unknown"
def period(r):
if not isinstance(r.date, str):
return "No date (import)"
return "Oct 4–10" if r.date < "2026-10-11" else "Oct 11 onward"
def analyse(d):
res = {"n": len(d), "factors": {k: factor_table(d, k) for k in OUT}}
res["models"] = {
"blue_meadow": model(d, "blue_meadow", {
"source": src2, "goal": goal2, "direction": lambda r: r.direction, "first_sighting": first2,
"resolution": lambda r: r.resolution, "company": lambda r: None if r.company == "Unknown" else r.company}),
"shadow": model(d, "shadow", {
"source": src2, "goal": goal2, "attempt": lambda r: r.attempt_group,
"direction": lambda r: r.direction}),
"bond_blue_meadow": model(d, "bond_blue_meadow", {
"source": src2, "goal": goal2, "direction": lambda r: r.direction, "resolution": lambda r: r.resolution}),
}
return res
if __name__ == "__main__":
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/derived/runs_clean.csv"
out = sys.argv[2] if len(sys.argv) > 2 else "../../tests/out/results_py.json"
d = pd.read_csv(src, keep_default_na=False, na_values=[""])
for c in ("saw_blue_meadow", "saw_shadow", "rescued"):
d[c] = d[c].astype(str) == "True"
res = analyse(d)
json.dump(res, open(out, "w"), indent=1, default=float)
print("wrote", out)
+301
View File
@@ -0,0 +1,301 @@
"""
clean.py — turn the raw "Threshing Routes Responses" Google Form export into one
coded row per threshing run.
Reference implementation in Python. The web page (packages/core/src/engine.js) implements the
same rules in JavaScript; tests/compare.py checks that both give identical output.
Usage:
python clean.py ../../data/raw/form/Threshing_Routes_Responses.xlsx -> writes ../../data/derived/runs_clean.csv
Every coding rule is listed in RULES below and in the "Methods" section of the page.
"""
import re
import sys
from datetime import datetime
import pandas as pd
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
def redact(v):
"""Free-text answers sometimes contain a respondent's email address; never display or export it."""
return v if v is None else EMAIL_RE.sub("[email removed]", str(v))
COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
# Columns are found by keywords in the question text, so the script keeps working
# if the form owner reorders or adds questions.
COLUMN_KEYS = {
"timestamp": ["timestamp"],
"wing_First": ["wing and section", "[first]"],
"wing_Second": ["wing and section", "[second]"],
"wing_Third": ["wing and section", "[third]"],
"wing_Fourth": ["wing and section", "[fourth]"],
"squad": ["what squad"],
"attempt": ["attempt number"],
"company": ["who accompanied"],
"dir_East": ["initial direction", "[east"],
"dir_West": ["initial direction", "[west"],
"dir_Center": ["initial direction", "[forest center"],
"cadet": ["outcome of your cadet encounter"],
"post": ["path did you take after the bully"],
"hour": ["after walking an hour"],
"enc_Red": ["attempt to bond", "[red"],
"enc_Green": ["attempt to bond", "[green"],
"enc_Orange": ["attempt to bond", "[orange"],
"enc_Brown": ["attempt to bond", "[brown"],
"enc_Blue": ["attempt to bond", "[blue"],
"enc_Black": ["attempt to bond", "[black"],
"death_note": ["those who died"],
"bond_note": ["those who bonded"],
"goal": ["colour dragon are you trying"],
"combat": ["combat choice"],
}
CADET_MAP = {
"Bullies > You dealt with them": ("Investigate & help", "Killed the bullies yourself"),
"Bullies > Squadmate's revenge": ("Investigate & help", "Squadmate's revenge"),
"Bullies > Left to the Dragons": ("Investigate & help", "Left bullies to the dragons"),
"Bullies > Assessed the situation": ("Investigate & help", "Unknown"),
"Injured Cadet > Tourniquet": ("Ignore the scream", "Tourniquet"),
"Injured Cadet > Left them there": ("Ignore the scream", "Left injured cadet"),
"Injured Cadet > Took their weapons": ("Ignore the scream", "Took their weapons"),
"Avoided combat (Investigated scream > Left them to it)": ("Investigate & leave", "Left them to it"),
"You died in combat": ("Investigate & help", "DIED in combat"),
"N/A - You died with initial dragon sighting": ("DIED at first sighting", "DIED at first sighting"),
}
COMBAT_MAP = {
"Bullies > Assessed the situation": "Assess the situation",
"Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)",
"Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)": "Put yourself in the middle (bigger cadet)",
}
HOUR_MAP = {
"Continued to Valley Centre": "Valley",
"Climbed the tallest tree": "Tree",
"N/A - You're dead": "DIED before the hour",
}
# First-sighting pools per (company, direction) according to the community PDF map.
# Used only to FLAG rows that report a dragon outside the documented pool.
SIGHTING_POOL = {
("Alone", "East"): {"Red", "Orange"},
("Alone", "West"): {"Red", "Brown", "Green"},
("Alone", "Center"): {"Red", "Brown"},
("1 Squadmate", "East"): {"Green", "Orange", "Red"}, # 'leave squadmate' -> alone pools also possible
("1 Squadmate", "West"): {"Red", "Brown", "Green"},
("1 Squadmate", "Center"): {"Red", "Brown"},
("2 Squadmates", "East"): {"Red", "Orange"},
("2 Squadmates", "West"): {"Green", "Brown", "Red"},
("2 Squadmates", "Center"): {"Red", "Brown"},
}
def find_columns(cols):
out = {}
low = {c: str(c).lower() for c in cols}
for key, words in COLUMN_KEYS.items():
hits = [c for c in cols if all(w in low[c] for w in words)]
if hits:
out[key] = hits[0]
missing = [k for k in COLUMN_KEYS if k not in out]
return out, missing
def blank(v):
if v is None or v is pd.NaT:
return True
if not isinstance(v, str):
try:
if pd.isna(v):
return True
except (TypeError, ValueError):
pass
return str(v).strip() in ("", "nan", "NaN", "NaT")
def s(v):
return None if blank(v) else str(v).strip()
def parse_timestamp(v):
"""Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'.
Returns 'YYYY-MM-DD' or None."""
if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v):
return pd.Timestamp(v).strftime("%Y-%m-%d")
if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date
return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d")
t = s(v)
if not t:
return None
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t)
if m:
return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY
if m:
return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
return None
def parse_attempt(v):
v = s(v)
if v is None:
return None
t = v.lower()
if "lost count" in t:
return None
words = {"first": 1, "second": 2, "third": 3, "fourth": 4, "fifth": 5}
for w, n in words.items():
if t.startswith(w):
return n
m = re.search(r"\d+", t)
return int(m.group()) if m else None
def code_row(r, C):
g = lambda k: s(r[C[k]]) if k in C else None
flags = []
ts = r[C["timestamp"]] if "timestamp" in C else None
d = parse_timestamp(ts)
if d:
source, date = "Form", d
else:
source, date = (s(ts) or "Unknown import"), None
wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)]
wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown")
section = g("wing_" + wings[0]) if len(wings) == 1 else None
if len(wings) > 1:
flags.append("multiple wings")
dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)]
if len(dirs) == 1:
direction, first = dirs[0], g("dir_" + dirs[0])
else:
direction, first = ("Multiple" if dirs else "Unknown"), None
flags.append("direction " + ("multiple" if dirs else "missing"))
company = g("company") or "Unknown"
if first and first != "Unsure" and (company, direction) in SIGHTING_POOL:
if first not in SIGHTING_POOL[(company, direction)]:
flags.append(f"first sighting {first} not in documented pool for {company}/{direction}")
cadet_raw = g("cadet")
scream, resolution = CADET_MAP.get(cadet_raw, ("Unknown", "Unknown"))
if cadet_raw and cadet_raw not in CADET_MAP:
flags.append("unrecognised cadet answer")
combat = COMBAT_MAP.get(g("combat")) if g("combat") else None
post_raw = g("post")
if scream == "Investigate & help" and not resolution.startswith("DIED"):
post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown"
elif scream in ("Ignore the scream", "Investigate & leave"):
post = "(no fork)"
if post_raw in ("Forest", "West", "East"):
post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch
else:
post = None
hour = HOUR_MAP.get(g("hour"))
died_early = scream.startswith("DIED") or resolution.startswith("DIED")
if died_early:
hour = None
elif hour is None:
hour = "Unknown"
enc = {c: g("enc_" + c) for c in COLORS}
bonded = [c for c in COLORS if enc[c] == "Bonded!"]
burned = [c for c in COLORS if enc[c] == "Incinerated"]
# Which encounter branch did the run take after the hour?
branch = None
if hour == "Tree":
if enc["Blue"] or enc["Black"]:
branch = "Shadow (rare)"
elif enc["Green"]:
branch = "Descend slowly: two Greens"
elif enc["Red"]:
branch = "Descend quickly: Red"
elif enc["Orange"] or enc["Brown"]:
branch = "Descend slowly: two Greens"
flags.append("tree run with Orange/Brown but no Green recorded")
elif hour == "Valley":
if enc["Green"]:
flags.append("Green recorded on a Valley run (no Greens in the valley)")
red_passed = enc["Red"] in ("Changed my mind",)
meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]]
if meadow or red_passed:
branch = "Passed the Red: meadow"
elif enc["Red"]:
branch = "Valley Red"
# Final outcome of the run
outcome, outcome_color = None, None
if died_early:
outcome = "Died before the hour"
elif len(bonded) == 1:
outcome, outcome_color = "Bonded", bonded[0]
elif len(bonded) > 1:
outcome = "Ambiguous"
flags.append("more than one dragon marked Bonded")
elif burned:
order = ["Red", "Green", "Orange", "Brown", "Blue", "Black"] if hour != "Valley" else \
["Red", "Orange", "Brown", "Blue", "Green", "Black"]
last = [c for c in order if c in burned][-1]
outcome, outcome_color = "Incinerated", last
if len(burned) > 1:
flags.append("more than one dragon marked Incinerated")
elif hour == "DIED before the hour":
outcome = "Died before the hour"
else:
outcome = "No outcome recorded"
# The dragon whose bonding encounter decided the run (attempted = Bonded or Incinerated)
rescued = enc["Green"] == "Saved by another" and outcome == "Bonded" and outcome_color in ("Orange", "Brown")
attempt = parse_attempt(g("attempt"))
return {
"source": source, "date": date, "wing": wing, "section": section,
"squad": g("squad") or "Unknown", "attempt": attempt,
"attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if attempt and attempt > 1 else "Unknown/lost count"),
"goal": g("goal") or "(not asked/blank)", "company": company,
"direction": direction, "first_sighting": first or "Unknown",
"scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)",
"post_path": post, "hour": hour, "branch": branch,
**{"enc_" + c: enc[c] for c in COLORS},
"outcome": outcome, "outcome_color": outcome_color, "rescued": rescued,
"saw_blue_meadow": bool(hour == "Valley" and enc["Blue"]),
"saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"])),
"data_flags": "; ".join(flags),
"death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")),
}
def load(path):
if str(path).lower().endswith(".json"): # the page's bundled export (data/public/form_responses.json)
import json
d = json.load(open(path, encoding="utf-8"))
raw = pd.DataFrame(d["rows"], columns=d["headers"])
elif str(path).lower().endswith(".csv"):
raw = pd.read_csv(path)
else:
xl = pd.ExcelFile(path)
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
raw = pd.read_excel(xl, sheet)
C, missing = find_columns(raw.columns)
rows = [code_row(r, C) for _, r in raw.iterrows()]
df = pd.DataFrame(rows)
df.insert(0, "run_id", range(1, len(df) + 1))
return df, missing
if __name__ == "__main__":
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/public/form_responses.json"
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/derived/runs_clean.csv"
df, missing = load(src)
if missing:
print("WARNING: columns not found:", missing)
df.to_csv(out, index=False)
print(f"{len(df)} runs -> {out}")
print(df.outcome.value_counts().to_string())
print("rows with flags:", (df["data_flags"] != "").sum())
+49
View File
@@ -0,0 +1,49 @@
"""export_raw.py — convert the Google Sheets .xlsx (or .csv) export into the raw JSON the web page loads.
python export_raw.py ../../data/raw/form/Threshing_Routes_Responses.xlsx ../../data/public/form_responses.json
The JSON is a list of objects keyed by the original question text (no recoding happens here, so the page
applies exactly the same cleaning rules to it as it would to a freshly uploaded export).
"""
import json
import sys
from datetime import datetime
import re
import pandas as pd
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
def load_raw(path):
if str(path).lower().endswith(".csv"):
return pd.read_csv(path)
xl = pd.ExcelFile(path)
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
return pd.read_excel(xl, sheet)
def cell(v):
if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)):
return None
if isinstance(v, (datetime, pd.Timestamp)):
return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S")
if isinstance(v, float) and v.is_integer():
return int(v)
if isinstance(v, str):
return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text
return v
if __name__ == "__main__":
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/form/Threshing_Routes_Responses.xlsx"
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/form_responses.json"
raw = load_raw(src)
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
# keep every header present even if a row is blank there, so column detection sees all questions
headers = [str(c) for c in raw.columns]
with open(out, "w", encoding="utf-8") as f:
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
"headers": headers, "rows": rows}, f, ensure_ascii=False, separators=(",", ":"))
print(f"{len(rows)} rows -> {out}")
+43
View File
@@ -0,0 +1,43 @@
"""export_register.py — convert the "Unofficial Dragon Register" .xlsx/.csv export into web/data/register.json.
python export_register.py ../../data/raw/register/Unofficial_Dragon_Register_Responses.xlsx ../../data/public/register.json
Raw rows are kept as-is, except that photo links and rider names are dropped. Dragons read from public dragonkind.com result pages
linked in the Reddit megathreads are stored separately under "extra" with their source URL.
"""
import json
import sys
from datetime import datetime
import pandas as pd
from export_raw import cell, load_raw
# Read from dragonkind.com/results/<id> pages linked in the Part 3 and Part 4 megathreads (2026-10-19).
REDDIT_RESULTS = [
("THRESH-VFRQVM5K3B0CP", "Thiduach", "Blue", "Swordtail", "Female", 51, "Gormfaileas"),
("THRESH-3M4P9MNFQGBS8", "Caenuis", "Blue", "Swordtail", "Female", 38, "Gormfaileas"),
("THRESH-5JQDJ21G1BYKR", "Oavuagh", "Green", "Morningstartail", "Female", 43, "Uaineloidsig"),
("THRESH-5KRAAVFB3D1MG", "Geagios", "Blue", "Clubtail", "Male", 99, "Gormfaileas"),
("THRESH-B5HSM00J4A2Z4", "Fiedheil", "Black", "Clubtail", "Male", 32, "Moerlaighen"),
("THRESH-BMPM2GAPK58JT", "Teirfua", "Green", "Morningstartail", "Female", 68, "Eifyrielle"),
("THRESH-MCQ8D6GAGF4K8", "Caollo", "Blue", "Morningstartail", "Female", 93, "Nyghmuiron"),
("THRESH-Q0XDBFMNBEB86", "Aenminn", "Red", "Swordtail", "Male", 59, "Berradhyr"),
("THRESH-W99DB29MG16QA", "Meinso", "Green", "Clubtail", "Female", 63, "Eifyrielle"),
("THRESH-XCJCY7HW7QSSP", "Einmial", "Blue", "Clubtail", "Male", 58, "Gormfaileas"),
("THRESH-XR6G68VPGT48M", "Recuagh", "Brown", "Morningstartail", "Female", 55, "Vormalgh"),
("THRESH-YQC97TK34214E", "Dannidh", "Orange", "Clubtail", "Male", 61, "Cariaslog"),
]
if __name__ == "__main__":
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/register/Unofficial_Dragon_Register_Responses.xlsx"
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/register.json"
raw = load_raw(src)
raw = raw[[c for c in raw.columns if "photo" not in str(c).lower() and "cadet name" not in str(c).lower()]]
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
extra = [{"source": "dragonkind.com result page linked on Reddit", "url": f"https://dragonkind.com/results/{i}",
"name": n, "colour": c, "tail": t, "sex": s, "age": a, "lineage": l} for i, n, c, t, s, a, l in REDDIT_RESULTS]
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
"headers": [str(c) for c in raw.columns], "rows": rows, "extra": extra},
open(out, "w", encoding="utf-8"), ensure_ascii=False, separators=(",", ":"))
print(f"{len(rows)} register rows + {len(extra)} Reddit result pages -> {out}")
+5
View File
@@ -0,0 +1,5 @@
pandas>=2.0
numpy>=1.24
scipy>=1.10
openpyxl>=3.1
pypdf>=4.0
+98
View File
@@ -0,0 +1,98 @@
"""Small, dependency-light statistics used by the analysis (mirrored in packages/core/src/engine.js)."""
import math
import numpy as np
from scipy.stats import fisher_exact, norm
Z95 = 1.959963984540054
def wilson(k, n, z=Z95):
"""Wilson score 95% interval for a proportion k/n. Returns (p, lo, hi); None when n == 0."""
if n == 0:
return (None, None, None)
p = k / n
d = 1 + z * z / n
c = (p + z * z / (2 * n)) / d
h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
return (p, max(0.0, c - h), min(1.0, c + h))
def odds_ratio(a, b, c, d):
"""2x2 table [[a, b], [c, d]] = [[exposed&event, exposed&no-event], [unexposed&event, unexposed&no-event]].
Returns OR, 95% CI (Woolf log method with Haldane-Anscombe +0.5 correction when any cell is 0),
and the two-sided Fisher exact p-value (computed on the uncorrected table)."""
corr = 0.5 if 0 in (a, b, c, d) else 0.0
A, B, C, D = a + corr, b + corr, c + corr, d + corr
or_ = (A * D) / (B * C)
se = math.sqrt(1 / A + 1 / B + 1 / C + 1 / D)
lo, hi = math.exp(math.log(or_) - Z95 * se), math.exp(math.log(or_) + Z95 * se)
p = fisher_exact([[a, b], [c, d]], alternative="two-sided")[1]
return or_, lo, hi, p, corr > 0
def bh(pvals):
"""Benjamini-Hochberg false-discovery-rate adjusted q-values."""
m = len(pvals)
if m == 0:
return []
order = sorted(range(m), key=lambda i: pvals[i])
q = [0.0] * m
prev = 1.0
for rank in range(m, 0, -1):
i = order[rank - 1]
prev = min(prev, pvals[i] * m / rank)
q[i] = prev
return q
def logistic_ridge(X, y, lam=1.0, iters=100, tol=1e-10):
"""Logistic regression fitted by Newton-Raphson (IRLS) with a small L2 penalty `lam`
on every coefficient except the intercept (column 0). The penalty keeps estimates finite
when a category has 0 events (complete separation). Returns beta and Wald standard errors
from the penalised information matrix."""
X = np.asarray(X, float)
y = np.asarray(y, float)
n, k = X.shape
beta = np.zeros(k)
P = np.eye(k) * lam
P[0, 0] = 0
for _ in range(iters):
eta = X @ beta
mu = 1 / (1 + np.exp(-eta))
W = mu * (1 - mu)
grad = X.T @ (y - mu) - P @ beta
H = (X * W[:, None]).T @ X + P
step = np.linalg.solve(H, grad)
beta += step
if np.max(np.abs(step)) < tol:
break
mu = 1 / (1 + np.exp(-(X @ beta)))
H = (X * (mu * (1 - mu))[:, None]).T @ X + P
se = np.sqrt(np.diag(np.linalg.inv(H)))
return beta, se
def p_from_z(z):
return 2 * (1 - norm.cdf(abs(z)))
def mantel_haenszel(tables):
"""Mantel-Haenszel common OR over 2x2 tables [a, b, c, d], Robins-Breslow-Greenland 95% CI,
and the CMH chi-square p-value (1 df, continuity-corrected). Mirrors mantelHaenszel() in engine.js."""
R = S = PR = PSQR = QS = sA = sE = sV = 0.0
for a, b, c, d in tables:
n = a + b + c + d
if n < 2:
continue
r, s_ = a * d / n, b * c / n
P, Q = (a + d) / n, (b + c) / n
R += r; S += s_; PR += P * r; PSQR += P * s_ + Q * r; QS += Q * s_
m1, m0, n1, n0 = a + b, c + d, a + c, b + d
sA += a; sE += m1 * n1 / n; sV += m1 * m0 * n1 * n0 / (n * n * (n - 1))
if not R or not S:
return None
or_ = R / S
se = math.sqrt(PR / (2 * R * R) + PSQR / (2 * R * S) + QS / (2 * S * S))
chi = max(0.0, abs(sA - sE) - 0.5) ** 2 / sV if sV > 0 else 0.0
return or_, math.exp(math.log(or_) - Z95 * se), math.exp(math.log(or_) + Z95 * se), p_from_z(math.sqrt(chi))
+182
View File
@@ -0,0 +1,182 @@
"""update_sources.py: check the community spreadsheets and the route PDF for new information.
python research/python/update_sources.py [--report report.md] [--form FILE] [--register FILE] [--pdf FILE]
Run from the repo root. Without file arguments it downloads the current versions from the URLs in
data/sources.json (the GitHub Action does this every day). With file arguments it uses local copies,
which is handy for testing or when a download link changes.
What it does:
* Threshing Routes form: re-exports data/public/form_responses.json and data/derived/runs_clean.csv
when the responses changed, and reports how many rows were added or edited.
* Dragon Register: re-exports data/public/register.json (rider names and photos dropped) when it changed.
The raw register export is never written into the repo.
* Route PDF: the PDF is not redistributed. Only a fingerprint is kept (data/derived/pdf_fingerprint.json):
the file hash, per-page text hashes and per-line hashes. When it changes, the report lists the pages that
changed and the lines that are new, so the coding rules and the scene crosswalk can be updated by hand.
Exit code is 0 either way; "changed" files are visible to git, and the report says what happened.
"""
import argparse
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tempfile
import urllib.request
from datetime import datetime, timezone
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
sys.path.insert(0, HERE)
from export_raw import cell, load_raw # noqa: E402
P = lambda *a: os.path.join(ROOT, *a) # noqa: E731
def sha(data, n=None):
h = hashlib.sha256(data if isinstance(data, bytes) else data.encode("utf-8")).hexdigest()
return h[:n] if n else h
def download(url, dest):
req = urllib.request.Request(url, headers={"User-Agent": "threshing-atlas-updater"})
with urllib.request.urlopen(req, timeout=60) as r, open(dest, "wb") as f:
data = r.read()
f.write(data)
head = data[:15].lower()
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
raise RuntimeError(f"{url} returned a web page, not a file (is the link still shared publicly?)")
return dest
def sheet_rows(path, drop=()):
raw = load_raw(path)
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
return [str(c) for c in raw.columns], rows
def diff_rows(old, new):
key = lambda r: json.dumps(r, sort_keys=True, ensure_ascii=False) # noqa: E731
o, n = {key(r) for r in old}, {key(r) for r in new}
return len(n - o), len(o - n)
def update_form(src, report):
out = P("data", "public", "form_responses.json")
current = json.load(open(out, encoding="utf-8"))
headers, rows = sheet_rows(src)
added, removed = diff_rows(current["rows"], rows)
if headers == current["headers"] and not added and not removed:
report.append(f"- **Threshing Routes form:** no change ({len(rows)} responses).")
return False
subprocess.run([sys.executable, P("research", "python", "export_raw.py"), src, out], check=True, cwd=HERE)
subprocess.run([sys.executable, P("research", "python", "clean.py"), out, P("data", "derived", "runs_clean.csv")], check=True, cwd=HERE)
raw_copy = P("data", "raw", "form", "Threshing_Routes_Responses.xlsx")
if os.path.abspath(src) != raw_copy:
shutil.copyfile(src, raw_copy)
new_q = [h for h in headers if h not in current["headers"]]
gone_q = [h for h in current["headers"] if h not in headers]
report.append(f"- **Threshing Routes form:** {len(current['rows'])} → {len(rows)} responses "
f"({added} new or edited, {removed} removed or edited).")
if new_q:
report.append(f" - New questions (check `COLUMN_KEYS` in engine.js and clean.py): {', '.join(new_q)}")
if gone_q:
report.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
return True
def update_register(src, report):
out = P("data", "public", "register.json")
current = json.load(open(out, encoding="utf-8"))
headers, rows = sheet_rows(src, drop=("photo", "cadet name"))
added, removed = diff_rows(current["rows"], rows)
if headers == current["headers"] and not added and not removed:
report.append(f"- **Dragon Register:** no change ({len(rows)} dragons).")
return False
subprocess.run([sys.executable, P("research", "python", "export_register.py"), src, out], check=True, cwd=HERE)
report.append(f"- **Dragon Register:** {len(current['rows'])} → {len(rows)} entries ({added} new or edited, {removed} removed or edited).")
return True
def pdf_lines(path):
try:
from pypdf import PdfReader
except ImportError:
sys.exit("pypdf is required: pip install -r research/python/requirements.txt")
pages = []
for page in PdfReader(path).pages:
text = page.extract_text() or ""
pages.append([" ".join(l.split()) for l in text.splitlines() if l.strip()])
return pages
def update_pdf(src, report):
out = P("data", "derived", "pdf_fingerprint.json")
old = json.load(open(out)) if os.path.exists(out) else None
data = open(src, "rb").read()
pages = pdf_lines(src)
fp = {
"sha256": sha(data),
"bytes": len(data),
"pages": len(pages),
"page_hashes": [sha("\n".join(p), 16) for p in pages],
"line_hashes": sorted({sha(l.lower(), 12) for p in pages for l in p}),
"checked": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
}
if old and old["sha256"] == fp["sha256"]:
report.append(f"- **Route PDF:** no change ({fp['pages']} pages).")
return False
if old and old.get("page_hashes") == fp["page_hashes"]:
report.append("- **Route PDF:** the file changed but its text did not (layout or images only). Worth a look.")
elif old:
changed = [i + 1 for i, h in enumerate(fp["page_hashes"]) if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
old_lines = set(old["line_hashes"])
new_lines = [l for p in pages for l in p if sha(l.lower(), 12) not in old_lines]
removed = len(old_lines - set(fp["line_hashes"]))
report.append(f"- **Route PDF:** changed. {old['pages']} → {fp['pages']} pages; pages with new text: "
f"{', '.join(map(str, changed)) or 'none'}; {len(new_lines)} new lines, {removed} lines gone.")
if new_lines:
report.append(" - New lines (update the coding rules or the scene crosswalk if they add options):")
report.extend(f" - {l}" for l in new_lines[:60])
if len(new_lines) > 60:
report.append(f" - … and {len(new_lines) - 60} more")
else:
report.append(f"- **Route PDF:** fingerprint recorded for the first time ({fp['pages']} pages).")
json.dump(fp, open(out, "w"), indent=1)
return True
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--report", default=None)
ap.add_argument("--form")
ap.add_argument("--register")
ap.add_argument("--pdf")
args = ap.parse_args()
sources = json.load(open(P("data", "sources.json")))
report, changed, failed = [], False, []
tmp = tempfile.mkdtemp(prefix="sources-")
steps = [("form", args.form, update_form), ("register", args.register, update_register), ("pdf", args.pdf, update_pdf)]
for key, local, fn in steps:
try:
path = local or download(sources[key]["download"], os.path.join(tmp, sources[key]["filename"]))
changed |= fn(os.path.abspath(path), report)
except Exception as e: # keep going: one broken link should not block the others
failed.append(key)
report.append(f"- **{sources[key]['name']}:** could not check ({e}).")
head = f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"
text = "\n".join([f"## {head}", ""] + report) + "\n"
print(text)
if args.report:
open(args.report, "w", encoding="utf-8").write(text)
if os.environ.get("GITHUB_OUTPUT"):
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
f.write(f"changed={'true' if changed else 'false'}\nfailed={','.join(failed)}\n")
if __name__ == "__main__":
main()