Threshing Atlas: monorepo, route graph, bot importer, daily source checks
- apps/atlas: the web page (valley map, route flow, Dragon Register) - packages/core: coding rules and statistics (engine.js) and the shared route graph (graph.js): scene graph, decision-variable graph, bot-to-form crosswalk, Beta credible intervals - packages/ingest: DragonKind bot importer to anonymised trajectories (no prose, no account identifiers); data/bot holds Ken's sample - research/python: independent reference implementation + update_sources.py - tests: JS/Python parity checks and graph/importer unit tests - CI, GitHub Pages deploy, daily check of the community sheets and route PDF - Email addresses in free-text form answers are redacted; the raw spreadsheet is not committed - MIT code licence, CC BY 4.0 data licence Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
0052d1a5fd
commit
c3d067b88c
@@ -0,0 +1,134 @@
|
||||
"""
|
||||
analyze.py — Python reference implementation of the statistics shown on the route-map page.
|
||||
|
||||
python analyze.py ../../data/derived/runs_clean.csv ../../tests/out/results_py.json
|
||||
|
||||
It mirrors analyse() in packages/core/src/engine.js (same denominators, same tests, same model); tests/compare_results.py
|
||||
checks that both produce the same numbers. Edit either, then re-run the comparison.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from stats_core import Z95, bh, logistic_ridge, odds_ratio, p_from_z, wilson
|
||||
|
||||
COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
|
||||
FACTORS = ["company", "direction", "first_sighting", "scream", "resolution", "combat_first",
|
||||
"post_path", "hour", "wing", "squad", "attempt_group", "source", "goal"]
|
||||
|
||||
|
||||
def W(k, n):
|
||||
p, lo, hi = wilson(k, n)
|
||||
return {"k": int(k), "n": int(n), "p": p, "lo": lo, "hi": hi}
|
||||
|
||||
|
||||
def outcomes():
|
||||
o = {
|
||||
"blue_meadow": (lambda d: d.branch == "Passed the Red: meadow", lambda d: d.saw_blue_meadow),
|
||||
"shadow": (lambda d: (d.hour == "Tree") & d.branch.notna(), lambda d: d.saw_shadow),
|
||||
"bond_blue_meadow": (lambda d: (d.branch == "Passed the Red: meadow") & d.enc_Blue.isin(["Bonded!", "Incinerated"]),
|
||||
lambda d: d.enc_Blue == "Bonded!"),
|
||||
"reach_hour": (lambda d: pd.Series(True, index=d.index), lambda d: d.hour.isin(["Tree", "Valley"])),
|
||||
}
|
||||
for c in COLORS:
|
||||
o["bonded_" + c] = (lambda d: d.hour.isin(["Tree", "Valley"]),
|
||||
(lambda c: lambda d: (d.outcome == "Bonded") & (d.outcome_color == c))(c))
|
||||
return o
|
||||
|
||||
|
||||
OUT = outcomes()
|
||||
|
||||
|
||||
def factor_table(d, key, min_n=5):
|
||||
den, ev = OUT[key]
|
||||
D = d[den(d)].copy()
|
||||
D["_e"] = ev(D).astype(bool)
|
||||
rows = []
|
||||
for f in FACTORS:
|
||||
sub = D[D[f].notna()]
|
||||
nf, ef = len(sub), int(sub._e.sum())
|
||||
for lv in sorted(sub[f].astype(str).unique()):
|
||||
g = sub[sub[f].astype(str) == lv]
|
||||
k, n = int(g._e.sum()), len(g)
|
||||
row = {"factor": f, "level": lv, **W(k, n)}
|
||||
rk, rn = ef - k, nf - n
|
||||
if n >= min_n and rn >= min_n and 0 < ef < nf:
|
||||
or_, lo, hi, p, corr = odds_ratio(k, n - k, rk, rn - rk)
|
||||
row.update({"or": or_, "or_lo": lo, "or_hi": hi, "fisher_p": p, "corrected": corr})
|
||||
rows.append(row)
|
||||
tested = [r for r in rows if "fisher_p" in r]
|
||||
for r, q in zip(tested, bh([r["fisher_p"] for r in tested])):
|
||||
r["q"] = q
|
||||
return {"outcome": key, "overall": W(int(D._e.sum()), len(D)), "rows": rows}
|
||||
|
||||
|
||||
def model(d, key, factors, lam=1.0):
|
||||
den, ev = OUT[key]
|
||||
D = d[den(d)].copy()
|
||||
y = ev(D).astype(int).values
|
||||
F = pd.DataFrame({name: D.apply(fn, axis=1) for name, fn in factors.items()}, index=D.index)
|
||||
keep = F.notna().all(axis=1).values
|
||||
F, y = F[keep], y[keep]
|
||||
cols, names = [np.ones(len(F))], []
|
||||
refs = {}
|
||||
for f in factors:
|
||||
counts = F[f].value_counts()
|
||||
levels = sorted(counts.index, key=lambda lv: (-counts[lv], lv))
|
||||
refs[f] = levels[0]
|
||||
for lv in levels[1:]:
|
||||
cols.append((F[f] == lv).astype(float).values)
|
||||
names.append((f, lv, int(counts[lv])))
|
||||
X = np.column_stack(cols)
|
||||
b, se = logistic_ridge(X, y, lam)
|
||||
terms = [{"term": f"{f} = {lv}", "factor": f, "level": lv, "n": n, "or": float(np.exp(b[i + 1])),
|
||||
"lo": float(np.exp(b[i + 1] - Z95 * se[i + 1])), "hi": float(np.exp(b[i + 1] + Z95 * se[i + 1])),
|
||||
"p": float(p_from_z(b[i + 1] / se[i + 1]))} for i, (f, lv, n) in enumerate(names)]
|
||||
return {"outcome": key, "n": int(len(y)), "events": int(y.sum()), "refs": refs, "lambda": lam, "terms": terms}
|
||||
|
||||
|
||||
def goal2(r):
|
||||
if r.goal in ("Blue", "Black"):
|
||||
return r.goal
|
||||
return "Blank/not asked" if r.goal == "(not asked/blank)" else "Other colour/any"
|
||||
|
||||
|
||||
def src2(r):
|
||||
return "Form submission" if r.source == "Form" else "Bulk log (import)"
|
||||
|
||||
|
||||
def first2(r):
|
||||
return r.first_sighting if r.first_sighting in ("Red", "Brown", "Orange", "Green") else "Unsure/unknown"
|
||||
|
||||
|
||||
def period(r):
|
||||
if not isinstance(r.date, str):
|
||||
return "No date (import)"
|
||||
return "Oct 4–10" if r.date < "2026-10-11" else "Oct 11 onward"
|
||||
|
||||
|
||||
def analyse(d):
|
||||
res = {"n": len(d), "factors": {k: factor_table(d, k) for k in OUT}}
|
||||
res["models"] = {
|
||||
"blue_meadow": model(d, "blue_meadow", {
|
||||
"source": src2, "goal": goal2, "direction": lambda r: r.direction, "first_sighting": first2,
|
||||
"resolution": lambda r: r.resolution, "company": lambda r: None if r.company == "Unknown" else r.company}),
|
||||
"shadow": model(d, "shadow", {
|
||||
"source": src2, "goal": goal2, "attempt": lambda r: r.attempt_group,
|
||||
"direction": lambda r: r.direction}),
|
||||
"bond_blue_meadow": model(d, "bond_blue_meadow", {
|
||||
"source": src2, "goal": goal2, "direction": lambda r: r.direction, "resolution": lambda r: r.resolution}),
|
||||
}
|
||||
return res
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/derived/runs_clean.csv"
|
||||
out = sys.argv[2] if len(sys.argv) > 2 else "../../tests/out/results_py.json"
|
||||
d = pd.read_csv(src, keep_default_na=False, na_values=[""])
|
||||
for c in ("saw_blue_meadow", "saw_shadow", "rescued"):
|
||||
d[c] = d[c].astype(str) == "True"
|
||||
res = analyse(d)
|
||||
json.dump(res, open(out, "w"), indent=1, default=float)
|
||||
print("wrote", out)
|
||||
@@ -0,0 +1,301 @@
|
||||
"""
|
||||
clean.py — turn the raw "Threshing Routes Responses" Google Form export into one
|
||||
coded row per threshing run.
|
||||
|
||||
Reference implementation in Python. The web page (packages/core/src/engine.js) implements the
|
||||
same rules in JavaScript; tests/compare.py checks that both give identical output.
|
||||
|
||||
Usage:
|
||||
python clean.py ../../data/raw/form/Threshing_Routes_Responses.xlsx -> writes ../../data/derived/runs_clean.csv
|
||||
|
||||
Every coding rule is listed in RULES below and in the "Methods" section of the page.
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
|
||||
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
|
||||
|
||||
def redact(v):
|
||||
"""Free-text answers sometimes contain a respondent's email address; never display or export it."""
|
||||
return v if v is None else EMAIL_RE.sub("[email removed]", str(v))
|
||||
|
||||
COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
|
||||
|
||||
# Columns are found by keywords in the question text, so the script keeps working
|
||||
# if the form owner reorders or adds questions.
|
||||
COLUMN_KEYS = {
|
||||
"timestamp": ["timestamp"],
|
||||
"wing_First": ["wing and section", "[first]"],
|
||||
"wing_Second": ["wing and section", "[second]"],
|
||||
"wing_Third": ["wing and section", "[third]"],
|
||||
"wing_Fourth": ["wing and section", "[fourth]"],
|
||||
"squad": ["what squad"],
|
||||
"attempt": ["attempt number"],
|
||||
"company": ["who accompanied"],
|
||||
"dir_East": ["initial direction", "[east"],
|
||||
"dir_West": ["initial direction", "[west"],
|
||||
"dir_Center": ["initial direction", "[forest center"],
|
||||
"cadet": ["outcome of your cadet encounter"],
|
||||
"post": ["path did you take after the bully"],
|
||||
"hour": ["after walking an hour"],
|
||||
"enc_Red": ["attempt to bond", "[red"],
|
||||
"enc_Green": ["attempt to bond", "[green"],
|
||||
"enc_Orange": ["attempt to bond", "[orange"],
|
||||
"enc_Brown": ["attempt to bond", "[brown"],
|
||||
"enc_Blue": ["attempt to bond", "[blue"],
|
||||
"enc_Black": ["attempt to bond", "[black"],
|
||||
"death_note": ["those who died"],
|
||||
"bond_note": ["those who bonded"],
|
||||
"goal": ["colour dragon are you trying"],
|
||||
"combat": ["combat choice"],
|
||||
}
|
||||
|
||||
CADET_MAP = {
|
||||
"Bullies > You dealt with them": ("Investigate & help", "Killed the bullies yourself"),
|
||||
"Bullies > Squadmate's revenge": ("Investigate & help", "Squadmate's revenge"),
|
||||
"Bullies > Left to the Dragons": ("Investigate & help", "Left bullies to the dragons"),
|
||||
"Bullies > Assessed the situation": ("Investigate & help", "Unknown"),
|
||||
"Injured Cadet > Tourniquet": ("Ignore the scream", "Tourniquet"),
|
||||
"Injured Cadet > Left them there": ("Ignore the scream", "Left injured cadet"),
|
||||
"Injured Cadet > Took their weapons": ("Ignore the scream", "Took their weapons"),
|
||||
"Avoided combat (Investigated scream > Left them to it)": ("Investigate & leave", "Left them to it"),
|
||||
"You died in combat": ("Investigate & help", "DIED in combat"),
|
||||
"N/A - You died with initial dragon sighting": ("DIED at first sighting", "DIED at first sighting"),
|
||||
}
|
||||
COMBAT_MAP = {
|
||||
"Bullies > Assessed the situation": "Assess the situation",
|
||||
"Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)",
|
||||
"Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)": "Put yourself in the middle (bigger cadet)",
|
||||
}
|
||||
HOUR_MAP = {
|
||||
"Continued to Valley Centre": "Valley",
|
||||
"Climbed the tallest tree": "Tree",
|
||||
"N/A - You're dead": "DIED before the hour",
|
||||
}
|
||||
# First-sighting pools per (company, direction) according to the community PDF map.
|
||||
# Used only to FLAG rows that report a dragon outside the documented pool.
|
||||
SIGHTING_POOL = {
|
||||
("Alone", "East"): {"Red", "Orange"},
|
||||
("Alone", "West"): {"Red", "Brown", "Green"},
|
||||
("Alone", "Center"): {"Red", "Brown"},
|
||||
("1 Squadmate", "East"): {"Green", "Orange", "Red"}, # 'leave squadmate' -> alone pools also possible
|
||||
("1 Squadmate", "West"): {"Red", "Brown", "Green"},
|
||||
("1 Squadmate", "Center"): {"Red", "Brown"},
|
||||
("2 Squadmates", "East"): {"Red", "Orange"},
|
||||
("2 Squadmates", "West"): {"Green", "Brown", "Red"},
|
||||
("2 Squadmates", "Center"): {"Red", "Brown"},
|
||||
}
|
||||
|
||||
|
||||
def find_columns(cols):
|
||||
out = {}
|
||||
low = {c: str(c).lower() for c in cols}
|
||||
for key, words in COLUMN_KEYS.items():
|
||||
hits = [c for c in cols if all(w in low[c] for w in words)]
|
||||
if hits:
|
||||
out[key] = hits[0]
|
||||
missing = [k for k in COLUMN_KEYS if k not in out]
|
||||
return out, missing
|
||||
|
||||
|
||||
def blank(v):
|
||||
if v is None or v is pd.NaT:
|
||||
return True
|
||||
if not isinstance(v, str):
|
||||
try:
|
||||
if pd.isna(v):
|
||||
return True
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return str(v).strip() in ("", "nan", "NaN", "NaT")
|
||||
|
||||
|
||||
def s(v):
|
||||
return None if blank(v) else str(v).strip()
|
||||
|
||||
|
||||
def parse_timestamp(v):
|
||||
"""Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'.
|
||||
Returns 'YYYY-MM-DD' or None."""
|
||||
if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v):
|
||||
return pd.Timestamp(v).strftime("%Y-%m-%d")
|
||||
if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date
|
||||
return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d")
|
||||
t = s(v)
|
||||
if not t:
|
||||
return None
|
||||
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
|
||||
m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY
|
||||
if m:
|
||||
return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
|
||||
return None
|
||||
|
||||
|
||||
def parse_attempt(v):
|
||||
v = s(v)
|
||||
if v is None:
|
||||
return None
|
||||
t = v.lower()
|
||||
if "lost count" in t:
|
||||
return None
|
||||
words = {"first": 1, "second": 2, "third": 3, "fourth": 4, "fifth": 5}
|
||||
for w, n in words.items():
|
||||
if t.startswith(w):
|
||||
return n
|
||||
m = re.search(r"\d+", t)
|
||||
return int(m.group()) if m else None
|
||||
|
||||
|
||||
def code_row(r, C):
|
||||
g = lambda k: s(r[C[k]]) if k in C else None
|
||||
flags = []
|
||||
ts = r[C["timestamp"]] if "timestamp" in C else None
|
||||
d = parse_timestamp(ts)
|
||||
if d:
|
||||
source, date = "Form", d
|
||||
else:
|
||||
source, date = (s(ts) or "Unknown import"), None
|
||||
|
||||
wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)]
|
||||
wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown")
|
||||
section = g("wing_" + wings[0]) if len(wings) == 1 else None
|
||||
if len(wings) > 1:
|
||||
flags.append("multiple wings")
|
||||
|
||||
dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)]
|
||||
if len(dirs) == 1:
|
||||
direction, first = dirs[0], g("dir_" + dirs[0])
|
||||
else:
|
||||
direction, first = ("Multiple" if dirs else "Unknown"), None
|
||||
flags.append("direction " + ("multiple" if dirs else "missing"))
|
||||
|
||||
company = g("company") or "Unknown"
|
||||
if first and first != "Unsure" and (company, direction) in SIGHTING_POOL:
|
||||
if first not in SIGHTING_POOL[(company, direction)]:
|
||||
flags.append(f"first sighting {first} not in documented pool for {company}/{direction}")
|
||||
|
||||
cadet_raw = g("cadet")
|
||||
scream, resolution = CADET_MAP.get(cadet_raw, ("Unknown", "Unknown"))
|
||||
if cadet_raw and cadet_raw not in CADET_MAP:
|
||||
flags.append("unrecognised cadet answer")
|
||||
combat = COMBAT_MAP.get(g("combat")) if g("combat") else None
|
||||
|
||||
post_raw = g("post")
|
||||
if scream == "Investigate & help" and not resolution.startswith("DIED"):
|
||||
post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown"
|
||||
elif scream in ("Ignore the scream", "Investigate & leave"):
|
||||
post = "(no fork)"
|
||||
if post_raw in ("Forest", "West", "East"):
|
||||
post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch
|
||||
else:
|
||||
post = None
|
||||
|
||||
hour = HOUR_MAP.get(g("hour"))
|
||||
died_early = scream.startswith("DIED") or resolution.startswith("DIED")
|
||||
if died_early:
|
||||
hour = None
|
||||
elif hour is None:
|
||||
hour = "Unknown"
|
||||
|
||||
enc = {c: g("enc_" + c) for c in COLORS}
|
||||
bonded = [c for c in COLORS if enc[c] == "Bonded!"]
|
||||
burned = [c for c in COLORS if enc[c] == "Incinerated"]
|
||||
|
||||
# Which encounter branch did the run take after the hour?
|
||||
branch = None
|
||||
if hour == "Tree":
|
||||
if enc["Blue"] or enc["Black"]:
|
||||
branch = "Shadow (rare)"
|
||||
elif enc["Green"]:
|
||||
branch = "Descend slowly: two Greens"
|
||||
elif enc["Red"]:
|
||||
branch = "Descend quickly: Red"
|
||||
elif enc["Orange"] or enc["Brown"]:
|
||||
branch = "Descend slowly: two Greens"
|
||||
flags.append("tree run with Orange/Brown but no Green recorded")
|
||||
elif hour == "Valley":
|
||||
if enc["Green"]:
|
||||
flags.append("Green recorded on a Valley run (no Greens in the valley)")
|
||||
red_passed = enc["Red"] in ("Changed my mind",)
|
||||
meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]]
|
||||
if meadow or red_passed:
|
||||
branch = "Passed the Red: meadow"
|
||||
elif enc["Red"]:
|
||||
branch = "Valley Red"
|
||||
|
||||
# Final outcome of the run
|
||||
outcome, outcome_color = None, None
|
||||
if died_early:
|
||||
outcome = "Died before the hour"
|
||||
elif len(bonded) == 1:
|
||||
outcome, outcome_color = "Bonded", bonded[0]
|
||||
elif len(bonded) > 1:
|
||||
outcome = "Ambiguous"
|
||||
flags.append("more than one dragon marked Bonded")
|
||||
elif burned:
|
||||
order = ["Red", "Green", "Orange", "Brown", "Blue", "Black"] if hour != "Valley" else \
|
||||
["Red", "Orange", "Brown", "Blue", "Green", "Black"]
|
||||
last = [c for c in order if c in burned][-1]
|
||||
outcome, outcome_color = "Incinerated", last
|
||||
if len(burned) > 1:
|
||||
flags.append("more than one dragon marked Incinerated")
|
||||
elif hour == "DIED before the hour":
|
||||
outcome = "Died before the hour"
|
||||
else:
|
||||
outcome = "No outcome recorded"
|
||||
|
||||
# The dragon whose bonding encounter decided the run (attempted = Bonded or Incinerated)
|
||||
rescued = enc["Green"] == "Saved by another" and outcome == "Bonded" and outcome_color in ("Orange", "Brown")
|
||||
|
||||
attempt = parse_attempt(g("attempt"))
|
||||
return {
|
||||
"source": source, "date": date, "wing": wing, "section": section,
|
||||
"squad": g("squad") or "Unknown", "attempt": attempt,
|
||||
"attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if attempt and attempt > 1 else "Unknown/lost count"),
|
||||
"goal": g("goal") or "(not asked/blank)", "company": company,
|
||||
"direction": direction, "first_sighting": first or "Unknown",
|
||||
"scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)",
|
||||
"post_path": post, "hour": hour, "branch": branch,
|
||||
**{"enc_" + c: enc[c] for c in COLORS},
|
||||
"outcome": outcome, "outcome_color": outcome_color, "rescued": rescued,
|
||||
"saw_blue_meadow": bool(hour == "Valley" and enc["Blue"]),
|
||||
"saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"])),
|
||||
"data_flags": "; ".join(flags),
|
||||
"death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")),
|
||||
}
|
||||
|
||||
|
||||
def load(path):
|
||||
if str(path).lower().endswith(".json"): # the page's bundled export (data/public/form_responses.json)
|
||||
import json
|
||||
d = json.load(open(path, encoding="utf-8"))
|
||||
raw = pd.DataFrame(d["rows"], columns=d["headers"])
|
||||
elif str(path).lower().endswith(".csv"):
|
||||
raw = pd.read_csv(path)
|
||||
else:
|
||||
xl = pd.ExcelFile(path)
|
||||
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
|
||||
raw = pd.read_excel(xl, sheet)
|
||||
C, missing = find_columns(raw.columns)
|
||||
rows = [code_row(r, C) for _, r in raw.iterrows()]
|
||||
df = pd.DataFrame(rows)
|
||||
df.insert(0, "run_id", range(1, len(df) + 1))
|
||||
return df, missing
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/public/form_responses.json"
|
||||
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/derived/runs_clean.csv"
|
||||
df, missing = load(src)
|
||||
if missing:
|
||||
print("WARNING: columns not found:", missing)
|
||||
df.to_csv(out, index=False)
|
||||
print(f"{len(df)} runs -> {out}")
|
||||
print(df.outcome.value_counts().to_string())
|
||||
print("rows with flags:", (df["data_flags"] != "").sum())
|
||||
@@ -0,0 +1,49 @@
|
||||
"""export_raw.py — convert the Google Sheets .xlsx (or .csv) export into the raw JSON the web page loads.
|
||||
|
||||
python export_raw.py ../../data/raw/form/Threshing_Routes_Responses.xlsx ../../data/public/form_responses.json
|
||||
|
||||
The JSON is a list of objects keyed by the original question text (no recoding happens here, so the page
|
||||
applies exactly the same cleaning rules to it as it would to a freshly uploaded export).
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import re
|
||||
|
||||
import pandas as pd
|
||||
|
||||
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
|
||||
|
||||
def load_raw(path):
|
||||
if str(path).lower().endswith(".csv"):
|
||||
return pd.read_csv(path)
|
||||
xl = pd.ExcelFile(path)
|
||||
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
|
||||
return pd.read_excel(xl, sheet)
|
||||
|
||||
|
||||
def cell(v):
|
||||
if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)):
|
||||
return None
|
||||
if isinstance(v, (datetime, pd.Timestamp)):
|
||||
return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S")
|
||||
if isinstance(v, float) and v.is_integer():
|
||||
return int(v)
|
||||
if isinstance(v, str):
|
||||
return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text
|
||||
return v
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/form/Threshing_Routes_Responses.xlsx"
|
||||
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/form_responses.json"
|
||||
raw = load_raw(src)
|
||||
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
|
||||
# keep every header present even if a row is blank there, so column detection sees all questions
|
||||
headers = [str(c) for c in raw.columns]
|
||||
with open(out, "w", encoding="utf-8") as f:
|
||||
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
|
||||
"headers": headers, "rows": rows}, f, ensure_ascii=False, separators=(",", ":"))
|
||||
print(f"{len(rows)} rows -> {out}")
|
||||
@@ -0,0 +1,43 @@
|
||||
"""export_register.py — convert the "Unofficial Dragon Register" .xlsx/.csv export into web/data/register.json.
|
||||
|
||||
python export_register.py ../../data/raw/register/Unofficial_Dragon_Register_Responses.xlsx ../../data/public/register.json
|
||||
|
||||
Raw rows are kept as-is, except that photo links and rider names are dropped. Dragons read from public dragonkind.com result pages
|
||||
linked in the Reddit megathreads are stored separately under "extra" with their source URL.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from export_raw import cell, load_raw
|
||||
|
||||
# Read from dragonkind.com/results/<id> pages linked in the Part 3 and Part 4 megathreads (2026-10-19).
|
||||
REDDIT_RESULTS = [
|
||||
("THRESH-VFRQVM5K3B0CP", "Thiduach", "Blue", "Swordtail", "Female", 51, "Gormfaileas"),
|
||||
("THRESH-3M4P9MNFQGBS8", "Caenuis", "Blue", "Swordtail", "Female", 38, "Gormfaileas"),
|
||||
("THRESH-5JQDJ21G1BYKR", "Oavuagh", "Green", "Morningstartail", "Female", 43, "Uaineloidsig"),
|
||||
("THRESH-5KRAAVFB3D1MG", "Geagios", "Blue", "Clubtail", "Male", 99, "Gormfaileas"),
|
||||
("THRESH-B5HSM00J4A2Z4", "Fiedheil", "Black", "Clubtail", "Male", 32, "Moerlaighen"),
|
||||
("THRESH-BMPM2GAPK58JT", "Teirfua", "Green", "Morningstartail", "Female", 68, "Eifyrielle"),
|
||||
("THRESH-MCQ8D6GAGF4K8", "Caollo", "Blue", "Morningstartail", "Female", 93, "Nyghmuiron"),
|
||||
("THRESH-Q0XDBFMNBEB86", "Aenminn", "Red", "Swordtail", "Male", 59, "Berradhyr"),
|
||||
("THRESH-W99DB29MG16QA", "Meinso", "Green", "Clubtail", "Female", 63, "Eifyrielle"),
|
||||
("THRESH-XCJCY7HW7QSSP", "Einmial", "Blue", "Clubtail", "Male", 58, "Gormfaileas"),
|
||||
("THRESH-XR6G68VPGT48M", "Recuagh", "Brown", "Morningstartail", "Female", 55, "Vormalgh"),
|
||||
("THRESH-YQC97TK34214E", "Dannidh", "Orange", "Clubtail", "Male", 61, "Cariaslog"),
|
||||
]
|
||||
|
||||
if __name__ == "__main__":
|
||||
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/register/Unofficial_Dragon_Register_Responses.xlsx"
|
||||
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/register.json"
|
||||
raw = load_raw(src)
|
||||
raw = raw[[c for c in raw.columns if "photo" not in str(c).lower() and "cadet name" not in str(c).lower()]]
|
||||
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
|
||||
extra = [{"source": "dragonkind.com result page linked on Reddit", "url": f"https://dragonkind.com/results/{i}",
|
||||
"name": n, "colour": c, "tail": t, "sex": s, "age": a, "lineage": l} for i, n, c, t, s, a, l in REDDIT_RESULTS]
|
||||
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
|
||||
"headers": [str(c) for c in raw.columns], "rows": rows, "extra": extra},
|
||||
open(out, "w", encoding="utf-8"), ensure_ascii=False, separators=(",", ":"))
|
||||
print(f"{len(rows)} register rows + {len(extra)} Reddit result pages -> {out}")
|
||||
@@ -0,0 +1,5 @@
|
||||
pandas>=2.0
|
||||
numpy>=1.24
|
||||
scipy>=1.10
|
||||
openpyxl>=3.1
|
||||
pypdf>=4.0
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Small, dependency-light statistics used by the analysis (mirrored in packages/core/src/engine.js)."""
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
from scipy.stats import fisher_exact, norm
|
||||
|
||||
Z95 = 1.959963984540054
|
||||
|
||||
|
||||
def wilson(k, n, z=Z95):
|
||||
"""Wilson score 95% interval for a proportion k/n. Returns (p, lo, hi); None when n == 0."""
|
||||
if n == 0:
|
||||
return (None, None, None)
|
||||
p = k / n
|
||||
d = 1 + z * z / n
|
||||
c = (p + z * z / (2 * n)) / d
|
||||
h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
|
||||
return (p, max(0.0, c - h), min(1.0, c + h))
|
||||
|
||||
|
||||
def odds_ratio(a, b, c, d):
|
||||
"""2x2 table [[a, b], [c, d]] = [[exposed&event, exposed&no-event], [unexposed&event, unexposed&no-event]].
|
||||
Returns OR, 95% CI (Woolf log method with Haldane-Anscombe +0.5 correction when any cell is 0),
|
||||
and the two-sided Fisher exact p-value (computed on the uncorrected table)."""
|
||||
corr = 0.5 if 0 in (a, b, c, d) else 0.0
|
||||
A, B, C, D = a + corr, b + corr, c + corr, d + corr
|
||||
or_ = (A * D) / (B * C)
|
||||
se = math.sqrt(1 / A + 1 / B + 1 / C + 1 / D)
|
||||
lo, hi = math.exp(math.log(or_) - Z95 * se), math.exp(math.log(or_) + Z95 * se)
|
||||
p = fisher_exact([[a, b], [c, d]], alternative="two-sided")[1]
|
||||
return or_, lo, hi, p, corr > 0
|
||||
|
||||
|
||||
def bh(pvals):
|
||||
"""Benjamini-Hochberg false-discovery-rate adjusted q-values."""
|
||||
m = len(pvals)
|
||||
if m == 0:
|
||||
return []
|
||||
order = sorted(range(m), key=lambda i: pvals[i])
|
||||
q = [0.0] * m
|
||||
prev = 1.0
|
||||
for rank in range(m, 0, -1):
|
||||
i = order[rank - 1]
|
||||
prev = min(prev, pvals[i] * m / rank)
|
||||
q[i] = prev
|
||||
return q
|
||||
|
||||
|
||||
def logistic_ridge(X, y, lam=1.0, iters=100, tol=1e-10):
|
||||
"""Logistic regression fitted by Newton-Raphson (IRLS) with a small L2 penalty `lam`
|
||||
on every coefficient except the intercept (column 0). The penalty keeps estimates finite
|
||||
when a category has 0 events (complete separation). Returns beta and Wald standard errors
|
||||
from the penalised information matrix."""
|
||||
X = np.asarray(X, float)
|
||||
y = np.asarray(y, float)
|
||||
n, k = X.shape
|
||||
beta = np.zeros(k)
|
||||
P = np.eye(k) * lam
|
||||
P[0, 0] = 0
|
||||
for _ in range(iters):
|
||||
eta = X @ beta
|
||||
mu = 1 / (1 + np.exp(-eta))
|
||||
W = mu * (1 - mu)
|
||||
grad = X.T @ (y - mu) - P @ beta
|
||||
H = (X * W[:, None]).T @ X + P
|
||||
step = np.linalg.solve(H, grad)
|
||||
beta += step
|
||||
if np.max(np.abs(step)) < tol:
|
||||
break
|
||||
mu = 1 / (1 + np.exp(-(X @ beta)))
|
||||
H = (X * (mu * (1 - mu))[:, None]).T @ X + P
|
||||
se = np.sqrt(np.diag(np.linalg.inv(H)))
|
||||
return beta, se
|
||||
|
||||
|
||||
def p_from_z(z):
|
||||
return 2 * (1 - norm.cdf(abs(z)))
|
||||
|
||||
|
||||
def mantel_haenszel(tables):
|
||||
"""Mantel-Haenszel common OR over 2x2 tables [a, b, c, d], Robins-Breslow-Greenland 95% CI,
|
||||
and the CMH chi-square p-value (1 df, continuity-corrected). Mirrors mantelHaenszel() in engine.js."""
|
||||
R = S = PR = PSQR = QS = sA = sE = sV = 0.0
|
||||
for a, b, c, d in tables:
|
||||
n = a + b + c + d
|
||||
if n < 2:
|
||||
continue
|
||||
r, s_ = a * d / n, b * c / n
|
||||
P, Q = (a + d) / n, (b + c) / n
|
||||
R += r; S += s_; PR += P * r; PSQR += P * s_ + Q * r; QS += Q * s_
|
||||
m1, m0, n1, n0 = a + b, c + d, a + c, b + d
|
||||
sA += a; sE += m1 * n1 / n; sV += m1 * m0 * n1 * n0 / (n * n * (n - 1))
|
||||
if not R or not S:
|
||||
return None
|
||||
or_ = R / S
|
||||
se = math.sqrt(PR / (2 * R * R) + PSQR / (2 * R * S) + QS / (2 * S * S))
|
||||
chi = max(0.0, abs(sA - sE) - 0.5) ** 2 / sV if sV > 0 else 0.0
|
||||
return or_, math.exp(math.log(or_) - Z95 * se), math.exp(math.log(or_) + Z95 * se), p_from_z(math.sqrt(chi))
|
||||
@@ -0,0 +1,182 @@
|
||||
"""update_sources.py: check the community spreadsheets and the route PDF for new information.
|
||||
|
||||
python research/python/update_sources.py [--report report.md] [--form FILE] [--register FILE] [--pdf FILE]
|
||||
|
||||
Run from the repo root. Without file arguments it downloads the current versions from the URLs in
|
||||
data/sources.json (the GitHub Action does this every day). With file arguments it uses local copies,
|
||||
which is handy for testing or when a download link changes.
|
||||
|
||||
What it does:
|
||||
* Threshing Routes form: re-exports data/public/form_responses.json and data/derived/runs_clean.csv
|
||||
when the responses changed, and reports how many rows were added or edited.
|
||||
* Dragon Register: re-exports data/public/register.json (rider names and photos dropped) when it changed.
|
||||
The raw register export is never written into the repo.
|
||||
* Route PDF: the PDF is not redistributed. Only a fingerprint is kept (data/derived/pdf_fingerprint.json):
|
||||
the file hash, per-page text hashes and per-line hashes. When it changes, the report lists the pages that
|
||||
changed and the lines that are new, so the coding rules and the scene crosswalk can be updated by hand.
|
||||
|
||||
Exit code is 0 either way; "changed" files are visible to git, and the report says what happened.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
||||
sys.path.insert(0, HERE)
|
||||
from export_raw import cell, load_raw # noqa: E402
|
||||
|
||||
P = lambda *a: os.path.join(ROOT, *a) # noqa: E731
|
||||
|
||||
|
||||
def sha(data, n=None):
|
||||
h = hashlib.sha256(data if isinstance(data, bytes) else data.encode("utf-8")).hexdigest()
|
||||
return h[:n] if n else h
|
||||
|
||||
|
||||
def download(url, dest):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "threshing-atlas-updater"})
|
||||
with urllib.request.urlopen(req, timeout=60) as r, open(dest, "wb") as f:
|
||||
data = r.read()
|
||||
f.write(data)
|
||||
head = data[:15].lower()
|
||||
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
|
||||
raise RuntimeError(f"{url} returned a web page, not a file (is the link still shared publicly?)")
|
||||
return dest
|
||||
|
||||
|
||||
def sheet_rows(path, drop=()):
|
||||
raw = load_raw(path)
|
||||
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
|
||||
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
|
||||
return [str(c) for c in raw.columns], rows
|
||||
|
||||
|
||||
def diff_rows(old, new):
|
||||
key = lambda r: json.dumps(r, sort_keys=True, ensure_ascii=False) # noqa: E731
|
||||
o, n = {key(r) for r in old}, {key(r) for r in new}
|
||||
return len(n - o), len(o - n)
|
||||
|
||||
|
||||
def update_form(src, report):
|
||||
out = P("data", "public", "form_responses.json")
|
||||
current = json.load(open(out, encoding="utf-8"))
|
||||
headers, rows = sheet_rows(src)
|
||||
added, removed = diff_rows(current["rows"], rows)
|
||||
if headers == current["headers"] and not added and not removed:
|
||||
report.append(f"- **Threshing Routes form:** no change ({len(rows)} responses).")
|
||||
return False
|
||||
subprocess.run([sys.executable, P("research", "python", "export_raw.py"), src, out], check=True, cwd=HERE)
|
||||
subprocess.run([sys.executable, P("research", "python", "clean.py"), out, P("data", "derived", "runs_clean.csv")], check=True, cwd=HERE)
|
||||
raw_copy = P("data", "raw", "form", "Threshing_Routes_Responses.xlsx")
|
||||
if os.path.abspath(src) != raw_copy:
|
||||
shutil.copyfile(src, raw_copy)
|
||||
new_q = [h for h in headers if h not in current["headers"]]
|
||||
gone_q = [h for h in current["headers"] if h not in headers]
|
||||
report.append(f"- **Threshing Routes form:** {len(current['rows'])} → {len(rows)} responses "
|
||||
f"({added} new or edited, {removed} removed or edited).")
|
||||
if new_q:
|
||||
report.append(f" - New questions (check `COLUMN_KEYS` in engine.js and clean.py): {', '.join(new_q)}")
|
||||
if gone_q:
|
||||
report.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
|
||||
return True
|
||||
|
||||
|
||||
def update_register(src, report):
|
||||
out = P("data", "public", "register.json")
|
||||
current = json.load(open(out, encoding="utf-8"))
|
||||
headers, rows = sheet_rows(src, drop=("photo", "cadet name"))
|
||||
added, removed = diff_rows(current["rows"], rows)
|
||||
if headers == current["headers"] and not added and not removed:
|
||||
report.append(f"- **Dragon Register:** no change ({len(rows)} dragons).")
|
||||
return False
|
||||
subprocess.run([sys.executable, P("research", "python", "export_register.py"), src, out], check=True, cwd=HERE)
|
||||
report.append(f"- **Dragon Register:** {len(current['rows'])} → {len(rows)} entries ({added} new or edited, {removed} removed or edited).")
|
||||
return True
|
||||
|
||||
|
||||
def pdf_lines(path):
|
||||
try:
|
||||
from pypdf import PdfReader
|
||||
except ImportError:
|
||||
sys.exit("pypdf is required: pip install -r research/python/requirements.txt")
|
||||
pages = []
|
||||
for page in PdfReader(path).pages:
|
||||
text = page.extract_text() or ""
|
||||
pages.append([" ".join(l.split()) for l in text.splitlines() if l.strip()])
|
||||
return pages
|
||||
|
||||
|
||||
def update_pdf(src, report):
|
||||
out = P("data", "derived", "pdf_fingerprint.json")
|
||||
old = json.load(open(out)) if os.path.exists(out) else None
|
||||
data = open(src, "rb").read()
|
||||
pages = pdf_lines(src)
|
||||
fp = {
|
||||
"sha256": sha(data),
|
||||
"bytes": len(data),
|
||||
"pages": len(pages),
|
||||
"page_hashes": [sha("\n".join(p), 16) for p in pages],
|
||||
"line_hashes": sorted({sha(l.lower(), 12) for p in pages for l in p}),
|
||||
"checked": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
|
||||
}
|
||||
if old and old["sha256"] == fp["sha256"]:
|
||||
report.append(f"- **Route PDF:** no change ({fp['pages']} pages).")
|
||||
return False
|
||||
if old and old.get("page_hashes") == fp["page_hashes"]:
|
||||
report.append("- **Route PDF:** the file changed but its text did not (layout or images only). Worth a look.")
|
||||
elif old:
|
||||
changed = [i + 1 for i, h in enumerate(fp["page_hashes"]) if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
|
||||
old_lines = set(old["line_hashes"])
|
||||
new_lines = [l for p in pages for l in p if sha(l.lower(), 12) not in old_lines]
|
||||
removed = len(old_lines - set(fp["line_hashes"]))
|
||||
report.append(f"- **Route PDF:** changed. {old['pages']} → {fp['pages']} pages; pages with new text: "
|
||||
f"{', '.join(map(str, changed)) or 'none'}; {len(new_lines)} new lines, {removed} lines gone.")
|
||||
if new_lines:
|
||||
report.append(" - New lines (update the coding rules or the scene crosswalk if they add options):")
|
||||
report.extend(f" - {l}" for l in new_lines[:60])
|
||||
if len(new_lines) > 60:
|
||||
report.append(f" - … and {len(new_lines) - 60} more")
|
||||
else:
|
||||
report.append(f"- **Route PDF:** fingerprint recorded for the first time ({fp['pages']} pages).")
|
||||
json.dump(fp, open(out, "w"), indent=1)
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--report", default=None)
|
||||
ap.add_argument("--form")
|
||||
ap.add_argument("--register")
|
||||
ap.add_argument("--pdf")
|
||||
args = ap.parse_args()
|
||||
sources = json.load(open(P("data", "sources.json")))
|
||||
report, changed, failed = [], False, []
|
||||
tmp = tempfile.mkdtemp(prefix="sources-")
|
||||
steps = [("form", args.form, update_form), ("register", args.register, update_register), ("pdf", args.pdf, update_pdf)]
|
||||
for key, local, fn in steps:
|
||||
try:
|
||||
path = local or download(sources[key]["download"], os.path.join(tmp, sources[key]["filename"]))
|
||||
changed |= fn(os.path.abspath(path), report)
|
||||
except Exception as e: # keep going: one broken link should not block the others
|
||||
failed.append(key)
|
||||
report.append(f"- **{sources[key]['name']}:** could not check ({e}).")
|
||||
head = f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"
|
||||
text = "\n".join([f"## {head}", ""] + report) + "\n"
|
||||
print(text)
|
||||
if args.report:
|
||||
open(args.report, "w", encoding="utf-8").write(text)
|
||||
if os.environ.get("GITHUB_OUTPUT"):
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
||||
f.write(f"changed={'true' if changed else 'false'}\nfailed={','.join(failed)}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user