Threshing Atlas: monorepo, route graph, bot importer, daily source checks
- apps/atlas: the web page (valley map, route flow, Dragon Register) - packages/core: coding rules and statistics (engine.js) and the shared route graph (graph.js): scene graph, decision-variable graph, bot-to-form crosswalk, Beta credible intervals - packages/ingest: DragonKind bot importer to anonymised trajectories (no prose, no account identifiers); data/bot holds Ken's sample - research/python: independent reference implementation + update_sources.py - tests: JS/Python parity checks and graph/importer unit tests - CI, GitHub Pages deploy, daily check of the community sheets and route PDF - Email addresses in free-text form answers are redacted; the raw spreadsheet is not committed - MIT code licence, CC BY 4.0 data licence Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
0052d1a5fd
commit
c3d067b88c
@@ -0,0 +1,301 @@
|
||||
"""
|
||||
clean.py — turn the raw "Threshing Routes Responses" Google Form export into one
|
||||
coded row per threshing run.
|
||||
|
||||
Reference implementation in Python. The web page (packages/core/src/engine.js) implements the
|
||||
same rules in JavaScript; tests/compare.py checks that both give identical output.
|
||||
|
||||
Usage:
|
||||
python clean.py ../../data/raw/form/Threshing_Routes_Responses.xlsx -> writes ../../data/derived/runs_clean.csv
|
||||
|
||||
Every coding rule is listed in RULES below and in the "Methods" section of the page.
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
|
||||
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
|
||||
|
||||
def redact(v):
|
||||
"""Free-text answers sometimes contain a respondent's email address; never display or export it."""
|
||||
return v if v is None else EMAIL_RE.sub("[email removed]", str(v))
|
||||
|
||||
COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
|
||||
|
||||
# Columns are found by keywords in the question text, so the script keeps working
|
||||
# if the form owner reorders or adds questions.
|
||||
COLUMN_KEYS = {
|
||||
"timestamp": ["timestamp"],
|
||||
"wing_First": ["wing and section", "[first]"],
|
||||
"wing_Second": ["wing and section", "[second]"],
|
||||
"wing_Third": ["wing and section", "[third]"],
|
||||
"wing_Fourth": ["wing and section", "[fourth]"],
|
||||
"squad": ["what squad"],
|
||||
"attempt": ["attempt number"],
|
||||
"company": ["who accompanied"],
|
||||
"dir_East": ["initial direction", "[east"],
|
||||
"dir_West": ["initial direction", "[west"],
|
||||
"dir_Center": ["initial direction", "[forest center"],
|
||||
"cadet": ["outcome of your cadet encounter"],
|
||||
"post": ["path did you take after the bully"],
|
||||
"hour": ["after walking an hour"],
|
||||
"enc_Red": ["attempt to bond", "[red"],
|
||||
"enc_Green": ["attempt to bond", "[green"],
|
||||
"enc_Orange": ["attempt to bond", "[orange"],
|
||||
"enc_Brown": ["attempt to bond", "[brown"],
|
||||
"enc_Blue": ["attempt to bond", "[blue"],
|
||||
"enc_Black": ["attempt to bond", "[black"],
|
||||
"death_note": ["those who died"],
|
||||
"bond_note": ["those who bonded"],
|
||||
"goal": ["colour dragon are you trying"],
|
||||
"combat": ["combat choice"],
|
||||
}
|
||||
|
||||
CADET_MAP = {
|
||||
"Bullies > You dealt with them": ("Investigate & help", "Killed the bullies yourself"),
|
||||
"Bullies > Squadmate's revenge": ("Investigate & help", "Squadmate's revenge"),
|
||||
"Bullies > Left to the Dragons": ("Investigate & help", "Left bullies to the dragons"),
|
||||
"Bullies > Assessed the situation": ("Investigate & help", "Unknown"),
|
||||
"Injured Cadet > Tourniquet": ("Ignore the scream", "Tourniquet"),
|
||||
"Injured Cadet > Left them there": ("Ignore the scream", "Left injured cadet"),
|
||||
"Injured Cadet > Took their weapons": ("Ignore the scream", "Took their weapons"),
|
||||
"Avoided combat (Investigated scream > Left them to it)": ("Investigate & leave", "Left them to it"),
|
||||
"You died in combat": ("Investigate & help", "DIED in combat"),
|
||||
"N/A - You died with initial dragon sighting": ("DIED at first sighting", "DIED at first sighting"),
|
||||
}
|
||||
COMBAT_MAP = {
|
||||
"Bullies > Assessed the situation": "Assess the situation",
|
||||
"Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)",
|
||||
"Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)": "Put yourself in the middle (bigger cadet)",
|
||||
}
|
||||
HOUR_MAP = {
|
||||
"Continued to Valley Centre": "Valley",
|
||||
"Climbed the tallest tree": "Tree",
|
||||
"N/A - You're dead": "DIED before the hour",
|
||||
}
|
||||
# First-sighting pools per (company, direction) according to the community PDF map.
|
||||
# Used only to FLAG rows that report a dragon outside the documented pool.
|
||||
SIGHTING_POOL = {
|
||||
("Alone", "East"): {"Red", "Orange"},
|
||||
("Alone", "West"): {"Red", "Brown", "Green"},
|
||||
("Alone", "Center"): {"Red", "Brown"},
|
||||
("1 Squadmate", "East"): {"Green", "Orange", "Red"}, # 'leave squadmate' -> alone pools also possible
|
||||
("1 Squadmate", "West"): {"Red", "Brown", "Green"},
|
||||
("1 Squadmate", "Center"): {"Red", "Brown"},
|
||||
("2 Squadmates", "East"): {"Red", "Orange"},
|
||||
("2 Squadmates", "West"): {"Green", "Brown", "Red"},
|
||||
("2 Squadmates", "Center"): {"Red", "Brown"},
|
||||
}
|
||||
|
||||
|
||||
def find_columns(cols):
|
||||
out = {}
|
||||
low = {c: str(c).lower() for c in cols}
|
||||
for key, words in COLUMN_KEYS.items():
|
||||
hits = [c for c in cols if all(w in low[c] for w in words)]
|
||||
if hits:
|
||||
out[key] = hits[0]
|
||||
missing = [k for k in COLUMN_KEYS if k not in out]
|
||||
return out, missing
|
||||
|
||||
|
||||
def blank(v):
|
||||
if v is None or v is pd.NaT:
|
||||
return True
|
||||
if not isinstance(v, str):
|
||||
try:
|
||||
if pd.isna(v):
|
||||
return True
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return str(v).strip() in ("", "nan", "NaN", "NaT")
|
||||
|
||||
|
||||
def s(v):
|
||||
return None if blank(v) else str(v).strip()
|
||||
|
||||
|
||||
def parse_timestamp(v):
|
||||
"""Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'.
|
||||
Returns 'YYYY-MM-DD' or None."""
|
||||
if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v):
|
||||
return pd.Timestamp(v).strftime("%Y-%m-%d")
|
||||
if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date
|
||||
return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d")
|
||||
t = s(v)
|
||||
if not t:
|
||||
return None
|
||||
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
|
||||
m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY
|
||||
if m:
|
||||
return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
|
||||
return None
|
||||
|
||||
|
||||
def parse_attempt(v):
|
||||
v = s(v)
|
||||
if v is None:
|
||||
return None
|
||||
t = v.lower()
|
||||
if "lost count" in t:
|
||||
return None
|
||||
words = {"first": 1, "second": 2, "third": 3, "fourth": 4, "fifth": 5}
|
||||
for w, n in words.items():
|
||||
if t.startswith(w):
|
||||
return n
|
||||
m = re.search(r"\d+", t)
|
||||
return int(m.group()) if m else None
|
||||
|
||||
|
||||
def code_row(r, C):
|
||||
g = lambda k: s(r[C[k]]) if k in C else None
|
||||
flags = []
|
||||
ts = r[C["timestamp"]] if "timestamp" in C else None
|
||||
d = parse_timestamp(ts)
|
||||
if d:
|
||||
source, date = "Form", d
|
||||
else:
|
||||
source, date = (s(ts) or "Unknown import"), None
|
||||
|
||||
wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)]
|
||||
wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown")
|
||||
section = g("wing_" + wings[0]) if len(wings) == 1 else None
|
||||
if len(wings) > 1:
|
||||
flags.append("multiple wings")
|
||||
|
||||
dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)]
|
||||
if len(dirs) == 1:
|
||||
direction, first = dirs[0], g("dir_" + dirs[0])
|
||||
else:
|
||||
direction, first = ("Multiple" if dirs else "Unknown"), None
|
||||
flags.append("direction " + ("multiple" if dirs else "missing"))
|
||||
|
||||
company = g("company") or "Unknown"
|
||||
if first and first != "Unsure" and (company, direction) in SIGHTING_POOL:
|
||||
if first not in SIGHTING_POOL[(company, direction)]:
|
||||
flags.append(f"first sighting {first} not in documented pool for {company}/{direction}")
|
||||
|
||||
cadet_raw = g("cadet")
|
||||
scream, resolution = CADET_MAP.get(cadet_raw, ("Unknown", "Unknown"))
|
||||
if cadet_raw and cadet_raw not in CADET_MAP:
|
||||
flags.append("unrecognised cadet answer")
|
||||
combat = COMBAT_MAP.get(g("combat")) if g("combat") else None
|
||||
|
||||
post_raw = g("post")
|
||||
if scream == "Investigate & help" and not resolution.startswith("DIED"):
|
||||
post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown"
|
||||
elif scream in ("Ignore the scream", "Investigate & leave"):
|
||||
post = "(no fork)"
|
||||
if post_raw in ("Forest", "West", "East"):
|
||||
post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch
|
||||
else:
|
||||
post = None
|
||||
|
||||
hour = HOUR_MAP.get(g("hour"))
|
||||
died_early = scream.startswith("DIED") or resolution.startswith("DIED")
|
||||
if died_early:
|
||||
hour = None
|
||||
elif hour is None:
|
||||
hour = "Unknown"
|
||||
|
||||
enc = {c: g("enc_" + c) for c in COLORS}
|
||||
bonded = [c for c in COLORS if enc[c] == "Bonded!"]
|
||||
burned = [c for c in COLORS if enc[c] == "Incinerated"]
|
||||
|
||||
# Which encounter branch did the run take after the hour?
|
||||
branch = None
|
||||
if hour == "Tree":
|
||||
if enc["Blue"] or enc["Black"]:
|
||||
branch = "Shadow (rare)"
|
||||
elif enc["Green"]:
|
||||
branch = "Descend slowly: two Greens"
|
||||
elif enc["Red"]:
|
||||
branch = "Descend quickly: Red"
|
||||
elif enc["Orange"] or enc["Brown"]:
|
||||
branch = "Descend slowly: two Greens"
|
||||
flags.append("tree run with Orange/Brown but no Green recorded")
|
||||
elif hour == "Valley":
|
||||
if enc["Green"]:
|
||||
flags.append("Green recorded on a Valley run (no Greens in the valley)")
|
||||
red_passed = enc["Red"] in ("Changed my mind",)
|
||||
meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]]
|
||||
if meadow or red_passed:
|
||||
branch = "Passed the Red: meadow"
|
||||
elif enc["Red"]:
|
||||
branch = "Valley Red"
|
||||
|
||||
# Final outcome of the run
|
||||
outcome, outcome_color = None, None
|
||||
if died_early:
|
||||
outcome = "Died before the hour"
|
||||
elif len(bonded) == 1:
|
||||
outcome, outcome_color = "Bonded", bonded[0]
|
||||
elif len(bonded) > 1:
|
||||
outcome = "Ambiguous"
|
||||
flags.append("more than one dragon marked Bonded")
|
||||
elif burned:
|
||||
order = ["Red", "Green", "Orange", "Brown", "Blue", "Black"] if hour != "Valley" else \
|
||||
["Red", "Orange", "Brown", "Blue", "Green", "Black"]
|
||||
last = [c for c in order if c in burned][-1]
|
||||
outcome, outcome_color = "Incinerated", last
|
||||
if len(burned) > 1:
|
||||
flags.append("more than one dragon marked Incinerated")
|
||||
elif hour == "DIED before the hour":
|
||||
outcome = "Died before the hour"
|
||||
else:
|
||||
outcome = "No outcome recorded"
|
||||
|
||||
# The dragon whose bonding encounter decided the run (attempted = Bonded or Incinerated)
|
||||
rescued = enc["Green"] == "Saved by another" and outcome == "Bonded" and outcome_color in ("Orange", "Brown")
|
||||
|
||||
attempt = parse_attempt(g("attempt"))
|
||||
return {
|
||||
"source": source, "date": date, "wing": wing, "section": section,
|
||||
"squad": g("squad") or "Unknown", "attempt": attempt,
|
||||
"attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if attempt and attempt > 1 else "Unknown/lost count"),
|
||||
"goal": g("goal") or "(not asked/blank)", "company": company,
|
||||
"direction": direction, "first_sighting": first or "Unknown",
|
||||
"scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)",
|
||||
"post_path": post, "hour": hour, "branch": branch,
|
||||
**{"enc_" + c: enc[c] for c in COLORS},
|
||||
"outcome": outcome, "outcome_color": outcome_color, "rescued": rescued,
|
||||
"saw_blue_meadow": bool(hour == "Valley" and enc["Blue"]),
|
||||
"saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"])),
|
||||
"data_flags": "; ".join(flags),
|
||||
"death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")),
|
||||
}
|
||||
|
||||
|
||||
def load(path):
|
||||
if str(path).lower().endswith(".json"): # the page's bundled export (data/public/form_responses.json)
|
||||
import json
|
||||
d = json.load(open(path, encoding="utf-8"))
|
||||
raw = pd.DataFrame(d["rows"], columns=d["headers"])
|
||||
elif str(path).lower().endswith(".csv"):
|
||||
raw = pd.read_csv(path)
|
||||
else:
|
||||
xl = pd.ExcelFile(path)
|
||||
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
|
||||
raw = pd.read_excel(xl, sheet)
|
||||
C, missing = find_columns(raw.columns)
|
||||
rows = [code_row(r, C) for _, r in raw.iterrows()]
|
||||
df = pd.DataFrame(rows)
|
||||
df.insert(0, "run_id", range(1, len(df) + 1))
|
||||
return df, missing
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/public/form_responses.json"
|
||||
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/derived/runs_clean.csv"
|
||||
df, missing = load(src)
|
||||
if missing:
|
||||
print("WARNING: columns not found:", missing)
|
||||
df.to_csv(out, index=False)
|
||||
print(f"{len(df)} runs -> {out}")
|
||||
print(df.outcome.value_counts().to_string())
|
||||
print("rows with flags:", (df["data_flags"] != "").sum())
|
||||
Reference in New Issue
Block a user