The Threshing Routes form was redesigned (5,651 responses). Both implementations
(engine.js and clean.py) now read either version:
- one direction question + one colour question, with the older per-direction
columns used only to fill gaps and cross-check
- cadet encounter as an action path (scream, first and second combat move)
plus a separate outcome
- new fields: sighting_action, combat_second, rare_reported, malek_intent,
left_squadmates, tail; "Why are you doing this run?" mapped onto the goal
("Blue or Black" for rare chasers)
- the rare-dragon question feeds saw_blue_meadow / saw_shadow and the branch
The old export still codes identically. JS/Python parity: all match.
Data: form 2,573 -> 5,651 rows, Dragon Register 216 -> 273, new route PDF
fingerprint. Supersedes the automated pull request #2.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
402 lines
18 KiB
Python
402 lines
18 KiB
Python
"""
|
|
clean.py — turn the raw "Threshing Routes Responses" Google Form export into one
|
|
coded row per threshing run.
|
|
|
|
Reference implementation in Python. The web page (packages/core/src/engine.js) implements the
|
|
same rules in JavaScript; tests/compare.py checks that both give identical output.
|
|
|
|
Usage:
|
|
python clean.py ../../data/raw/form/Threshing_Routes_Responses.xlsx -> writes ../../data/derived/runs_clean.csv
|
|
|
|
Every coding rule is listed in RULES below and in the "Methods" section of the page.
|
|
"""
|
|
import re
|
|
import sys
|
|
from datetime import datetime
|
|
|
|
import pandas as pd
|
|
|
|
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
|
|
|
|
|
def redact(v):
|
|
"""Free-text answers sometimes contain a respondent's email address; never display or export it."""
|
|
return v if v is None else EMAIL_RE.sub("[email removed]", str(v))
|
|
|
|
COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
|
|
|
|
# Columns are found by keywords in the question text, so the script keeps working
|
|
# if the form owner reorders or adds questions.
|
|
COLUMN_KEYS = {
|
|
"timestamp": ["timestamp"],
|
|
"wing_First": ["wing and section", "[first]"],
|
|
"wing_Second": ["wing and section", "[second]"],
|
|
"wing_Third": ["wing and section", "[third]"],
|
|
"wing_Fourth": ["wing and section", "[fourth]"],
|
|
"squad": ["what squad"],
|
|
"attempt": ["attempt number"],
|
|
"company": ["who accompanied"],
|
|
"dir_East": ["initial direction", "[east"],
|
|
"dir_West": ["initial direction", "[west"],
|
|
"dir_Center": ["initial direction", "[forest center"],
|
|
"cadet": ["outcome of your cadet encounter"],
|
|
"post": ["path did you take after the bully"],
|
|
"hour": ["after walking an hour"],
|
|
"enc_Red": ["attempt to bond", "[red"],
|
|
"enc_Green": ["attempt to bond", "[green"],
|
|
"enc_Orange": ["attempt to bond", "[orange"],
|
|
"enc_Brown": ["attempt to bond", "[brown"],
|
|
"enc_Blue": ["attempt to bond", "[blue"],
|
|
"enc_Black": ["attempt to bond", "[black"],
|
|
"death_note": ["those who died"],
|
|
"bond_note": ["those who bonded"],
|
|
"goal": ["colour dragon are you trying"],
|
|
"combat": ["combat choice"],
|
|
# Questions added in the redesigned form (late October 2026). "!x" means the header must NOT contain x.
|
|
"first_dir": ["what was your first direction"],
|
|
"first_colour": ["colour was your initial dragon sighting", "!["],
|
|
"leg_West": ["colour was your initial dragon sighting", "[west"],
|
|
"leg_Center": ["colour was your initial dragon sighting", "[forest center"],
|
|
"sighting_action": ["action did you take during the first dragon sighting"],
|
|
"cadet_actions": ["during your cadet encounter"],
|
|
"goal_v2": ["why are you doing this run"],
|
|
"malek": ["seek malek"],
|
|
"rare": ["encounter a rare dragon"],
|
|
"tail": ["what tail does your dragon have"],
|
|
}
|
|
# A key is only reported missing when neither it nor its replacement in the other form version was found.
|
|
ALTERNATIVE = {
|
|
"dir_East": "first_dir", "dir_West": "first_dir", "dir_Center": "first_dir", "first_dir": "dir_Center",
|
|
"first_colour": "dir_Center", "goal": "goal_v2", "goal_v2": "goal", "combat": "cadet_actions", "cadet_actions": "combat",
|
|
}
|
|
OPTIONAL = {"leg_West", "leg_Center", "sighting_action", "malek", "rare", "tail"}
|
|
|
|
CADET_MAP = {
|
|
"Bullies > You dealt with them": ("Investigate & help", "Killed the bullies yourself"),
|
|
"Bullies > Squadmate's revenge": ("Investigate & help", "Squadmate's revenge"),
|
|
"Bullies > Left to the Dragons": ("Investigate & help", "Left bullies to the dragons"),
|
|
"Bullies > Assessed the situation": ("Investigate & help", "Unknown"),
|
|
"Injured Cadet > Tourniquet": ("Ignore the scream", "Tourniquet"),
|
|
"Injured Cadet > Left them there": ("Ignore the scream", "Left injured cadet"),
|
|
"Injured Cadet > Took their weapons": ("Ignore the scream", "Took their weapons"),
|
|
"Avoided combat (Investigated scream > Left them to it)": ("Investigate & leave", "Left them to it"),
|
|
"You died in combat": ("Investigate & help", "DIED in combat"),
|
|
"N/A - You died with initial dragon sighting": ("DIED at first sighting", "DIED at first sighting"),
|
|
}
|
|
COMBAT_MAP = {
|
|
"Bullies > Assessed the situation": "Assess the situation",
|
|
"Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)",
|
|
"Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)": "Put yourself in the middle (bigger cadet)",
|
|
}
|
|
# Redesigned form: the cadet encounter is asked as a path of actions plus a separate outcome.
|
|
# value: (scream, first combat move, second combat move, resolution when the action decides it)
|
|
CADET_ACTIONS = {
|
|
"Bullies > Assess > Unknown": ("Investigate & help", "Assess the situation", None, None),
|
|
"Bullies > Assess > Smaller Cadet": ("Investigate & help", "Assess the situation", "Engage the smaller cadet", None),
|
|
"Bullies > Assess > Daggers or not": ("Investigate & help", "Assess the situation", "Daggers or not (protect squadmate)", None),
|
|
"Bullies > Unknown": ("Investigate & help", None, None, None),
|
|
"Bullies > Smaller Cadet (Did not assess)": ("Investigate & help", "Even the numbers (smaller cadet)", None, None),
|
|
"Bullies > Bigger Cadet (Did not assess)": ("Investigate & help", "Put yourself in the middle (bigger cadet)", None, None),
|
|
"Injured Cadet > Tourniquet": ("Ignore the scream", None, None, "Tourniquet"),
|
|
"Injured Cadet > Left them there": ("Ignore the scream", None, None, "Left injured cadet"),
|
|
"Injured Cadet > Took their weapons": ("Ignore the scream", None, None, "Took their weapons"),
|
|
"Avoided combat (Investigated scream > Left them to it)": ("Investigate & leave", None, None, "Left them to it"),
|
|
"N/A - You died with initial dragon sighting": ("DIED at first sighting", None, None, "DIED at first sighting"),
|
|
"Unsure": ("Unknown", None, None, "Unknown"),
|
|
}
|
|
CADET_OUTCOME = {
|
|
"Squadmate's revenge": "Squadmate's revenge",
|
|
"You ended the bullies yourself": "Killed the bullies yourself",
|
|
"Left to the Dragons": "Left bullies to the dragons",
|
|
"You died in combat": "DIED in combat",
|
|
}
|
|
DIRECTION_MAP = {"Forest Center": "Center", "West / Scent": "West", "East / Roar": "East"}
|
|
GOAL_V2 = {
|
|
"Hoping for a Rare Dragon (Blue or Black)": "Blue or Black",
|
|
"Trying to bond the Red dragon under the tree": "Red (tree)",
|
|
"Daily Run Assignment data": "Daily run data",
|
|
"Just here for fun, and maybe a dragon": "Any (for fun)",
|
|
}
|
|
SIGHTING_ACTION = {
|
|
"Hold Ground / Stay Still": "Hold still", "Respect / Face it": "Respect / face it",
|
|
"Retreat / Run / Back Away": "Retreat / run", "Hide / Duck": "Hide", "Draw / Drop Weapon": "Weapon",
|
|
"Meet Gaze": "Meet its gaze", "Protect / Help Sqaudmate": "Protect squadmate", "Approach": "Approach",
|
|
"Stand on your own": "Stand alone", "Unsure": "Unknown", "Unsure - You don't remember": "Unknown",
|
|
}
|
|
RARE_MAP = {"Yes - A blue dragon in the valley": "Blue (valley)", "Yes - Shadow!!!": "Shadow", "No": "No"}
|
|
|
|
HOUR_MAP = {
|
|
"Continued to Valley Centre": "Valley",
|
|
"Climbed the tallest tree": "Tree",
|
|
"N/A - You're dead": "DIED before the hour",
|
|
}
|
|
# First-sighting pools per (company, direction) according to the community PDF map.
|
|
# Used only to FLAG rows that report a dragon outside the documented pool.
|
|
SIGHTING_POOL = {
|
|
("Alone", "East"): {"Red", "Orange"},
|
|
("Alone", "West"): {"Red", "Brown", "Green"},
|
|
("Alone", "Center"): {"Red", "Brown"},
|
|
("1 Squadmate", "East"): {"Green", "Orange", "Red"}, # 'leave squadmate' -> alone pools also possible
|
|
("1 Squadmate", "West"): {"Red", "Brown", "Green"},
|
|
("1 Squadmate", "Center"): {"Red", "Brown"},
|
|
("2 Squadmates", "East"): {"Red", "Orange"},
|
|
("2 Squadmates", "West"): {"Green", "Brown", "Red"},
|
|
("2 Squadmates", "Center"): {"Red", "Brown"},
|
|
}
|
|
|
|
|
|
def find_columns(cols):
|
|
out = {}
|
|
low = {c: str(c).lower() for c in cols}
|
|
for key, words in COLUMN_KEYS.items():
|
|
hits = [c for c in cols if all((w[1:] not in low[c]) if w[0] == "!" else (w in low[c]) for w in words)]
|
|
if hits:
|
|
out[key] = hits[0]
|
|
missing = [k for k in COLUMN_KEYS if k not in out and k not in OPTIONAL and ALTERNATIVE.get(k) not in out]
|
|
return out, missing
|
|
|
|
|
|
def blank(v):
|
|
if v is None or v is pd.NaT:
|
|
return True
|
|
if not isinstance(v, str):
|
|
try:
|
|
if pd.isna(v):
|
|
return True
|
|
except (TypeError, ValueError):
|
|
pass
|
|
return str(v).strip() in ("", "nan", "NaN", "NaT")
|
|
|
|
|
|
def s(v):
|
|
return None if blank(v) else str(v).strip()
|
|
|
|
|
|
def parse_timestamp(v):
|
|
"""Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'.
|
|
Returns 'YYYY-MM-DD' or None."""
|
|
if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v):
|
|
return pd.Timestamp(v).strftime("%Y-%m-%d")
|
|
if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date
|
|
return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d")
|
|
t = s(v)
|
|
if not t:
|
|
return None
|
|
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t)
|
|
if m:
|
|
return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
|
|
m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY
|
|
if m:
|
|
return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
|
|
return None
|
|
|
|
|
|
def parse_attempt(v):
|
|
v = s(v)
|
|
if v is None:
|
|
return None
|
|
t = v.lower()
|
|
if "lost count" in t:
|
|
return None
|
|
words = {"first": 1, "second": 2, "third": 3, "fourth": 4, "fifth": 5}
|
|
for w, n in words.items():
|
|
if t.startswith(w):
|
|
return n
|
|
m = re.search(r"\d+", t)
|
|
return int(m.group()) if m else None
|
|
|
|
|
|
def code_row(r, C):
|
|
g = lambda k: s(r[C[k]]) if k in C else None
|
|
flags = []
|
|
ts = r[C["timestamp"]] if "timestamp" in C else None
|
|
d = parse_timestamp(ts)
|
|
if d:
|
|
source, date = "Form", d
|
|
else:
|
|
source, date = (s(ts) or "Unknown import"), None
|
|
|
|
wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)]
|
|
wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown")
|
|
section = g("wing_" + wings[0]) if len(wings) == 1 else None
|
|
if len(wings) > 1:
|
|
flags.append("multiple wings")
|
|
|
|
if "first_dir" in C:
|
|
# Redesigned form: one direction question and one colour question. The older per-direction colour
|
|
# columns are still in the sheet for some rows and are used only to fill gaps and to cross-check.
|
|
direction = DIRECTION_MAP.get(g("first_dir"))
|
|
first = g("first_colour")
|
|
legacy = [d for d in ("West", "Center") if g("leg_" + d)]
|
|
if not direction and len(legacy) == 1:
|
|
direction = legacy[0]
|
|
flags.append("direction taken from the older sighting column")
|
|
leg = g("leg_" + direction) if direction else None
|
|
if not first and leg:
|
|
first = leg
|
|
elif first and leg and first != leg and first != "Unsure" and leg != "Unsure":
|
|
flags.append("first sighting differs between the old and new columns")
|
|
if not direction:
|
|
direction = "Unknown"
|
|
flags.append("direction missing")
|
|
else:
|
|
dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)]
|
|
if len(dirs) == 1:
|
|
direction, first = dirs[0], g("dir_" + dirs[0])
|
|
else:
|
|
direction, first = ("Multiple" if dirs else "Unknown"), None
|
|
flags.append("direction " + ("multiple" if dirs else "missing"))
|
|
|
|
company_raw = g("company") or "Unknown"
|
|
left_squad = company_raw.endswith(" > Left them")
|
|
company = company_raw[: -len(" > Left them")] if left_squad else company_raw
|
|
# Leaving your squadmates sends you into the solo pools, so either pool is plausible.
|
|
pool = list(SIGHTING_POOL.get((company, direction), [])) + (list(SIGHTING_POOL.get(("Alone", direction), [])) if left_squad else [])
|
|
if first and first != "Unsure" and pool and first not in pool:
|
|
flags.append(f"first sighting {first} not in documented pool for {company}/{direction}")
|
|
|
|
cadet_raw = g("cadet")
|
|
combat = combat2 = None
|
|
if "cadet_actions" in C:
|
|
act = g("cadet_actions")
|
|
a = CADET_ACTIONS.get(act, ("Unknown", None, None, "Unknown"))
|
|
if act and act not in CADET_ACTIONS:
|
|
flags.append("unrecognised cadet answer")
|
|
scream, combat, combat2 = a[0], a[1], a[2]
|
|
out = CADET_OUTCOME.get(cadet_raw)
|
|
if a[3] is not None:
|
|
resolution = a[3]
|
|
if out == "DIED in combat" and not resolution.startswith("DIED"):
|
|
flags.append("died in combat on a path without combat")
|
|
else:
|
|
resolution = out or "Unknown"
|
|
else:
|
|
scream, resolution = CADET_MAP.get(cadet_raw, ("Unknown", "Unknown"))
|
|
if cadet_raw and cadet_raw not in CADET_MAP:
|
|
flags.append("unrecognised cadet answer")
|
|
combat = COMBAT_MAP.get(g("combat")) if g("combat") else None
|
|
|
|
post_raw = g("post")
|
|
if scream == "Investigate & help" and not resolution.startswith("DIED"):
|
|
post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown"
|
|
elif scream in ("Ignore the scream", "Investigate & leave"):
|
|
post = "(no fork)"
|
|
if post_raw in ("Forest", "West", "East"):
|
|
post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch
|
|
else:
|
|
post = None
|
|
|
|
hour = HOUR_MAP.get(g("hour"))
|
|
died_early = scream.startswith("DIED") or resolution.startswith("DIED")
|
|
if died_early:
|
|
hour = None
|
|
elif hour is None:
|
|
hour = "Unknown"
|
|
|
|
enc = {c: g("enc_" + c) for c in COLORS}
|
|
rare = RARE_MAP.get(g("rare"))
|
|
bonded = [c for c in COLORS if enc[c] == "Bonded!"]
|
|
burned = [c for c in COLORS if enc[c] == "Incinerated"]
|
|
|
|
# Which encounter branch did the run take after the hour?
|
|
branch = None
|
|
if hour == "Tree":
|
|
if enc["Blue"] or enc["Black"] or rare == "Shadow":
|
|
branch = "Shadow (rare)"
|
|
elif enc["Green"]:
|
|
branch = "Descend slowly: two Greens"
|
|
elif enc["Red"]:
|
|
branch = "Descend quickly: Red"
|
|
elif enc["Orange"] or enc["Brown"]:
|
|
branch = "Descend slowly: two Greens"
|
|
flags.append("tree run with Orange/Brown but no Green recorded")
|
|
elif hour == "Valley":
|
|
if enc["Green"]:
|
|
flags.append("Green recorded on a Valley run (no Greens in the valley)")
|
|
red_passed = enc["Red"] in ("Changed my mind",)
|
|
meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]] or rare == "Blue (valley)"
|
|
if meadow or red_passed:
|
|
branch = "Passed the Red: meadow"
|
|
elif enc["Red"]:
|
|
branch = "Valley Red"
|
|
|
|
# Final outcome of the run
|
|
outcome, outcome_color = None, None
|
|
if died_early:
|
|
outcome = "Died before the hour"
|
|
elif len(bonded) == 1:
|
|
outcome, outcome_color = "Bonded", bonded[0]
|
|
elif len(bonded) > 1:
|
|
outcome = "Ambiguous"
|
|
flags.append("more than one dragon marked Bonded")
|
|
elif burned:
|
|
order = ["Red", "Green", "Orange", "Brown", "Blue", "Black"] if hour != "Valley" else \
|
|
["Red", "Orange", "Brown", "Blue", "Green", "Black"]
|
|
last = [c for c in order if c in burned][-1]
|
|
outcome, outcome_color = "Incinerated", last
|
|
if len(burned) > 1:
|
|
flags.append("more than one dragon marked Incinerated")
|
|
elif hour == "DIED before the hour":
|
|
outcome = "Died before the hour"
|
|
else:
|
|
outcome = "No outcome recorded"
|
|
|
|
# The dragon whose bonding encounter decided the run (attempted = Bonded or Incinerated)
|
|
rescued = enc["Green"] == "Saved by another" and outcome == "Bonded" and outcome_color in ("Orange", "Brown")
|
|
|
|
attempt = parse_attempt(g("attempt"))
|
|
if (rare == "Blue (valley)" and hour == "Tree") or (rare == "Shadow" and hour == "Valley"):
|
|
flags.append("rare-dragon answer does not match the path")
|
|
goal_raw = g("goal_v2")
|
|
goal = g("goal") or (GOAL_V2.get(goal_raw, goal_raw) if goal_raw and goal_raw != "Unknown" else None) or "(not asked/blank)"
|
|
return {
|
|
"source": source, "date": date, "wing": wing, "section": section,
|
|
"squad": g("squad") or "Unknown", "attempt": attempt,
|
|
"attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if attempt and attempt > 1 else "Unknown/lost count"),
|
|
"goal": goal, "company": company, "left_squadmates": left_squad,
|
|
"direction": direction, "first_sighting": first or "Unknown",
|
|
"sighting_action": (SIGHTING_ACTION.get(g("sighting_action")) or g("sighting_action") or "Unknown") if "sighting_action" in C else "(not asked)",
|
|
"scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)",
|
|
"combat_second": combat2 or "(not recorded)",
|
|
"post_path": post, "hour": hour, "branch": branch,
|
|
**{"enc_" + c: enc[c] for c in COLORS},
|
|
"outcome": outcome, "outcome_color": outcome_color, "rescued": rescued,
|
|
"saw_blue_meadow": bool(hour == "Valley" and (enc["Blue"] or rare == "Blue (valley)")),
|
|
"saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"] or rare == "Shadow")),
|
|
"rare_reported": (rare or "Unknown") if "rare" in C else "(not asked)",
|
|
"malek_intent": (g("malek") or "Unknown") if "malek" in C else "(not asked)",
|
|
"tail": g("tail"),
|
|
"data_flags": "; ".join(flags),
|
|
"death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")),
|
|
}
|
|
|
|
|
|
def load(path):
|
|
if str(path).lower().endswith(".json"): # the page's bundled export (data/public/form_responses.json)
|
|
import json
|
|
d = json.load(open(path, encoding="utf-8"))
|
|
raw = pd.DataFrame(d["rows"], columns=d["headers"])
|
|
elif str(path).lower().endswith(".csv"):
|
|
raw = pd.read_csv(path)
|
|
else:
|
|
xl = pd.ExcelFile(path)
|
|
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
|
|
raw = pd.read_excel(xl, sheet)
|
|
C, missing = find_columns(raw.columns)
|
|
rows = [code_row(r, C) for _, r in raw.iterrows()]
|
|
df = pd.DataFrame(rows)
|
|
df.insert(0, "run_id", range(1, len(df) + 1))
|
|
return df, missing
|
|
|
|
|
|
if __name__ == "__main__":
|
|
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/public/form_responses.json"
|
|
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/derived/runs_clean.csv"
|
|
df, missing = load(src)
|
|
if missing:
|
|
print("WARNING: columns not found:", missing)
|
|
df.to_csv(out, index=False)
|
|
print(f"{len(df)} runs -> {out}")
|
|
print(df.outcome.value_counts().to_string())
|
|
print("rows with flags:", (df["data_flags"] != "").sum())
|