Coding rules for the redesigned community form; import the new data

The Threshing Routes form was redesigned (5,651 responses). Both implementations
(engine.js and clean.py) now read either version:
- one direction question + one colour question, with the older per-direction
  columns used only to fill gaps and cross-check
- cadet encounter as an action path (scream, first and second combat move)
  plus a separate outcome
- new fields: sighting_action, combat_second, rare_reported, malek_intent,
  left_squadmates, tail; "Why are you doing this run?" mapped onto the goal
  ("Blue or Black" for rare chasers)
- the rare-dragon question feeds saw_blue_meadow / saw_shadow and the branch
The old export still codes identically. JS/Python parity: all match.

Data: form 2,573 -> 5,651 rows, Dragon Register 216 -> 273, new route PDF
fingerprint. Supersedes the automated pull request #2.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-01 22:10:51 +00:00
co-authored by Claude Opus 5.5
parent 9c0e35423a
commit 46ee08a56b
8 changed files with 5911 additions and 5707 deletions
+2 -2
View File
@@ -16,7 +16,7 @@ from stats_core import Z95, bh, logistic_ridge, odds_ratio, p_from_z, wilson
COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
FACTORS = ["company", "direction", "first_sighting", "scream", "resolution", "combat_first",
"post_path", "hour", "wing", "squad", "attempt_group", "source", "goal"]
"post_path", "hour", "wing", "squad", "attempt_group", "source", "goal", "sighting_action", "combat_second", "malek_intent"]
def W(k, n):
@@ -89,7 +89,7 @@ def model(d, key, factors, lam=1.0):
def goal2(r):
if r.goal in ("Blue", "Black"):
if r.goal in ("Blue", "Black", "Blue or Black"):
return r.goal
return "Blank/not asked" if r.goal == "(not asked/blank)" else "Other colour/any"
+120 -20
View File
@@ -52,7 +52,24 @@ COLUMN_KEYS = {
"bond_note": ["those who bonded"],
"goal": ["colour dragon are you trying"],
"combat": ["combat choice"],
# Questions added in the redesigned form (late October 2026). "!x" means the header must NOT contain x.
"first_dir": ["what was your first direction"],
"first_colour": ["colour was your initial dragon sighting", "!["],
"leg_West": ["colour was your initial dragon sighting", "[west"],
"leg_Center": ["colour was your initial dragon sighting", "[forest center"],
"sighting_action": ["action did you take during the first dragon sighting"],
"cadet_actions": ["during your cadet encounter"],
"goal_v2": ["why are you doing this run"],
"malek": ["seek malek"],
"rare": ["encounter a rare dragon"],
"tail": ["what tail does your dragon have"],
}
# A key is only reported missing when neither it nor its replacement in the other form version was found.
ALTERNATIVE = {
"dir_East": "first_dir", "dir_West": "first_dir", "dir_Center": "first_dir", "first_dir": "dir_Center",
"first_colour": "dir_Center", "goal": "goal_v2", "goal_v2": "goal", "combat": "cadet_actions", "cadet_actions": "combat",
}
OPTIONAL = {"leg_West", "leg_Center", "sighting_action", "malek", "rare", "tail"}
CADET_MAP = {
"Bullies > You dealt with them": ("Investigate & help", "Killed the bullies yourself"),
@@ -71,6 +88,43 @@ COMBAT_MAP = {
"Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)",
"Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)": "Put yourself in the middle (bigger cadet)",
}
# Redesigned form: the cadet encounter is asked as a path of actions plus a separate outcome.
# value: (scream, first combat move, second combat move, resolution when the action decides it)
CADET_ACTIONS = {
"Bullies > Assess > Unknown": ("Investigate & help", "Assess the situation", None, None),
"Bullies > Assess > Smaller Cadet": ("Investigate & help", "Assess the situation", "Engage the smaller cadet", None),
"Bullies > Assess > Daggers or not": ("Investigate & help", "Assess the situation", "Daggers or not (protect squadmate)", None),
"Bullies > Unknown": ("Investigate & help", None, None, None),
"Bullies > Smaller Cadet (Did not assess)": ("Investigate & help", "Even the numbers (smaller cadet)", None, None),
"Bullies > Bigger Cadet (Did not assess)": ("Investigate & help", "Put yourself in the middle (bigger cadet)", None, None),
"Injured Cadet > Tourniquet": ("Ignore the scream", None, None, "Tourniquet"),
"Injured Cadet > Left them there": ("Ignore the scream", None, None, "Left injured cadet"),
"Injured Cadet > Took their weapons": ("Ignore the scream", None, None, "Took their weapons"),
"Avoided combat (Investigated scream > Left them to it)": ("Investigate & leave", None, None, "Left them to it"),
"N/A - You died with initial dragon sighting": ("DIED at first sighting", None, None, "DIED at first sighting"),
"Unsure": ("Unknown", None, None, "Unknown"),
}
CADET_OUTCOME = {
"Squadmate's revenge": "Squadmate's revenge",
"You ended the bullies yourself": "Killed the bullies yourself",
"Left to the Dragons": "Left bullies to the dragons",
"You died in combat": "DIED in combat",
}
DIRECTION_MAP = {"Forest Center": "Center", "West / Scent": "West", "East / Roar": "East"}
GOAL_V2 = {
"Hoping for a Rare Dragon (Blue or Black)": "Blue or Black",
"Trying to bond the Red dragon under the tree": "Red (tree)",
"Daily Run Assignment data": "Daily run data",
"Just here for fun, and maybe a dragon": "Any (for fun)",
}
SIGHTING_ACTION = {
"Hold Ground / Stay Still": "Hold still", "Respect / Face it": "Respect / face it",
"Retreat / Run / Back Away": "Retreat / run", "Hide / Duck": "Hide", "Draw / Drop Weapon": "Weapon",
"Meet Gaze": "Meet its gaze", "Protect / Help Sqaudmate": "Protect squadmate", "Approach": "Approach",
"Stand on your own": "Stand alone", "Unsure": "Unknown", "Unsure - You don't remember": "Unknown",
}
RARE_MAP = {"Yes - A blue dragon in the valley": "Blue (valley)", "Yes - Shadow!!!": "Shadow", "No": "No"}
HOUR_MAP = {
"Continued to Valley Centre": "Valley",
"Climbed the tallest tree": "Tree",
@@ -95,10 +149,10 @@ def find_columns(cols):
out = {}
low = {c: str(c).lower() for c in cols}
for key, words in COLUMN_KEYS.items():
hits = [c for c in cols if all(w in low[c] for w in words)]
hits = [c for c in cols if all((w[1:] not in low[c]) if w[0] == "!" else (w in low[c]) for w in words)]
if hits:
out[key] = hits[0]
missing = [k for k in COLUMN_KEYS if k not in out]
missing = [k for k in COLUMN_KEYS if k not in out and k not in OPTIONAL and ALTERNATIVE.get(k) not in out]
return out, missing
@@ -168,23 +222,59 @@ def code_row(r, C):
if len(wings) > 1:
flags.append("multiple wings")
dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)]
if len(dirs) == 1:
direction, first = dirs[0], g("dir_" + dirs[0])
if "first_dir" in C:
# Redesigned form: one direction question and one colour question. The older per-direction colour
# columns are still in the sheet for some rows and are used only to fill gaps and to cross-check.
direction = DIRECTION_MAP.get(g("first_dir"))
first = g("first_colour")
legacy = [d for d in ("West", "Center") if g("leg_" + d)]
if not direction and len(legacy) == 1:
direction = legacy[0]
flags.append("direction taken from the older sighting column")
leg = g("leg_" + direction) if direction else None
if not first and leg:
first = leg
elif first and leg and first != leg and first != "Unsure" and leg != "Unsure":
flags.append("first sighting differs between the old and new columns")
if not direction:
direction = "Unknown"
flags.append("direction missing")
else:
direction, first = ("Multiple" if dirs else "Unknown"), None
flags.append("direction " + ("multiple" if dirs else "missing"))
dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)]
if len(dirs) == 1:
direction, first = dirs[0], g("dir_" + dirs[0])
else:
direction, first = ("Multiple" if dirs else "Unknown"), None
flags.append("direction " + ("multiple" if dirs else "missing"))
company = g("company") or "Unknown"
if first and first != "Unsure" and (company, direction) in SIGHTING_POOL:
if first not in SIGHTING_POOL[(company, direction)]:
flags.append(f"first sighting {first} not in documented pool for {company}/{direction}")
company_raw = g("company") or "Unknown"
left_squad = company_raw.endswith(" > Left them")
company = company_raw[: -len(" > Left them")] if left_squad else company_raw
# Leaving your squadmates sends you into the solo pools, so either pool is plausible.
pool = list(SIGHTING_POOL.get((company, direction), [])) + (list(SIGHTING_POOL.get(("Alone", direction), [])) if left_squad else [])
if first and first != "Unsure" and pool and first not in pool:
flags.append(f"first sighting {first} not in documented pool for {company}/{direction}")
cadet_raw = g("cadet")
scream, resolution = CADET_MAP.get(cadet_raw, ("Unknown", "Unknown"))
if cadet_raw and cadet_raw not in CADET_MAP:
flags.append("unrecognised cadet answer")
combat = COMBAT_MAP.get(g("combat")) if g("combat") else None
combat = combat2 = None
if "cadet_actions" in C:
act = g("cadet_actions")
a = CADET_ACTIONS.get(act, ("Unknown", None, None, "Unknown"))
if act and act not in CADET_ACTIONS:
flags.append("unrecognised cadet answer")
scream, combat, combat2 = a[0], a[1], a[2]
out = CADET_OUTCOME.get(cadet_raw)
if a[3] is not None:
resolution = a[3]
if out == "DIED in combat" and not resolution.startswith("DIED"):
flags.append("died in combat on a path without combat")
else:
resolution = out or "Unknown"
else:
scream, resolution = CADET_MAP.get(cadet_raw, ("Unknown", "Unknown"))
if cadet_raw and cadet_raw not in CADET_MAP:
flags.append("unrecognised cadet answer")
combat = COMBAT_MAP.get(g("combat")) if g("combat") else None
post_raw = g("post")
if scream == "Investigate & help" and not resolution.startswith("DIED"):
@@ -204,13 +294,14 @@ def code_row(r, C):
hour = "Unknown"
enc = {c: g("enc_" + c) for c in COLORS}
rare = RARE_MAP.get(g("rare"))
bonded = [c for c in COLORS if enc[c] == "Bonded!"]
burned = [c for c in COLORS if enc[c] == "Incinerated"]
# Which encounter branch did the run take after the hour?
branch = None
if hour == "Tree":
if enc["Blue"] or enc["Black"]:
if enc["Blue"] or enc["Black"] or rare == "Shadow":
branch = "Shadow (rare)"
elif enc["Green"]:
branch = "Descend slowly: two Greens"
@@ -223,7 +314,7 @@ def code_row(r, C):
if enc["Green"]:
flags.append("Green recorded on a Valley run (no Greens in the valley)")
red_passed = enc["Red"] in ("Changed my mind",)
meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]]
meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]] or rare == "Blue (valley)"
if meadow or red_passed:
branch = "Passed the Red: meadow"
elif enc["Red"]:
@@ -254,18 +345,27 @@ def code_row(r, C):
rescued = enc["Green"] == "Saved by another" and outcome == "Bonded" and outcome_color in ("Orange", "Brown")
attempt = parse_attempt(g("attempt"))
if (rare == "Blue (valley)" and hour == "Tree") or (rare == "Shadow" and hour == "Valley"):
flags.append("rare-dragon answer does not match the path")
goal_raw = g("goal_v2")
goal = g("goal") or (GOAL_V2.get(goal_raw, goal_raw) if goal_raw and goal_raw != "Unknown" else None) or "(not asked/blank)"
return {
"source": source, "date": date, "wing": wing, "section": section,
"squad": g("squad") or "Unknown", "attempt": attempt,
"attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if attempt and attempt > 1 else "Unknown/lost count"),
"goal": g("goal") or "(not asked/blank)", "company": company,
"goal": goal, "company": company, "left_squadmates": left_squad,
"direction": direction, "first_sighting": first or "Unknown",
"sighting_action": (SIGHTING_ACTION.get(g("sighting_action")) or g("sighting_action") or "Unknown") if "sighting_action" in C else "(not asked)",
"scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)",
"combat_second": combat2 or "(not recorded)",
"post_path": post, "hour": hour, "branch": branch,
**{"enc_" + c: enc[c] for c in COLORS},
"outcome": outcome, "outcome_color": outcome_color, "rescued": rescued,
"saw_blue_meadow": bool(hour == "Valley" and enc["Blue"]),
"saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"])),
"saw_blue_meadow": bool(hour == "Valley" and (enc["Blue"] or rare == "Blue (valley)")),
"saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"] or rare == "Shadow")),
"rare_reported": (rare or "Unknown") if "rare" in C else "(not asked)",
"malek_intent": (g("malek") or "Unknown") if "malek" in C else "(not asked)",
"tail": g("tail"),
"data_flags": "; ".join(flags),
"death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")),
}