diff --git a/README.md b/README.md index 3ac3a95..4b15ed9 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,10 @@ Unofficial fan project. Not affiliated with Dragonkind or the Fourth Wing publis The project has four parts, built on one shared core. Each folder has its own README. ``` -core/ the shared model every part uses: the run format and coding rules (engine.js, with the - Python mirror in core/python/clean.py), the route graph and bot crosswalk (graph.js), and the - decision model and bot advisor (solver.js). No dependencies; runs in Node and the browser. +core/ the shared model every part uses. spec/ holds the declarative rules (form.yaml: every question + and answer of the community form); atlas/ is the Python package that reads them (coding.py). + engine.js, graph.js and solver.js are the JavaScript the site and Ken's bot still use; they move + to Python + spec files step by step (docs/ROADMAP.md). collection/ getting data in: sources/ (the form sheet, the Dragon Register, the route PDF check, the daily update job) and bot/ (the DragonKind bot importer). Reddit is not collected yet. analysis/ making sense of it: stats/ (Python statistics, mirrored in core/engine.js), network/ (builds @@ -72,20 +73,22 @@ looks like prose. See `docs/BOT_DATA_PROTOCOL.md` for what the bot should log. In the browser: open the page and use **Load a newer export** with the responses spreadsheet (Google Sheets → File → Download → Microsoft Excel). Everything recomputes. Questions are matched by wording; a renamed question is listed as "not found", and then `COLUMN_KEYS` in -`core/engine.js` and `core/python/clean.py` need updating. +`core/spec/form.yaml` (and, until the site stops coding rows itself, `core/engine.js`) need updating. To update the bundled snapshot, put the new export in `data/raw/form/`, then: ``` python collection/sources/export_raw.py data/raw/form/NEW_EXPORT.xlsx data/public/form_responses.json -python core/python/clean.py data/public/form_responses.json data/derived/runs_clean.csv +PYTHONPATH=core python -m atlas.coding data/public/form_responses.json data/derived/runs_clean.csv npm test ``` ## Changing a coding rule -Every rule lives in `codeRow()` in `core/engine.js` and `code_row()` in -`core/python/clean.py`. Change both and run `npm test`. +The tables (how each question is found, what each answer means, which answers count as missing) are in +`core/spec/form.yaml`; the logic that combines them is `code_row()` in `core/atlas/coding.py`. Add a case to +`tests/core/test_coding.py` for every rule you change, then run `npm test`. Until the site switches to +precomputed statistics, `codeRow()` in `core/engine.js` must be changed the same way (the parity tests check it). ## Statistics in one paragraph diff --git a/analysis/stats/analyze.py b/analysis/stats/analyze.py index d784851..cfe4d39 100644 --- a/analysis/stats/analyze.py +++ b/analysis/stats/analyze.py @@ -14,14 +14,8 @@ import pandas as pd from stats_core import Z95, bh, logistic_ridge, odds_ratio, p_from_z, wilson -COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"] -# answers with no information about the choice: treated as missing everywhere (mirrors MISSING in engine.js) -MISSING = {"Unknown", "Unsure", "Unsure/unknown", "Multiple", "(not recorded)", "(not asked)", - "(not asked/blank)", "Unknown/lost count", "Blank/not asked"} - - -def is_missing(v): - return v is None or (isinstance(v, float) and v != v) or v == "" or v in MISSING +# colours and the answers that count as missing come from core/spec/form.yaml, like everything else +from atlas.coding import COLORS, MISSING, is_missing # noqa: E402 FACTORS = ["company", "direction", "first_sighting", "scream", "resolution", "combat_first", "post_path", "hour", "wing", "squad", "attempt_group", "source", "goal", "sighting_action", "combat_second", "malek_intent"] diff --git a/collection/README.md b/collection/README.md index 2cf10f7..997b501 100644 --- a/collection/README.md +++ b/collection/README.md @@ -5,7 +5,7 @@ under `data/`. | Source | Code | Output | |---|---|---| -| Community "Threshing Routes" form (Google Sheet) | `sources/export_raw.py`, then `core/python/clean.py` | `data/public/form_responses.json`, `data/derived/runs_clean.csv` | +| Community "Threshing Routes" form (Google Sheet) | `sources/export_raw.py`, then `core/atlas/coding.py` | `data/public/form_responses.json`, `data/derived/runs_clean.csv` | | Unofficial Dragon Register (Google Sheet) | `sources/export_register.py` | `data/public/register.json` (rider names stay local) | | Community route PDF | `sources/update_sources.py` | `data/derived/pdf_fingerprint.json` only; the PDF is never committed | | DragonKind bot (Ken) | `bot/import-bot.js`, `bot/bot.js` | `data/bot/scenes.json`, `data/bot/trajectories-.jsonl` | diff --git a/collection/sources/update_sources.py b/collection/sources/update_sources.py index 54c36db..0a8d4fd 100644 --- a/collection/sources/update_sources.py +++ b/collection/sources/update_sources.py @@ -31,6 +31,7 @@ from datetime import datetime, timezone HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.abspath(os.path.join(HERE, "..", "..")) sys.path.insert(0, HERE) +sys.path.insert(0, os.path.join(ROOT, "core")) from export_raw import cell, load_raw # noqa: E402 P = lambda *a: os.path.join(ROOT, *a) # noqa: E731 @@ -74,7 +75,8 @@ def update_form(src, report): report.append(f"- **Threshing Routes form:** no change ({len(rows)} responses).") return False subprocess.run([sys.executable, P("collection", "sources", "export_raw.py"), src, out], check=True, cwd=HERE) - subprocess.run([sys.executable, P("core", "python", "clean.py"), out, P("data", "derived", "runs_clean.csv")], check=True, cwd=HERE) + from atlas import coding # noqa: E402 + coding.main([out, P("data", "derived", "runs_clean.csv")]) raw_copy = P("data", "raw", "form", "Threshing_Routes_Responses.xlsx") if os.path.abspath(src) != raw_copy: # local copy only; data/raw/form/*.xlsx is git-ignored os.makedirs(os.path.dirname(raw_copy), exist_ok=True) @@ -84,7 +86,7 @@ def update_form(src, report): report.append(f"- **Threshing Routes form:** {len(current['rows'])} → {len(rows)} responses " f"({added} new or edited, {removed} removed or edited).") if new_q: - report.append(f" - New questions (check `COLUMN_KEYS` in engine.js and clean.py): {', '.join(new_q)}") + report.append(f" - New questions (add them to `core/spec/form.yaml`): {', '.join(new_q)}") if gone_q: report.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}") return True diff --git a/core/README.md b/core/README.md index 64d3865..5d4d1e0 100644 --- a/core/README.md +++ b/core/README.md @@ -5,7 +5,8 @@ The shared model. Every other part uses it, so a change here touches everything: | File | What it does | |---|---| | `engine.js` | Turns a row of the community form (any version of it) into one coded run: the run format, every coding rule, missing answers, email redaction. Also the statistics the site shows. | -| `python/clean.py` | The same coding rules in Python. `tests/parity/` fails if the two ever disagree. | +| `spec/form.yaml` | **Declarative.** Every question of the community form (how to find it in an export) and every answer (what it is coded as), the answers that count as missing, email redaction. | +| `atlas/coding.py` | The Python coding rules: reads `form.yaml`, codes every row (`python -m atlas.coding`). Rule tests: `tests/core/test_coding.py`. | | `graph.js` | The route graph: the bot's scene graph, the decision-variable graph shared by all sources, the crosswalk from the bot's on-screen text to form answers, Beta credible intervals. | | `solver.js` | The decision model: every choice and roll in a run, chance estimates with empirical-Bayes back-off, backward induction for each dragon, and the advisor the site and the bot use to get advice from any point in a run. | @@ -13,4 +14,6 @@ All three JS files have no dependencies and run unchanged in Node and the browse the published copies (`https://ekcschiano.github.io/threshing-atlas/solver.js` etc.), so their file names on the site must not change. -To change a coding rule, edit `codeRow()` in `engine.js` **and** `code_row()` in `python/clean.py`. +To support a new answer, add it to `spec/form.yaml`. To change a rule, edit `atlas/coding.py`, add a test case, and (until the site stops coding rows in the browser) mirror it in `codeRow()` in `engine.js`; `tests/parity/` fails if they disagree. + +The project is moving to Python plus declarative spec files; JavaScript stays only where it must run in the browser or in Ken's bot (see `docs/ROADMAP.md`). diff --git a/core/atlas/__init__.py b/core/atlas/__init__.py new file mode 100644 index 0000000..1de0d99 --- /dev/null +++ b/core/atlas/__init__.py @@ -0,0 +1,8 @@ +"""threshing-atlas core: the shared model every part of the project uses. + +The rules and the game live in declarative files under core/spec/ (form.yaml, game.yaml, crosswalk.yaml); +the modules here read them: + + atlas.spec load the spec files + atlas.coding community form rows -> coded runs +""" diff --git a/core/atlas/coding.py b/core/atlas/coding.py new file mode 100644 index 0000000..4fe1e13 --- /dev/null +++ b/core/atlas/coding.py @@ -0,0 +1,306 @@ +"""Community form rows -> one coded run per row. + +The tables (how each question is found, what every answer means, which answers count as missing) live in +core/spec/form.yaml. This module holds the logic that combines answers into a run: which way the rider went, +which branch they took after the hour, how the run ended, and which rows look inconsistent (data_flags). + + python -m atlas.coding [form_responses.json|.xlsx|.csv] [runs_clean.csv] + +Defaults: data/public/form_responses.json -> data/derived/runs_clean.csv +""" +import json +import re +import sys +from datetime import datetime +from pathlib import Path + +import pandas as pd + +from . import spec + +ROOT = Path(__file__).resolve().parents[2] +F = spec.form() +A = F["answers"] +COLORS = F["colours"] +MISSING = frozenset(F["missing_values"]) +ENC = F["encounter_answers"] +_EMAIL = re.compile(F["redact"]["pattern"]) + + +def is_missing(v): + """True for answers that carry no information about the choice (blank, "Unsure", not asked...).""" + return v is None or v == "" or v in MISSING or (isinstance(v, float) and v != v) + + +def redact(v): + """Replace email addresses in free text.""" + return v if v is None else _EMAIL.sub(F["redact"]["replacement"], str(v)) + + +# ---------------------------------------------------------------- reading cells + +def find_columns(cols): + """Map each question key in form.yaml to the export's column header. Returns (columns, missing keys).""" + out = {} + low = {c: str(c).lower() for c in cols} + for key, col in F["columns"].items(): + words = col["match"] + hits = [c for c in cols if all((w[1:] not in low[c]) if w[0] == "!" else (w in low[c]) for w in words)] + if hits: + out[key] = hits[0] + missing = [k for k, col in F["columns"].items() + if k not in out and not col.get("optional") and col.get("replaced_by") not in out] + return out, missing + + +def blank(v): + if v is None or v is pd.NaT: + return True + if not isinstance(v, str): + try: + if pd.isna(v): + return True + except (TypeError, ValueError): + pass + return str(v).strip() in ("", "nan", "NaN", "NaT") + + +def text(v): + return None if blank(v) else str(v).strip() + + +def parse_timestamp(v): + """Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'. + Returns 'YYYY-MM-DD' or None.""" + if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v): + return pd.Timestamp(v).strftime("%Y-%m-%d") + if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date + return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d") + t = text(v) + if not t: + return None + m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t) + if m: + return f"{m.group(1)}-{m.group(2)}-{m.group(3)}" + m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY + if m: + return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}" + return None + + +def parse_attempt(v): + v = text(v) + if v is None: + return None + t = v.lower() + if "lost count" in t: + return None + for w, n in F["attempt_words"].items(): + if t.startswith(w): + return n + m = re.search(r"\d+", t) + return int(m.group()) if m else None + + +# ---------------------------------------------------------------- one row -> one run + +def code_row(r, C): + """Code one export row. r: mapping header -> cell; C: from find_columns().""" + g = lambda k: text(r[C[k]]) if k in C else None # noqa: E731 + flags = [] + ts = r[C["timestamp"]] if "timestamp" in C else None + d = parse_timestamp(ts) + source, date = ("Form", d) if d else ((text(ts) or "Unknown import"), None) + + wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)] + wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown") + section = g("wing_" + wings[0]) if len(wings) == 1 else None + if len(wings) > 1: + flags.append("multiple wings") + + if "first_dir" in C: + # Redesigned form: one direction question and one colour question. The older per-direction colour + # columns are still in the sheet for some rows and are used only to fill gaps and to cross-check. + direction = A["direction"].get(g("first_dir")) + first = g("first_colour") + legacy = [d for d in ("West", "Center") if g("leg_" + d)] + if not direction and len(legacy) == 1: + direction = legacy[0] + flags.append("direction taken from the older sighting column") + leg = g("leg_" + direction) if direction else None + if not first and leg: + first = leg + elif first and leg and first != leg and first != "Unsure" and leg != "Unsure": + flags.append("first sighting differs between the old and new columns") + if not direction: + direction = "Unknown" + flags.append("direction missing") + else: + dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)] + if len(dirs) == 1: + direction, first = dirs[0], g("dir_" + dirs[0]) + else: + direction, first = ("Multiple" if dirs else "Unknown"), None + flags.append("direction " + ("multiple" if dirs else "missing")) + + company_raw = g("company") or "Unknown" + suffix = F["company_left_suffix"] + left_squad = company_raw.endswith(suffix) + company = company_raw[: -len(suffix)] if left_squad else company_raw + pools = F["sighting_pool"] + pool = list(pools.get(company, {}).get(direction, [])) + (list(pools["Alone"].get(direction, [])) if left_squad else []) + if first and first != "Unsure" and pool and first not in pool: + flags.append(f"first sighting {first} not in documented pool for {company}/{direction}") + + cadet_raw = g("cadet") + combat = combat2 = None + if "cadet_actions" in C: + act = g("cadet_actions") + a = A["cadet_actions"].get(act, ["Unknown", None, None, "Unknown"]) + if act and act not in A["cadet_actions"]: + flags.append("unrecognised cadet answer") + scream, combat, combat2 = a[0], a[1], a[2] + out = A["cadet_outcome"].get(cadet_raw) + if a[3] is not None: + resolution = a[3] + if out == "DIED in combat" and not resolution.startswith("DIED"): + flags.append("died in combat on a path without combat") + else: + resolution = out or "Unknown" + else: + scream, resolution = A["cadet_legacy"].get(cadet_raw, ["Unknown", "Unknown"]) + if cadet_raw and cadet_raw not in A["cadet_legacy"]: + flags.append("unrecognised cadet answer") + combat = A["combat_legacy"].get(g("combat")) if g("combat") else None + + post_raw = g("post") + if scream == "Investigate & help" and not resolution.startswith("DIED"): + post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown" + elif scream in ("Ignore the scream", "Investigate & leave"): + post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch + else: + post = None + + hour = A["hour"].get(g("hour")) + died_early = scream.startswith("DIED") or resolution.startswith("DIED") + if died_early: + hour = None + elif hour is None: + hour = "Unknown" + + enc = {c: g("enc_" + c) for c in COLORS} + rare = A["rare"].get(g("rare")) + bonded = [c for c in COLORS if enc[c] == ENC["bonded"]] + burned = [c for c in COLORS if enc[c] == ENC["burned"]] + + # Which encounter branch did the run take after the hour? + branch = None + if hour == "Tree": + if enc["Blue"] or enc["Black"] or rare == "Shadow": + branch = "Shadow (rare)" + elif enc["Green"]: + branch = "Descend slowly: two Greens" + elif enc["Red"]: + branch = "Descend quickly: Red" + elif enc["Orange"] or enc["Brown"]: + branch = "Descend slowly: two Greens" + flags.append("tree run with Orange/Brown but no Green recorded") + elif hour == "Valley": + if enc["Green"]: + flags.append("Green recorded on a Valley run (no Greens in the valley)") + meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]] or rare == "Blue (valley)" + if meadow or enc["Red"] == ENC["passed"]: + branch = "Passed the Red: meadow" + elif enc["Red"]: + branch = "Valley Red" + + # How the run ended + outcome, outcome_color = None, None + if died_early: + outcome = "Died before the hour" + elif len(bonded) == 1: + outcome, outcome_color = "Bonded", bonded[0] + elif len(bonded) > 1: + outcome = "Ambiguous" + flags.append("more than one dragon marked Bonded") + elif burned: + order = F["burn_order"]["valley" if hour == "Valley" else "tree"] + outcome, outcome_color = "Incinerated", [c for c in order if c in burned][-1] + if len(burned) > 1: + flags.append("more than one dragon marked Incinerated") + elif hour == "DIED before the hour": + outcome = "Died before the hour" + else: + outcome = "No outcome recorded" + + rescued = enc["Green"] == ENC["rescued"] and outcome == "Bonded" and outcome_color in ("Orange", "Brown") + attempt = parse_attempt(g("attempt")) + if (rare == "Blue (valley)" and hour == "Tree") or (rare == "Shadow" and hour == "Valley"): + flags.append("rare-dragon answer does not match the path") + goal_raw = g("goal_v2") + goal = g("goal") or (A["goal_v2"].get(goal_raw, goal_raw) if goal_raw and goal_raw != "Unknown" else None) or "(not asked/blank)" + return { + "source": source, "date": date, "wing": wing, "section": section, + "squad": g("squad") or "Unknown", "attempt": attempt, + # "I've lost count" means many attempts, so it is a later attempt, not missing + "attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if (attempt and attempt > 1) or "lost count" in (g("attempt") or "").lower() else "Unknown"), + "goal": goal, "company": company, "left_squadmates": left_squad, + "direction": direction, "first_sighting": first or "Unknown", + "sighting_action": (A["sighting_action"].get(g("sighting_action")) or g("sighting_action") or "Unknown") if "sighting_action" in C else "(not asked)", + "scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)", + "combat_second": combat2 or "(not recorded)", + "post_path": post, "hour": hour, "branch": branch, + **{"enc_" + c: enc[c] for c in COLORS}, + "outcome": outcome, "outcome_color": outcome_color, "rescued": rescued, + "saw_blue_meadow": bool(hour == "Valley" and (enc["Blue"] or rare == "Blue (valley)")), + "saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"] or rare == "Shadow")), + "rare_reported": (rare or "Unknown") if "rare" in C else "(not asked)", + "malek_intent": (g("malek") or "Unknown") if "malek" in C else "(not asked)", + "tail": g("tail"), + "data_flags": "; ".join(flags), + "death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")), + } + + +# ---------------------------------------------------------------- files + +def read_export(path): + """An export as a DataFrame: the bundled JSON (data/public/form_responses.json), a CSV or the .xlsx.""" + p = str(path).lower() + if p.endswith(".json"): + d = json.load(open(path, encoding="utf-8")) + return pd.DataFrame(d["rows"], columns=d["headers"]) + if p.endswith(".csv"): + return pd.read_csv(path) + xl = pd.ExcelFile(path) + sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0]) + return pd.read_excel(xl, sheet) + + +def code_frame(raw): + """Code every row of an export. Returns (runs DataFrame with run_id, missing question keys).""" + C, missing = find_columns(raw.columns) + df = pd.DataFrame([code_row(r, C) for _, r in raw.iterrows()]) + df.insert(0, "run_id", range(1, len(df) + 1)) + return df, missing + + +def load(path): + return code_frame(read_export(path)) + + +def main(argv=None): + argv = sys.argv[1:] if argv is None else argv + src = argv[0] if argv else ROOT / "data/public/form_responses.json" + out = argv[1] if len(argv) > 1 else ROOT / "data/derived/runs_clean.csv" + df, missing = load(src) + if missing: + print("WARNING: columns not found:", missing) + df.to_csv(out, index=False) + print(f"{len(df)} runs -> {out}") + print(df.outcome.value_counts().to_string()) + print("rows with flags:", (df["data_flags"] != "").sum()) + + +if __name__ == "__main__": + main() diff --git a/core/atlas/spec.py b/core/atlas/spec.py new file mode 100644 index 0000000..f26c54d --- /dev/null +++ b/core/atlas/spec.py @@ -0,0 +1,18 @@ +"""Load the declarative spec files in core/spec/.""" +from functools import lru_cache +from pathlib import Path + +import yaml + +SPEC_DIR = Path(__file__).resolve().parent.parent / "spec" + + +@lru_cache(maxsize=None) +def load(name): + """The parsed contents of core/spec/.yaml (cached; treat as read-only).""" + with open(SPEC_DIR / f"{name}.yaml", encoding="utf-8") as f: + return yaml.safe_load(f) + + +def form(): + return load("form") diff --git a/core/python/clean.py b/core/python/clean.py deleted file mode 100644 index f8d01db..0000000 --- a/core/python/clean.py +++ /dev/null @@ -1,402 +0,0 @@ -""" -clean.py — turn the raw "Threshing Routes Responses" Google Form export into one -coded row per threshing run. - -Reference implementation in Python. The web page (core/engine.js) implements the -same rules in JavaScript; tests/parity/compare_runs.py checks that both give identical output. - -Usage: - python clean.py ../../data/raw/form/Threshing_Routes_Responses.xlsx -> writes ../../data/derived/runs_clean.csv - -Every coding rule is listed in RULES below and in the "Methods" section of the page. -""" -import re -import sys -from datetime import datetime - -import pandas as pd - -EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") - - -def redact(v): - """Free-text answers sometimes contain a respondent's email address; never display or export it.""" - return v if v is None else EMAIL_RE.sub("[email removed]", str(v)) - -COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"] - -# Columns are found by keywords in the question text, so the script keeps working -# if the form owner reorders or adds questions. -COLUMN_KEYS = { - "timestamp": ["timestamp"], - "wing_First": ["wing and section", "[first]"], - "wing_Second": ["wing and section", "[second]"], - "wing_Third": ["wing and section", "[third]"], - "wing_Fourth": ["wing and section", "[fourth]"], - "squad": ["what squad"], - "attempt": ["attempt number"], - "company": ["who accompanied"], - "dir_East": ["initial direction", "[east"], - "dir_West": ["initial direction", "[west"], - "dir_Center": ["initial direction", "[forest center"], - "cadet": ["outcome of your cadet encounter"], - "post": ["path did you take after the bully"], - "hour": ["after walking an hour"], - "enc_Red": ["attempt to bond", "[red"], - "enc_Green": ["attempt to bond", "[green"], - "enc_Orange": ["attempt to bond", "[orange"], - "enc_Brown": ["attempt to bond", "[brown"], - "enc_Blue": ["attempt to bond", "[blue"], - "enc_Black": ["attempt to bond", "[black"], - "death_note": ["those who died"], - "bond_note": ["those who bonded"], - "goal": ["colour dragon are you trying"], - "combat": ["combat choice"], - # Questions added in the redesigned form (late October 2026). "!x" means the header must NOT contain x. - "first_dir": ["what was your first direction"], - "first_colour": ["colour was your initial dragon sighting", "!["], - "leg_West": ["colour was your initial dragon sighting", "[west"], - "leg_Center": ["colour was your initial dragon sighting", "[forest center"], - "sighting_action": ["action did you take during the first dragon sighting"], - "cadet_actions": ["during your cadet encounter"], - "goal_v2": ["why are you doing this run"], - "malek": ["seek malek"], - "rare": ["encounter a rare dragon"], - "tail": ["what tail does your dragon have"], -} -# A key is only reported missing when neither it nor its replacement in the other form version was found. -ALTERNATIVE = { - "dir_East": "first_dir", "dir_West": "first_dir", "dir_Center": "first_dir", "first_dir": "dir_Center", - "first_colour": "dir_Center", "goal": "goal_v2", "goal_v2": "goal", "combat": "cadet_actions", "cadet_actions": "combat", -} -OPTIONAL = {"leg_West", "leg_Center", "sighting_action", "malek", "rare", "tail"} - -CADET_MAP = { - "Bullies > You dealt with them": ("Investigate & help", "Killed the bullies yourself"), - "Bullies > Squadmate's revenge": ("Investigate & help", "Squadmate's revenge"), - "Bullies > Left to the Dragons": ("Investigate & help", "Left bullies to the dragons"), - "Bullies > Assessed the situation": ("Investigate & help", "Unknown"), - "Injured Cadet > Tourniquet": ("Ignore the scream", "Tourniquet"), - "Injured Cadet > Left them there": ("Ignore the scream", "Left injured cadet"), - "Injured Cadet > Took their weapons": ("Ignore the scream", "Took their weapons"), - "Avoided combat (Investigated scream > Left them to it)": ("Investigate & leave", "Left them to it"), - "You died in combat": ("Investigate & help", "DIED in combat"), - "N/A - You died with initial dragon sighting": ("DIED at first sighting", "DIED at first sighting"), -} -COMBAT_MAP = { - "Bullies > Assessed the situation": "Assess the situation", - "Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)", - "Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)": "Put yourself in the middle (bigger cadet)", -} -# Redesigned form: the cadet encounter is asked as a path of actions plus a separate outcome. -# value: (scream, first combat move, second combat move, resolution when the action decides it) -CADET_ACTIONS = { - "Bullies > Assess > Unknown": ("Investigate & help", "Assess the situation", None, None), - "Bullies > Assess > Smaller Cadet": ("Investigate & help", "Assess the situation", "Engage the smaller cadet", None), - "Bullies > Assess > Daggers or not": ("Investigate & help", "Assess the situation", "Daggers or not (protect squadmate)", None), - "Bullies > Unknown": ("Investigate & help", None, None, None), - "Bullies > Smaller Cadet (Did not assess)": ("Investigate & help", "Even the numbers (smaller cadet)", None, None), - "Bullies > Bigger Cadet (Did not assess)": ("Investigate & help", "Put yourself in the middle (bigger cadet)", None, None), - "Injured Cadet > Tourniquet": ("Ignore the scream", None, None, "Tourniquet"), - "Injured Cadet > Left them there": ("Ignore the scream", None, None, "Left injured cadet"), - "Injured Cadet > Took their weapons": ("Ignore the scream", None, None, "Took their weapons"), - "Avoided combat (Investigated scream > Left them to it)": ("Investigate & leave", None, None, "Left them to it"), - "N/A - You died with initial dragon sighting": ("DIED at first sighting", None, None, "DIED at first sighting"), - "Unsure": ("Unknown", None, None, "Unknown"), -} -CADET_OUTCOME = { - "Squadmate's revenge": "Squadmate's revenge", - "You ended the bullies yourself": "Killed the bullies yourself", - "Left to the Dragons": "Left bullies to the dragons", - "You died in combat": "DIED in combat", -} -DIRECTION_MAP = {"Forest Center": "Center", "West / Scent": "West", "East / Roar": "East"} -GOAL_V2 = { - "Hoping for a Rare Dragon (Blue or Black)": "Blue or Black", - "Trying to bond the Red dragon under the tree": "Red (tree)", - "Daily Run Assignment data": "Daily run data", - "Just here for fun, and maybe a dragon": "Any (for fun)", -} -SIGHTING_ACTION = { - "Hold Ground / Stay Still": "Hold still", "Respect / Face it": "Respect / face it", - "Retreat / Run / Back Away": "Retreat / run", "Hide / Duck": "Hide", "Draw / Drop Weapon": "Weapon", - "Meet Gaze": "Meet its gaze", "Protect / Help Sqaudmate": "Protect squadmate", "Approach": "Approach", - "Stand on your own": "Stand alone", "Unsure": "Unknown", "Unsure - You don't remember": "Unknown", -} -RARE_MAP = {"Yes - A blue dragon in the valley": "Blue (valley)", "Yes - Shadow!!!": "Shadow", "No": "No"} - -HOUR_MAP = { - "Continued to Valley Centre": "Valley", - "Climbed the tallest tree": "Tree", - "N/A - You're dead": "DIED before the hour", -} -# First-sighting pools per (company, direction) according to the community PDF map. -# Used only to FLAG rows that report a dragon outside the documented pool. -SIGHTING_POOL = { - ("Alone", "East"): {"Red", "Orange"}, - ("Alone", "West"): {"Red", "Brown", "Green"}, - ("Alone", "Center"): {"Red", "Brown"}, - ("1 Squadmate", "East"): {"Green", "Orange", "Red"}, # 'leave squadmate' -> alone pools also possible - ("1 Squadmate", "West"): {"Red", "Brown", "Green"}, - ("1 Squadmate", "Center"): {"Red", "Brown"}, - ("2 Squadmates", "East"): {"Red", "Orange"}, - ("2 Squadmates", "West"): {"Green", "Brown", "Red"}, - ("2 Squadmates", "Center"): {"Red", "Brown"}, -} - - -def find_columns(cols): - out = {} - low = {c: str(c).lower() for c in cols} - for key, words in COLUMN_KEYS.items(): - hits = [c for c in cols if all((w[1:] not in low[c]) if w[0] == "!" else (w in low[c]) for w in words)] - if hits: - out[key] = hits[0] - missing = [k for k in COLUMN_KEYS if k not in out and k not in OPTIONAL and ALTERNATIVE.get(k) not in out] - return out, missing - - -def blank(v): - if v is None or v is pd.NaT: - return True - if not isinstance(v, str): - try: - if pd.isna(v): - return True - except (TypeError, ValueError): - pass - return str(v).strip() in ("", "nan", "NaN", "NaT") - - -def s(v): - return None if blank(v) else str(v).strip() - - -def parse_timestamp(v): - """Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'. - Returns 'YYYY-MM-DD' or None.""" - if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v): - return pd.Timestamp(v).strftime("%Y-%m-%d") - if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date - return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d") - t = s(v) - if not t: - return None - m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t) - if m: - return f"{m.group(1)}-{m.group(2)}-{m.group(3)}" - m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY - if m: - return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}" - return None - - -def parse_attempt(v): - v = s(v) - if v is None: - return None - t = v.lower() - if "lost count" in t: - return None - words = {"first": 1, "second": 2, "third": 3, "fourth": 4, "fifth": 5} - for w, n in words.items(): - if t.startswith(w): - return n - m = re.search(r"\d+", t) - return int(m.group()) if m else None - - -def code_row(r, C): - g = lambda k: s(r[C[k]]) if k in C else None - flags = [] - ts = r[C["timestamp"]] if "timestamp" in C else None - d = parse_timestamp(ts) - if d: - source, date = "Form", d - else: - source, date = (s(ts) or "Unknown import"), None - - wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)] - wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown") - section = g("wing_" + wings[0]) if len(wings) == 1 else None - if len(wings) > 1: - flags.append("multiple wings") - - if "first_dir" in C: - # Redesigned form: one direction question and one colour question. The older per-direction colour - # columns are still in the sheet for some rows and are used only to fill gaps and to cross-check. - direction = DIRECTION_MAP.get(g("first_dir")) - first = g("first_colour") - legacy = [d for d in ("West", "Center") if g("leg_" + d)] - if not direction and len(legacy) == 1: - direction = legacy[0] - flags.append("direction taken from the older sighting column") - leg = g("leg_" + direction) if direction else None - if not first and leg: - first = leg - elif first and leg and first != leg and first != "Unsure" and leg != "Unsure": - flags.append("first sighting differs between the old and new columns") - if not direction: - direction = "Unknown" - flags.append("direction missing") - else: - dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)] - if len(dirs) == 1: - direction, first = dirs[0], g("dir_" + dirs[0]) - else: - direction, first = ("Multiple" if dirs else "Unknown"), None - flags.append("direction " + ("multiple" if dirs else "missing")) - - company_raw = g("company") or "Unknown" - left_squad = company_raw.endswith(" > Left them") - company = company_raw[: -len(" > Left them")] if left_squad else company_raw - # Leaving your squadmates sends you into the solo pools, so either pool is plausible. - pool = list(SIGHTING_POOL.get((company, direction), [])) + (list(SIGHTING_POOL.get(("Alone", direction), [])) if left_squad else []) - if first and first != "Unsure" and pool and first not in pool: - flags.append(f"first sighting {first} not in documented pool for {company}/{direction}") - - cadet_raw = g("cadet") - combat = combat2 = None - if "cadet_actions" in C: - act = g("cadet_actions") - a = CADET_ACTIONS.get(act, ("Unknown", None, None, "Unknown")) - if act and act not in CADET_ACTIONS: - flags.append("unrecognised cadet answer") - scream, combat, combat2 = a[0], a[1], a[2] - out = CADET_OUTCOME.get(cadet_raw) - if a[3] is not None: - resolution = a[3] - if out == "DIED in combat" and not resolution.startswith("DIED"): - flags.append("died in combat on a path without combat") - else: - resolution = out or "Unknown" - else: - scream, resolution = CADET_MAP.get(cadet_raw, ("Unknown", "Unknown")) - if cadet_raw and cadet_raw not in CADET_MAP: - flags.append("unrecognised cadet answer") - combat = COMBAT_MAP.get(g("combat")) if g("combat") else None - - post_raw = g("post") - if scream == "Investigate & help" and not resolution.startswith("DIED"): - post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown" - elif scream in ("Ignore the scream", "Investigate & leave"): - post = "(no fork)" - if post_raw in ("Forest", "West", "East"): - post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch - else: - post = None - - hour = HOUR_MAP.get(g("hour")) - died_early = scream.startswith("DIED") or resolution.startswith("DIED") - if died_early: - hour = None - elif hour is None: - hour = "Unknown" - - enc = {c: g("enc_" + c) for c in COLORS} - rare = RARE_MAP.get(g("rare")) - bonded = [c for c in COLORS if enc[c] == "Bonded!"] - burned = [c for c in COLORS if enc[c] == "Incinerated"] - - # Which encounter branch did the run take after the hour? - branch = None - if hour == "Tree": - if enc["Blue"] or enc["Black"] or rare == "Shadow": - branch = "Shadow (rare)" - elif enc["Green"]: - branch = "Descend slowly: two Greens" - elif enc["Red"]: - branch = "Descend quickly: Red" - elif enc["Orange"] or enc["Brown"]: - branch = "Descend slowly: two Greens" - flags.append("tree run with Orange/Brown but no Green recorded") - elif hour == "Valley": - if enc["Green"]: - flags.append("Green recorded on a Valley run (no Greens in the valley)") - red_passed = enc["Red"] in ("Changed my mind",) - meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]] or rare == "Blue (valley)" - if meadow or red_passed: - branch = "Passed the Red: meadow" - elif enc["Red"]: - branch = "Valley Red" - - # Final outcome of the run - outcome, outcome_color = None, None - if died_early: - outcome = "Died before the hour" - elif len(bonded) == 1: - outcome, outcome_color = "Bonded", bonded[0] - elif len(bonded) > 1: - outcome = "Ambiguous" - flags.append("more than one dragon marked Bonded") - elif burned: - order = ["Red", "Green", "Orange", "Brown", "Blue", "Black"] if hour != "Valley" else \ - ["Red", "Orange", "Brown", "Blue", "Green", "Black"] - last = [c for c in order if c in burned][-1] - outcome, outcome_color = "Incinerated", last - if len(burned) > 1: - flags.append("more than one dragon marked Incinerated") - elif hour == "DIED before the hour": - outcome = "Died before the hour" - else: - outcome = "No outcome recorded" - - # The dragon whose bonding encounter decided the run (attempted = Bonded or Incinerated) - rescued = enc["Green"] == "Saved by another" and outcome == "Bonded" and outcome_color in ("Orange", "Brown") - - attempt = parse_attempt(g("attempt")) - if (rare == "Blue (valley)" and hour == "Tree") or (rare == "Shadow" and hour == "Valley"): - flags.append("rare-dragon answer does not match the path") - goal_raw = g("goal_v2") - goal = g("goal") or (GOAL_V2.get(goal_raw, goal_raw) if goal_raw and goal_raw != "Unknown" else None) or "(not asked/blank)" - return { - "source": source, "date": date, "wing": wing, "section": section, - "squad": g("squad") or "Unknown", "attempt": attempt, - # "I've lost count" means many attempts, so it is a later attempt, not missing - "attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if (attempt and attempt > 1) or "lost count" in (g("attempt") or "").lower() else "Unknown"), - "goal": goal, "company": company, "left_squadmates": left_squad, - "direction": direction, "first_sighting": first or "Unknown", - "sighting_action": (SIGHTING_ACTION.get(g("sighting_action")) or g("sighting_action") or "Unknown") if "sighting_action" in C else "(not asked)", - "scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)", - "combat_second": combat2 or "(not recorded)", - "post_path": post, "hour": hour, "branch": branch, - **{"enc_" + c: enc[c] for c in COLORS}, - "outcome": outcome, "outcome_color": outcome_color, "rescued": rescued, - "saw_blue_meadow": bool(hour == "Valley" and (enc["Blue"] or rare == "Blue (valley)")), - "saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"] or rare == "Shadow")), - "rare_reported": (rare or "Unknown") if "rare" in C else "(not asked)", - "malek_intent": (g("malek") or "Unknown") if "malek" in C else "(not asked)", - "tail": g("tail"), - "data_flags": "; ".join(flags), - "death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")), - } - - -def load(path): - if str(path).lower().endswith(".json"): # the page's bundled export (data/public/form_responses.json) - import json - d = json.load(open(path, encoding="utf-8")) - raw = pd.DataFrame(d["rows"], columns=d["headers"]) - elif str(path).lower().endswith(".csv"): - raw = pd.read_csv(path) - else: - xl = pd.ExcelFile(path) - sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0]) - raw = pd.read_excel(xl, sheet) - C, missing = find_columns(raw.columns) - rows = [code_row(r, C) for _, r in raw.iterrows()] - df = pd.DataFrame(rows) - df.insert(0, "run_id", range(1, len(df) + 1)) - return df, missing - - -if __name__ == "__main__": - src = sys.argv[1] if len(sys.argv) > 1 else "../../data/public/form_responses.json" - out = sys.argv[2] if len(sys.argv) > 2 else "../../data/derived/runs_clean.csv" - df, missing = load(src) - if missing: - print("WARNING: columns not found:", missing) - df.to_csv(out, index=False) - print(f"{len(df)} runs -> {out}") - print(df.outcome.value_counts().to_string()) - print("rows with flags:", (df["data_flags"] != "").sum()) diff --git a/core/spec/form.yaml b/core/spec/form.yaml new file mode 100644 index 0000000..d639228 --- /dev/null +++ b/core/spec/form.yaml @@ -0,0 +1,147 @@ +# The community "Threshing Routes" form: how each question is found in an export and how every answer is coded. +# +# This file is the single source of truth for the coding rules' tables. core/atlas/coding.py reads it; the logic +# that combines answers (which branch a run took, how it ended) lives there and is described on the page's +# Methods tab. To support a new or renamed answer, add it here and add a case to tests/core/test_coding.py. +version: 1 + +colours: [Red, Green, Orange, Brown, Blue, Black] + +# Answers that carry no information about the choice. They are kept in the coded data for transparency, but every +# analysis treats them as missing: never shown or compared as if they were a choice. +missing_values: ["Unknown", "Unsure", "Unsure/unknown", "Multiple", "(not recorded)", "(not asked)", + "(not asked/blank)", "Unknown/lost count", "Blank/not asked"] + +# Free-text answers sometimes contain an email address; it is replaced before anything is displayed or exported. +redact: + pattern: '[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}' + replacement: "[email removed]" + +# Columns are found by words in the question text (case-insensitive), so the coding keeps working when the form +# owner reorders, renames slightly or adds questions. A word starting with "!" must NOT appear in the header. +columns: + timestamp: { match: ["timestamp"] } + wing_First: { match: ["wing and section", "[first]"] } + wing_Second: { match: ["wing and section", "[second]"] } + wing_Third: { match: ["wing and section", "[third]"] } + wing_Fourth: { match: ["wing and section", "[fourth]"] } + squad: { match: ["what squad"] } + attempt: { match: ["attempt number"] } + company: { match: ["who accompanied"] } + dir_East: { match: ["initial direction", "[east"], replaced_by: first_dir } + dir_West: { match: ["initial direction", "[west"], replaced_by: first_dir } + dir_Center: { match: ["initial direction", "[forest center"], replaced_by: first_dir } + cadet: { match: ["outcome of your cadet encounter"] } + post: { match: ["path did you take after the bully"] } + hour: { match: ["after walking an hour"] } + enc_Red: { match: ["attempt to bond", "[red"] } + enc_Green: { match: ["attempt to bond", "[green"] } + enc_Orange: { match: ["attempt to bond", "[orange"] } + enc_Brown: { match: ["attempt to bond", "[brown"] } + enc_Blue: { match: ["attempt to bond", "[blue"] } + enc_Black: { match: ["attempt to bond", "[black"] } + death_note: { match: ["those who died"] } + bond_note: { match: ["those who bonded"] } + goal: { match: ["colour dragon are you trying"], replaced_by: goal_v2 } + combat: { match: ["combat choice"], replaced_by: cadet_actions } + # Questions added in the redesigned form (late October 2026) + first_dir: { match: ["what was your first direction"], replaced_by: dir_Center } + first_colour: { match: ["colour was your initial dragon sighting", "!["], replaced_by: dir_Center } + leg_West: { match: ["colour was your initial dragon sighting", "[west"], optional: true } + leg_Center: { match: ["colour was your initial dragon sighting", "[forest center"], optional: true } + sighting_action: { match: ["action did you take during the first dragon sighting"], optional: true } + cadet_actions: { match: ["during your cadet encounter"], replaced_by: combat } + goal_v2: { match: ["why are you doing this run"], replaced_by: goal } + malek: { match: ["seek malek"], optional: true } + rare: { match: ["encounter a rare dragon"], optional: true } + tail: { match: ["what tail does your dragon have"], optional: true } + +# Answer → coded value, one table per question. +answers: + # Older form: one answer for the whole cadet encounter → [scream, resolution] + cadet_legacy: + "Bullies > You dealt with them": ["Investigate & help", "Killed the bullies yourself"] + "Bullies > Squadmate's revenge": ["Investigate & help", "Squadmate's revenge"] + "Bullies > Left to the Dragons": ["Investigate & help", "Left bullies to the dragons"] + "Bullies > Assessed the situation": ["Investigate & help", "Unknown"] + "Injured Cadet > Tourniquet": ["Ignore the scream", "Tourniquet"] + "Injured Cadet > Left them there": ["Ignore the scream", "Left injured cadet"] + "Injured Cadet > Took their weapons": ["Ignore the scream", "Took their weapons"] + "Avoided combat (Investigated scream > Left them to it)": ["Investigate & leave", "Left them to it"] + "You died in combat": ["Investigate & help", "DIED in combat"] + "N/A - You died with initial dragon sighting": ["DIED at first sighting", "DIED at first sighting"] + # Older form: the first move in the fight + combat_legacy: + "Bullies > Assessed the situation": "Assess the situation" + "Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)" + "Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)": "Put yourself in the middle (bigger cadet)" + # Redesigned form: the encounter as a path of actions → [scream, first move, second move, resolution or null] + # (null resolution: the separate outcome question decides it, see cadet_outcome) + cadet_actions: + "Bullies > Assess > Unknown": ["Investigate & help", "Assess the situation", null, null] + "Bullies > Assess > Smaller Cadet": ["Investigate & help", "Assess the situation", "Engage the smaller cadet", null] + "Bullies > Assess > Daggers or not": ["Investigate & help", "Assess the situation", "Daggers or not (protect squadmate)", null] + "Bullies > Unknown": ["Investigate & help", null, null, null] + "Bullies > Smaller Cadet (Did not assess)": ["Investigate & help", "Even the numbers (smaller cadet)", null, null] + "Bullies > Bigger Cadet (Did not assess)": ["Investigate & help", "Put yourself in the middle (bigger cadet)", null, null] + "Injured Cadet > Tourniquet": ["Ignore the scream", null, null, "Tourniquet"] + "Injured Cadet > Left them there": ["Ignore the scream", null, null, "Left injured cadet"] + "Injured Cadet > Took their weapons": ["Ignore the scream", null, null, "Took their weapons"] + "Avoided combat (Investigated scream > Left them to it)": ["Investigate & leave", null, null, "Left them to it"] + "N/A - You died with initial dragon sighting": ["DIED at first sighting", null, null, "DIED at first sighting"] + "Unsure": ["Unknown", null, null, "Unknown"] + cadet_outcome: + "Squadmate's revenge": "Squadmate's revenge" + "You ended the bullies yourself": "Killed the bullies yourself" + "Left to the Dragons": "Left bullies to the dragons" + "You died in combat": "DIED in combat" + direction: + "Forest Center": Center + "West / Scent": West + "East / Roar": East + goal_v2: + "Hoping for a Rare Dragon (Blue or Black)": "Blue or Black" + "Trying to bond the Red dragon under the tree": "Red (tree)" + "Daily Run Assignment data": "Daily run data" + "Just here for fun, and maybe a dragon": "Any (for fun)" + sighting_action: + "Hold Ground / Stay Still": "Hold still" + "Respect / Face it": "Respect / face it" + "Retreat / Run / Back Away": "Retreat / run" + "Hide / Duck": "Hide" + "Draw / Drop Weapon": "Weapon" + "Meet Gaze": "Meet its gaze" + "Protect / Help Sqaudmate": "Protect squadmate" + "Approach": "Approach" + "Stand on your own": "Stand alone" + "Unsure": "Unknown" + "Unsure - You don't remember": "Unknown" + rare: + "Yes - A blue dragon in the valley": "Blue (valley)" + "Yes - Shadow!!!": "Shadow" + "No": "No" + hour: + "Continued to Valley Centre": Valley + "Climbed the tallest tree": Tree + "N/A - You're dead": "DIED before the hour" + +# Suffix on the company answer that means the rider later left their squadmates (redesigned form only). +company_left_suffix: " > Left them" + +# The encounter answers in the "attempt to bond" grid +encounter_answers: { bonded: "Bonded!", burned: "Incinerated", passed: "Changed my mind", rescued: "Saved by another" } + +# When more than one dragon is marked Incinerated, the run ended on the last one met, in this order. +burn_order: + tree: [Red, Green, Orange, Brown, Blue, Black] + valley: [Red, Orange, Brown, Blue, Green, Black] + +# First-dragon pools per company and direction, from the community PDF. Only used to FLAG rows that report a +# dragon outside the documented pool. Leaving your squadmates sends you into the solo pools too. +sighting_pool: + Alone: { East: [Red, Orange], West: [Red, Brown, Green], Center: [Red, Brown] } + 1 Squadmate: { East: [Green, Orange, Red], West: [Red, Brown, Green], Center: [Red, Brown] } + 2 Squadmates: { East: [Red, Orange], West: [Green, Brown, Red], Center: [Red, Brown] } + +# How a bulk-pasted row's attempt answer is read +attempt_words: { first: 1, second: 2, third: 3, fourth: 4, fifth: 5 } diff --git a/docs/PRIVACY.md b/docs/PRIVACY.md index 8f59303..b468253 100644 --- a/docs/PRIVACY.md +++ b/docs/PRIVACY.md @@ -7,7 +7,7 @@ Rules every part of the project follows. If a change would break one, stop and a - **Code** is MIT (`LICENSE`). **Data this project produces** is CC BY 4.0 (`DATA_LICENSE.md`). Third-party sources keep their own terms. - **Community form responses:** published as exported, with email addresses in free-text answers redacted by - the coding rules (`core/engine.js`, `core/python/clean.py`). + the coding rules (`core/spec/form.yaml`, `core/atlas/coding.py`). - **Dragon Register:** only `data/public/register.json`. The raw export has rider names and stays local (`data/raw/register/` is git-ignored). - **The community route PDF:** never committed or redistributed. Only a fingerprint diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index b69b0e8..8b5d13e 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -23,3 +23,17 @@ policy that maximises the chance of bonding it, with honest uncertainty. policy overlay for a chosen colour, and untested branches marked. 6. **Bot loop** (advisor and sync published, see `docs/BOT_INTEGRATION.md`; experiments.json to do): the atlas publishes `policy.json` and `experiments.json` (ranked by value of information); the private DragonKind bot consumes them and returns anonymised runs. + +## Python first, declarative rules (November 2026) + +The repo is moving to Python, with the rules and the game written as declarative spec files. JavaScript stays +only where it has to run: the site's drawing and interaction, and a small route advisor the site and Ken's bot +share. + +1. Coding rules: `core/spec/form.yaml` + `core/atlas/coding.py`, with rule tests. **Done.** +2. Route model in Python, built from `core/spec/game.yaml`; bot text in `core/spec/crosswalk.yaml`. +3. Fix the coding bugs found in the core review (letting the Red pass, "left your squadmates" before the + redesigned form, missing bot text), each with a rule test. +4. The site shows precomputed statistics with fixed filter presets; the upload button and `engine.js` go; a small + JS advisor that reads the Python-built model replaces `solver.js`. +5. `data/dictionary.yaml` for every published column, and Croissant metadata for the datasets. diff --git a/package.json b/package.json index 1d25a26..7b3f6e2 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "build": "node site/build.js", "dev": "node site/build.js && python3 -m http.server 8000 -d dist", "test:js": "node tests/parity/run_engine.js data/public/form_responses.json tests/out", - "test:py": "cd core/python && python3 clean.py && cd ../../analysis/stats && python3 analyze.py", + "test:py": "PYTHONPATH=core python3 -m atlas.coding && cd analysis/stats && PYTHONPATH=../../core python3 analyze.py && cd ../.. && PYTHONPATH=core python3 -m unittest discover -s tests/core -p 'test_*.py'", "test:parity": "python3 tests/parity/compare_runs.py && python3 tests/parity/compare_results.py && python3 tests/parity/compare_routes.py", "test:unit": "node --test tests/core/graph.test.js tests/core/solver.test.js", "test": "npm run test:js && npm run test:py && npm run test:parity && npm run test:unit", diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..493d969 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "threshing-atlas" +version = "0.2.0" +description = "Route probabilities and path finding for the Dragonkind threshing" +license = { text = "MIT" } +requires-python = ">=3.10" +dependencies = ["pandas>=2.0", "numpy>=1.24", "scipy>=1.10", "openpyxl>=3.1", "pypdf>=4.0", "pyyaml>=6.0"] + +[project.optional-dependencies] +dev = ["pytest>=7", "scikit-image>=0.20"] + +[tool.setuptools] +package-dir = { "" = "core" } +packages = ["atlas"] + +[tool.setuptools.package-data] +atlas = [] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["core"] diff --git a/requirements.txt b/requirements.txt index 98c17f3..f6cc88d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ scipy>=1.10 openpyxl>=3.1 pypdf>=4.0 scikit-image>=0.20 +pyyaml>=6.0 +pytest>=7 diff --git a/site/atlas/index.html b/site/atlas/index.html index c00ddd3..d9e2c1d 100644 --- a/site/atlas/index.html +++ b/site/atlas/index.html @@ -618,7 +618,7 @@ a { color: var(--accent); text-underline-offset: 2px; }

The Reddit post is written from the current numbers and filters.

Read the code

-
+

The Python files are an independent second implementation. The test scripts confirm that both produce identical coded runs and identical odds ratios, p-values and model estimates.

diff --git a/site/build.js b/site/build.js index 957cab9..27cb251 100644 --- a/site/build.js +++ b/site/build.js @@ -25,7 +25,8 @@ copy("core/graph.js", "graph.js"); copy("core/solver.js", "solver.js"); for (const f of ["form_responses.json", "register.json", "graph.json", "policy.json"]) copy(`data/public/${f}`, `data/${f}`); // Source files shown in the page's "view the code" panel -copy("core/python/clean.py", "python/clean.py"); +copy("core/atlas/coding.py", "python/coding.py"); +copy("core/spec/form.yaml", "python/form.yaml"); for (const f of ["analyze.py", "stats_core.py"]) copy(`analysis/stats/${f}`, `python/${f}`); copy("README.md", "README.md"); fs.writeFileSync(path.join(out, ".nojekyll"), ""); diff --git a/tests/core/test_coding.py b/tests/core/test_coding.py new file mode 100644 index 0000000..fcfcb56 --- /dev/null +++ b/tests/core/test_coding.py @@ -0,0 +1,132 @@ +"""Rule tests for the coding of community form rows (core/atlas/coding.py + core/spec/form.yaml). + +Each test is one small, hand-written row with the answer it must produce. Unlike the parity tests, these check +that the rules are right, not just that two implementations agree. Run: python -m pytest tests/core +(or: PYTHONPATH=core python -m unittest discover -s tests/core -p 'test_*.py'). +""" +import unittest + +from atlas import coding + +# Question texts as they appear in the exports (shortened, but containing the words form.yaml looks for) +OLD = { + "timestamp": "Timestamp", + "company": "Who accompanied you?", + "dir_East": "Initial direction [East / Roar]", + "dir_West": "Initial direction [West / Scent]", + "dir_Center": "Initial direction [Forest Center]", + "cadet": "What was the outcome of your cadet encounter?", + "combat": "Combat choice", + "post": "Which path did you take after the bully fight?", + "hour": "After walking an hour, what did you do?", + "goal": "Which colour dragon are you trying to bond?", + **{f"enc_{c}": f"Did you attempt to bond? [{c}]" for c in coding.COLORS}, +} +NEW = { + "timestamp": "Timestamp", + "company": "Who accompanied you?", + "first_dir": "What was your first direction?", + "first_colour": "What colour was your initial dragon sighting?", + "sighting_action": "What action did you take during the first dragon sighting?", + "cadet_actions": "What did you do during your cadet encounter?", + "cadet": "What was the outcome of your cadet encounter?", + "post": "Which path did you take after the bully fight?", + "hour": "After walking an hour, what did you do?", + "goal_v2": "Why are you doing this run?", + "malek": "Would you seek Malek?", + **{f"enc_{c}": f"Did you attempt to bond? [{c}]" for c in coding.COLORS}, +} + + +def code(form, **answers): + """Code one row given answers by question key.""" + headers = list(form.values()) + row = {h: None for h in headers} + for k, v in answers.items(): + row[form[k]] = v + C, _ = coding.find_columns(headers) + return coding.code_row(row, C) + + +class Columns(unittest.TestCase): + def test_both_form_versions_are_recognised(self): + for form in (OLD, NEW): + C, missing = coding.find_columns(list(form.values())) + self.assertFalse(set(missing) & set(form), missing) + for key, header in form.items(): + self.assertEqual(C[key], header) + + def test_a_missing_required_question_is_reported(self): + headers = [h for k, h in OLD.items() if k != "hour"] + self.assertIn("hour", coding.find_columns(headers)[1]) + + +class Rows(unittest.TestCase): + def test_form_submission_vs_bulk_import(self): + self.assertEqual(code(OLD, timestamp="2026-10-03 14:02:11")["source"], "Form") + r = code(OLD, timestamp="Data Import") + self.assertEqual((r["source"], r["date"]), ("Data Import", None)) + + def test_old_form_direction_and_first_dragon(self): + r = code(OLD, dir_Center="Brown") + self.assertEqual((r["direction"], r["first_sighting"]), ("Center", "Brown")) + r = code(OLD, dir_Center="Brown", dir_East="Red") + self.assertEqual(r["direction"], "Multiple") + self.assertIn("direction multiple", r["data_flags"]) + + def test_new_form_direction_and_first_dragon(self): + r = code(NEW, first_dir="West / Scent", first_colour="Green", sighting_action="Hold Ground / Stay Still") + self.assertEqual((r["direction"], r["first_sighting"], r["sighting_action"]), ("West", "Green", "Hold still")) + + def test_left_squadmates_suffix(self): + r = code(NEW, company="1 Squadmate > Left them") + self.assertEqual((r["company"], r["left_squadmates"]), ("1 Squadmate", True)) + + def test_old_cadet_answer_and_fight_move(self): + r = code(OLD, cadet="Bullies > Squadmate's revenge", combat="Bullies > Even the numbers (Rushed the smaller cadet)", post="West") + self.assertEqual((r["scream"], r["resolution"], r["combat_first"], r["post_path"]), + ("Investigate & help", "Squadmate's revenge", "Even the numbers (smaller cadet)", "West")) + + def test_new_cadet_path_plus_outcome(self): + r = code(NEW, cadet_actions="Bullies > Assess > Daggers or not", cadet="Left to the Dragons", post="Forest") + self.assertEqual((r["combat_first"], r["combat_second"], r["resolution"]), + ("Assess the situation", "Daggers or not (protect squadmate)", "Left bullies to the dragons")) + + def test_ignoring_the_scream_has_no_fork(self): + r = code(OLD, cadet="Injured Cadet > Tourniquet", post="East") + self.assertEqual((r["scream"], r["resolution"], r["post_path"]), ("Ignore the scream", "Tourniquet", "(no fork)")) + + def test_death_at_the_first_dragon(self): + r = code(OLD, cadet="N/A - You died with initial dragon sighting", hour="Climbed the tallest tree") + self.assertEqual((r["outcome"], r["hour"]), ("Died before the hour", None)) + + def test_tree_branches(self): + self.assertEqual(code(OLD, hour="Climbed the tallest tree", enc_Red="Incinerated")["branch"], "Descend quickly: Red") + self.assertEqual(code(OLD, hour="Climbed the tallest tree", enc_Green="Bonded!")["branch"], "Descend slowly: two Greens") + self.assertEqual(code(OLD, hour="Climbed the tallest tree", enc_Black="Bonded!")["branch"], "Shadow (rare)") + + def test_valley_branches(self): + self.assertEqual(code(OLD, hour="Continued to Valley Centre", enc_Red="Bonded!")["branch"], "Valley Red") + r = code(OLD, hour="Continued to Valley Centre", enc_Red="Changed my mind", enc_Blue="Bonded!") + self.assertEqual((r["branch"], r["outcome"], r["outcome_color"], r["saw_blue_meadow"]), + ("Passed the Red: meadow", "Bonded", "Blue", True)) + + def test_greens_rescue(self): + r = code(OLD, hour="Climbed the tallest tree", enc_Green="Saved by another", enc_Orange="Bonded!") + self.assertTrue(r["rescued"]) + + def test_last_dragon_met_decides_an_incineration(self): + r = code(OLD, hour="Continued to Valley Centre", enc_Red="Incinerated", enc_Orange="Incinerated") + self.assertEqual(r["outcome_color"], "Orange") + + def test_unsure_answers_are_missing(self): + r = code(NEW, cadet_actions="Unsure") + self.assertTrue(coding.is_missing(r["scream"])) + self.assertTrue(coding.is_missing(r["resolution"])) + + def test_emails_are_redacted(self): + self.assertEqual(coding.redact("write to me at rider@example.com"), "write to me at [email removed]") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/parity/compare_runs.py b/tests/parity/compare_runs.py index 846eed6..c285ac7 100644 --- a/tests/parity/compare_runs.py +++ b/tests/parity/compare_runs.py @@ -1,4 +1,4 @@ -"""Check that the JS engine (core/engine.js) and the Python reference (python/clean.py) code every run identically.""" +"""Check that the JS engine (core/engine.js) and the Python coding rules (core/atlas/coding.py) code every run identically.""" import json, sys import pandas as pd js = pd.DataFrame(json.load(open(sys.argv[1] if len(sys.argv) > 1 else "tests/out/runs_js.json")))