"""Build: every public source file -> one coded run per row, for data/public/runs_clean.csv. One coder per source (pipeline.yaml `runs:`). Every coder returns rows keyed by the columns of data/dictionary.yaml; missing answers get the same "not known" values the community form's coding uses, so every source can be filtered the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label. """ import collections import functools import hashlib import json import re from pathlib import Path from .common import path, read_jsonl, read_yaml from . import report_form DICT = read_yaml("data/dictionary.yaml") COLUMNS = list(DICT["columns"]) COLOURS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"] # what "not known" is, per column, for sources that did not ask (the community form's coding uses the same) NOT_KNOWN = { "wing": "Unknown", "squad": "Unknown", "attempt_group": "Unknown", "goal": "(not asked/blank)", "company": "Unknown", "direction": "Unknown", "first_sighting": "Unknown", "sighting_action": "Unknown", "scream": "Unknown", "resolution": "Unknown", "combat_first": "(not recorded)", "combat_second": "(not recorded)", "post_path": "Unknown", "hour": "Unknown", "rare_reported": "(not asked)", "malek_intent": "(not asked)", "outcome": "No outcome recorded", "data_flags": "", } def blank_row(**values): row = {c: None for c in COLUMNS} row.update(values) for c, v in NOT_KNOWN.items(): if row[c] is None: row[c] = v return row def attempt_group(n): return "First attempt" if n == 1 else "Later attempt" if n and n > 1 else "Unknown" def encounter_columns(row, intentional=False): """The bonding-grid column of the dragon that ended the run.""" col = row["outcome_color"] if col and not intentional: if row["outcome"] == "Bonded": row["enc_" + col] = "Bonded!" elif row["outcome"] == "Incinerated": row["enc_" + col] = "Incinerated" # ---------------------------------------------------------------- coders, by pipeline.yaml `runs:` def fingerprints(records, prefix): """Stable ids for rows that have no id of their own (the form, the register): `-`, so deleting or re-sorting rows in the sheet changes no other row's id, and a column added later (empty in old rows) changes none. Rows that are exact copies of each other get -2, -3... in the order they come; they are identical, so which copy is which does not matter. A row edited in the sheet gets a new id: it is a different record.""" seen = collections.Counter() out = [] for rec in records: answers = {str(k): v for k, v in rec.items() if v is not None and v != "" and v == v} # v == v: not NaN h = hashlib.sha1(json.dumps(answers, sort_keys=True, ensure_ascii=False, default=str).encode("utf-8")).hexdigest()[:12] seen[h] += 1 out.append(f"{prefix}-{h}" + (f"-{seen[h]}" if seen[h] > 1 else "")) return out def spreadsheet(name, spec): """The community form: core/spec/form.yaml + core/atlas/coding.py, plus the game-rule checks every source that answers the form's questions gets (rule_flags; a flag the coder already set is not repeated).""" from atlas import coding raw = coding.read_export(path(spec["load"])) df, missing = coding.code_frame(raw) if missing: print(f"WARNING: {name}: questions not found in the sheet: {missing}") ids = fingerprints(raw.astype(object).where(raw.notna(), None).to_dict("records"), "sheet") df["attempt"] = df["attempt"].astype("Int64") rows = [] for i, rec in enumerate(df.astype(object).where(df.notna(), None).to_dict("records")): rec.pop("run_id") # the coder's row number: shifts when rows are deleted or re-sorted entry = rec.pop("source") row = {**{c: None for c in COLUMNS}, **rec, "run_id": ids[i], "source": spec["run_source"], "entry": entry} have = [f for f in (row["data_flags"] or "").split("; ") if f] row["data_flags"] = "; ".join(have + [f for f in rule_flags(row) if f not in have]) rows.append(row) return rows def reddit_results(name, spec): """Dragons read from result pages linked in the megathreads: bonded, colour and tail known, route unknown.""" from .common import pipeline comments = pipeline()["sources"].get("reddit_comments") told = {r.get("result") for r in _comment_runs(comments)} if comments else set() rows = [] for r in read_yaml(spec["transform"]["manual"])["results"]: if r["id"] in told: continue # the same run, with its route, is a "reddit comment" row row = blank_row(run_id=f"thread-{r['id']}", source=spec["run_source"], outcome="Bonded", outcome_color=r["colour"], tail=r["tail"]) encounter_columns(row) rows.append(row) return rows def register(name, spec): """The Unofficial Dragon Register: one bonded dragon per row, with the rider's assignment, no route. A rider may have logged the same run in the community form, so every row is flagged; nothing can match the two.""" p = path(spec["load"]) data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"headers": [], "rows": []} def col(*words): return next((h for h in data["headers"] if all(w in h.lower() for w in words)), None) h = {k: col(*w) for k, w in {"when": ("timestamp",), "wing": ("wing",), "section": ("section",), "squad": ("squad",), "colour": ("colour",), "tail": ("tail",)}.items()} values = {c: DICT["columns"][c].get("values") or [] for c in ("wing", "section", "squad", "outcome_color", "tail")} rows = [] ids = fingerprints(data["rows"], "register") for n, r in enumerate(data["rows"], 1): get = lambda k: (str(r.get(h[k])).strip() if h[k] and r.get(h[k]) is not None else None) # noqa: E731 flags = ["outcome only (register): the same run may also be in the spreadsheet"] colour = (get("colour") or "").title() or None if colour not in values["outcome_color"]: flags.append("colour not recognised") colour = None tail = (get("tail") or "").replace(" ", "").capitalize() or None row = blank_row(run_id=ids[n - 1], source=spec["run_source"], date=(get("when") or "")[:10] or None, outcome="Bonded", outcome_color=colour, tail=tail if tail in values["tail"] else None) for c in ("wing", "section", "squad"): if get(c) and (not values[c] or get(c) in values[c]): # section is free text in the dictionary row[c] = get(c) encounter_columns(row) row["data_flags"] = "; ".join(flags) rows.append(row) return rows def reports(name, spec): """Runs reported through the "Report a run" issue form (collection/specs/report_form.yaml).""" p = path(spec["load"]) data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"reports": []} return [answers_row(f"reported-{rep['number']}", spec["run_source"], rep.get("opened"), rep["answers"]) for rep in data["reports"]] def _comment_runs(spec): """data/public/reddit_comments.json with the hand corrections of collection/manual/reddit_comments.yaml: `reject` (comment ids), `fix` (id -> answers that replace the read ones; null removes one) and `add` (runs the reader missed: {id, thread, posted, answers}).""" p = path(spec["load"]) data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"runs": []} manual = read_yaml(spec["manual"]) if spec.get("manual") and path(spec["manual"]).exists() else {} reject = {str(x) for x in manual.get("reject") or []} fix = {str(k): v for k, v in (manual.get("fix") or {}).items()} out = [] for run in list(data["runs"]) + list(manual.get("add") or []): if str(run["id"]) in reject or str(run["id"]).split("-")[0] in reject: continue answers = dict(run["answers"]) for k, v in (fix.get(str(run["id"])) or {}).items(): if v is None: answers.pop(k, None) else: answers[k] = str(v) out.append({**run, "answers": answers, "checked": str(run["id"]) in fix or run in (manual.get("add") or [])}) return out def reddit_comments(name, spec): """Runs riders described in the megathread comments, read into the "Report a run" answers (automatic: every row is flagged, and those not checked by hand say so).""" rows = [] for run in _comment_runs(spec): rows.append(answers_row(f"comment-{run['id']}", spec["run_source"], run.get("posted"), run["answers"], ["read from a Reddit comment" + ("" if run["checked"] else " automatically, not checked by hand")])) return rows @functools.lru_cache(maxsize=1) def _form_fields(): return {f["id"]: f for f in report_form.spec()["fields"]} def answers_row(run_id, source, date, answers, flags=None): """One run from the "Report a run" answers (option labels) -> a coded row.""" fields = _form_fields() vals, flags = {}, list(flags or []) for fid, answer in answers.items(): f = fields.get(fid) if not f: continue if f.get("type") == "integer": m = re.search(r"\d+", answer) vals[f["column"]] = int(m.group()) if m else None elif answer in f["options"]: vals[f["column"]] = f["options"][answer] else: flags.append(f"unrecognised answer to {fid}") row = blank_row(run_id=run_id, source=source, date=date, **vals) row["attempt_group"] = attempt_group(row["attempt"]) if row["outcome"] == "Died before the hour" and row["hour"] == "Unknown": row["hour"] = "DIED before the hour" branch = row["branch"] if branch: tree = branch in ("Shadow (rare)", "Descend quickly: Red", "Descend quickly: let the Red pass", "Descend slowly: two Greens") if row["hour"] == "Unknown": row["hour"] = "Tree" if tree else "Valley" elif row["hour"] != ("Tree" if tree else "Valley"): flags.append("encounter does not match the tree/valley answer") row["saw_shadow"] = branch == "Shadow (rare)" if tree else None if branch == "Passed the Red: meadow" and row["outcome_color"] == "Blue": row["saw_blue_meadow"] = True if row["outcome"] == "Bonded" and not row["outcome_color"]: flags.append("bonded, colour not given") flags += rule_flags(row) encounter_columns(row) row["data_flags"] = "; ".join(flags) return row # the scream answer each cadet outcome follows (a run cannot end the encounter on another path) CADET_PATH = { "Killed the bullies yourself": "Investigate & help", "Squadmate's revenge": "Investigate & help", "Left bullies to the dragons": "Investigate & help", "DIED in combat": "Investigate & help", "Tourniquet": "Ignore the scream", "Left injured cadet": "Ignore the scream", "Took their weapons": "Ignore the scream", "Left them to it": "Investigate & leave", } REACHED = ("Tree", "Valley") @functools.lru_cache(maxsize=1) def _form_rules(): from atlas import spec as atlas_spec f = atlas_spec.form() return f["sighting_pool"], {k: set(v) for k, v in f["encounter_colours"].items()} def rule_flags(row): """The game's own rules, for runs answered through "Report a run": the checks core/atlas/coding.py makes on the community form (first-dragon pools, Greens in the valley) plus the encounter's dragons and the cadet path. Each broken rule is a flag; the run is kept.""" pools, enc_colours = _form_rules() flags = [] company, direction, first = row["company"], row["direction"], row["first_sighting"] if company in pools and direction in pools[company] and first in COLOURS: pool = list(pools[company][direction]) + (list(pools["Alone"][direction]) if row["left_squadmates"] is True else []) if first not in pool: flags.append(f"first sighting {first} not in documented pool for {company}/{direction}") colour, branch = row["outcome_color"], row["branch"] if row["outcome"] in ("Bonded", "Incinerated") and colour: if branch in enc_colours and colour not in enc_colours[branch]: flags.append(f"{colour} at the {branch} encounter, which has only {'/'.join(sorted(enc_colours[branch]))}") if row["hour"] == "Valley" and colour == "Green": flags.append("Green recorded on a Valley run (no Greens in the valley)") if row["hour"] == "Valley" and colour == "Black": flags.append("Black recorded on a Valley run (the Black is only reached from the tree)") scream, res = row["scream"], row["resolution"] if scream in ("Investigate & help", "Ignore the scream", "Investigate & leave"): if res in CADET_PATH and CADET_PATH[res] != scream: flags.append(f"cadet outcome {res!r} does not follow {scream!r}") if scream != "Investigate & help" and (row["combat_first"] not in (None, "(not recorded)") or row["combat_second"] not in (None, "(not recorded)")): flags.append("fight answers on a run that did not help") if scream != "Investigate & help" and row["post_path"] in ("Forest", "West", "East"): flags.append("a path after the fight on a run without a fight") if row["combat_second"] not in (None, "(not recorded)") and row["combat_first"] != "Assess the situation": flags.append("a second fight move without pausing to assess") died_early = (scream or "").startswith("DIED") or (res or "").startswith("DIED") if died_early and (row["hour"] in REACHED or row["outcome"] in ("Bonded", "Incinerated")): flags.append("died before the hour, yet reached the tree, the valley or a bonding encounter") return flags def _norm(text): from .transform.bot import normalize return normalize(text) def bot_trajectories(spec): """Every published trajectory (data/bot/trajectories-*.jsonl), each attempt once.""" seen, out = set(), [] for f in sorted(Path(path(spec["load"])).glob("trajectories-*.jsonl")): for t in read_jsonl(f): if t["id"] not in seen: seen.add(t["id"]) out.append(t) return out def bot_variables(traj, crosswalk): """A bot attempt -> columns, through the crosswalk in collection/specs/bot.yaml. First match wins.""" v, unmapped = {}, [] for s in played(traj["steps"]): t = _norm(s["optionText"]) hit = next((c for c in crosswalk if c[0].search(t)), None) if hit: v.setdefault(hit[1], hit[2]) else: unmapped.append(s["optionText"]) # the first dragon is the first tinted scene once the direction is chosen if v.get("direction") and not v.get("first_sighting") and s.get("tint"): v["first_sighting"] = s["tint"].capitalize() return v, unmapped BURN_REASONS = { # why the bot chose to die, in words (the bot's own reason codes) "TREE_NO_SHADOW_AUTO_BURN": "no shadow at the tree", "MEADOW_NO_BLUE_AUTO_BURN": "no Blue in the meadow", } def played(steps): """The steps that were actually played. The bot logs a pick it then overrode (its own rules against the atlas's advice) as a step whose `to` is not where the run went, followed by the pick it made on the same screen. Those superseded picks are dropped: counting them codes a run with choices it never made.""" return [s for i, s in enumerate(steps) if not (i + 1 < len(steps) and steps[i + 1]["scene"] == s["scene"] and s.get("to") != steps[i + 1]["scene"])] def _seen_after(steps, start, key): """Values the bot saw (step["seen"][key]) from step index `start` on.""" out = [] for st in steps[start:]: v = (st.get("seen") or {}).get(key) if v is not None: out += v if isinstance(v, list) else [v] return out def bot(name, spec): cw = [(re.compile(rx), col, val) for rx, col, val in read_yaml(spec["transform"]["spec"])["crosswalk"]] rows = [] for traj in bot_trajectories(spec): traj = {**traj, "steps": played(traj["steps"])} steps = traj["steps"] v, _ = bot_variables(traj, cw) red = v.pop("red", None) v.pop("scream_investigate", None) if red == "Let it pass (two Greens)": v["branch"] = "Descend quickly: let the Red pass" o = traj["outcome"] meta = traj.get("meta") or {} flags = [] if o["type"] == "BONDED": outcome, colour = "Bonded", (o.get("colour") or "").capitalize() or None elif o["type"] == "BURNED": last_tint = steps[-1].get("tint") if steps else None outcome = "Incinerated" if v.get("hour") else "Died before the hour" colour = last_tint.capitalize() if last_tint and v.get("hour") else None if o.get("intentional"): why = BURN_REASONS.get(o.get("reason") or "", "") flags.append("deliberate burn (the bot chose to die" + (f": {why}" if why else "") + ")") else: # ABORTED, ENDED_NO_BOND: the run stopped without an ending the bot saw outcome, colour = "No outcome recorded", None if not steps: flags.append("bonded dragon only (the bot logged no steps)") target = (meta.get("target") or "").capitalize() or None row = blank_row(run_id=traj["id"], source=spec["run_source"], bot_mode=meta.get("decisionMode"), date=(meta.get("startedAt") or "")[:10] or None, attempt=meta.get("attempt"), goal=target if target in COLOURS else None, outcome=outcome, outcome_color=colour, data_flags="; ".join(flags), **v) row["attempt_group"] = attempt_group(row["attempt"]) if target: # the bot never bonds a dragon it was not sent for: it dies on purpose instead (Malek) row["malek_intent"] = "Yes" if outcome == "Died before the hour" and not o.get("intentional"): # where the game killed it if row["scream"] == "Unknown": row["scream"] = "DIED at first sighting" elif row["scream"] == "Investigate & help" and row["resolution"] == "Unknown": row["resolution"] = "DIED in combat" else: row["hour"] = "DIED before the hour" # what the bot saw: the shadow at the treetop, and which dragons waited in the meadow shadow = _seen_after(steps, 0, "shadow") if row["hour"] == "Tree" and shadow: row["saw_shadow"] = any(shadow) if row["branch"] == "Passed the Red: meadow": i = next((k for k, st in enumerate(steps) if re.search(r"no way this is your dragon", _norm(st["optionText"]))), len(steps)) meadow = _seen_after(steps, i + 1, "dragons") if meadow or o.get("reason") == "MEADOW_NO_BLUE_AUTO_BURN": row["saw_blue_meadow"] = "blue" in meadow or colour == "Blue" encounter_columns(row, intentional=bool(o.get("intentional"))) rows.append(row) return rows CODERS = {"spreadsheet": spreadsheet, "reddit_results": reddit_results, "reddit_comments": reddit_comments, "reports": reports, "bot": bot, "register": register} def build_runs(pipe): rows = [] for name in pipe["runs"]["order"]: spec = pipe["sources"][name] if spec.get("runs"): rows += CODERS[spec["runs"]](name, spec) return rows