Reported and Reddit-comment runs now get the checks the community form gets in core/atlas/coding.py, as data_flags (the run is kept): - first dragon outside the documented pool for the company/direction - a bond or burn by a dragon the encounter does not have (new encounter_colours table in core/spec/form.yaml; Orange/Brown after the two Greens allowed, as a Green's rescue) - a Green on a valley run - a cadet outcome off its scream path, fight answers or a post-fight path on a run that did not help, a second fight move without assessing - died before the hour, yet reached the tree/valley or an encounter planner/policy.yaml drop_flagged: flagged runs of sources read from free text (reddit comment) stay out of the route model; provenance notes do not count as flags. No effect today: the source is not in `sources` yet. On the first 82 scraped runs: 6 flagged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
374 lines
18 KiB
Python
374 lines
18 KiB
Python
"""Build: every public source file -> one coded run per row, for data/public/runs_clean.csv.
|
|
|
|
One coder per source (pipeline.yaml `runs:`). Every coder returns rows keyed by the columns of data/dictionary.yaml;
|
|
missing answers get the same "not known" values the community form's coding uses, so every source can be filtered
|
|
the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label.
|
|
"""
|
|
import functools
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
from .common import path, read_jsonl, read_yaml
|
|
from . import report_form
|
|
|
|
DICT = read_yaml("data/dictionary.yaml")
|
|
COLUMNS = list(DICT["columns"])
|
|
COLOURS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
|
|
|
|
# what "not known" is, per column, for sources that did not ask (the community form's coding uses the same)
|
|
NOT_KNOWN = {
|
|
"wing": "Unknown", "squad": "Unknown", "attempt_group": "Unknown", "goal": "(not asked/blank)",
|
|
"company": "Unknown", "direction": "Unknown", "first_sighting": "Unknown", "sighting_action": "Unknown",
|
|
"scream": "Unknown", "resolution": "Unknown", "combat_first": "(not recorded)", "combat_second": "(not recorded)",
|
|
"post_path": "Unknown", "hour": "Unknown", "rare_reported": "(not asked)", "malek_intent": "(not asked)",
|
|
"outcome": "No outcome recorded", "data_flags": "",
|
|
}
|
|
|
|
|
|
def blank_row(**values):
|
|
row = {c: None for c in COLUMNS}
|
|
row.update(values)
|
|
for c, v in NOT_KNOWN.items():
|
|
if row[c] is None:
|
|
row[c] = v
|
|
return row
|
|
|
|
|
|
def attempt_group(n):
|
|
return "First attempt" if n == 1 else "Later attempt" if n and n > 1 else "Unknown"
|
|
|
|
|
|
def encounter_columns(row, intentional=False):
|
|
"""The bonding-grid column of the dragon that ended the run."""
|
|
col = row["outcome_color"]
|
|
if col and not intentional:
|
|
if row["outcome"] == "Bonded":
|
|
row["enc_" + col] = "Bonded!"
|
|
elif row["outcome"] == "Incinerated":
|
|
row["enc_" + col] = "Incinerated"
|
|
|
|
|
|
# ---------------------------------------------------------------- coders, by pipeline.yaml `runs:`
|
|
|
|
def spreadsheet(name, spec):
|
|
"""The community form: core/spec/form.yaml + core/atlas/coding.py, unchanged."""
|
|
from atlas import coding
|
|
df, missing = coding.code_frame(coding.read_export(path(spec["load"])))
|
|
if missing:
|
|
print(f"WARNING: {name}: questions not found in the sheet: {missing}")
|
|
df["attempt"] = df["attempt"].astype("Int64")
|
|
rows = []
|
|
for rec in df.astype(object).where(df.notna(), None).to_dict("records"):
|
|
n = rec.pop("run_id")
|
|
entry = rec.pop("source")
|
|
rows.append({**{c: None for c in COLUMNS}, **rec, "run_id": f"sheet-{n}", "source": spec["run_source"], "entry": entry})
|
|
return rows
|
|
|
|
|
|
def reddit_results(name, spec):
|
|
"""Dragons read from result pages linked in the megathreads: bonded, colour and tail known, route unknown."""
|
|
from .common import pipeline
|
|
comments = pipeline()["sources"].get("reddit_comments")
|
|
told = {r.get("result") for r in _comment_runs(comments)} if comments else set()
|
|
rows = []
|
|
for r in read_yaml(spec["transform"]["manual"])["results"]:
|
|
if r["id"] in told:
|
|
continue # the same run, with its route, is a "reddit comment" row
|
|
row = blank_row(run_id=f"thread-{r['id']}", source=spec["run_source"], outcome="Bonded",
|
|
outcome_color=r["colour"], tail=r["tail"])
|
|
encounter_columns(row)
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def register(name, spec):
|
|
"""The Unofficial Dragon Register: one bonded dragon per row, with the rider's assignment, no route. A rider may
|
|
have logged the same run in the community form, so every row is flagged; nothing can match the two."""
|
|
p = path(spec["load"])
|
|
data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"headers": [], "rows": []}
|
|
|
|
def col(*words):
|
|
return next((h for h in data["headers"] if all(w in h.lower() for w in words)), None)
|
|
|
|
h = {k: col(*w) for k, w in {"when": ("timestamp",), "wing": ("wing",), "section": ("section",), "squad": ("squad",),
|
|
"colour": ("colour",), "tail": ("tail",)}.items()}
|
|
values = {c: DICT["columns"][c].get("values") or [] for c in ("wing", "section", "squad", "outcome_color", "tail")}
|
|
rows = []
|
|
for n, r in enumerate(data["rows"], 1):
|
|
get = lambda k: (str(r.get(h[k])).strip() if h[k] and r.get(h[k]) is not None else None) # noqa: E731
|
|
flags = ["outcome only (register): the same run may also be in the spreadsheet"]
|
|
colour = (get("colour") or "").title() or None
|
|
if colour not in values["outcome_color"]:
|
|
flags.append("colour not recognised")
|
|
colour = None
|
|
tail = (get("tail") or "").replace(" ", "").capitalize() or None
|
|
row = blank_row(run_id=f"register-{n}", source=spec["run_source"], date=(get("when") or "")[:10] or None,
|
|
outcome="Bonded", outcome_color=colour, tail=tail if tail in values["tail"] else None)
|
|
for c in ("wing", "section", "squad"):
|
|
if get(c) and (not values[c] or get(c) in values[c]): # section is free text in the dictionary
|
|
row[c] = get(c)
|
|
encounter_columns(row)
|
|
row["data_flags"] = "; ".join(flags)
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def reports(name, spec):
|
|
"""Runs reported through the "Report a run" issue form (collection/specs/report_form.yaml)."""
|
|
p = path(spec["load"])
|
|
data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"reports": []}
|
|
return [answers_row(f"reported-{rep['number']}", spec["run_source"], rep.get("opened"), rep["answers"])
|
|
for rep in data["reports"]]
|
|
|
|
|
|
def _comment_runs(spec):
|
|
"""data/public/reddit_comments.json with the hand corrections of collection/manual/reddit_comments.yaml:
|
|
`reject` (comment ids), `fix` (id -> answers that replace the read ones; null removes one) and `add` (runs the
|
|
reader missed: {id, thread, posted, answers})."""
|
|
p = path(spec["load"])
|
|
data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"runs": []}
|
|
manual = read_yaml(spec["manual"]) if spec.get("manual") and path(spec["manual"]).exists() else {}
|
|
reject = {str(x) for x in manual.get("reject") or []}
|
|
fix = {str(k): v for k, v in (manual.get("fix") or {}).items()}
|
|
out = []
|
|
for run in list(data["runs"]) + list(manual.get("add") or []):
|
|
if str(run["id"]) in reject or str(run["id"]).split("-")[0] in reject:
|
|
continue
|
|
answers = dict(run["answers"])
|
|
for k, v in (fix.get(str(run["id"])) or {}).items():
|
|
if v is None:
|
|
answers.pop(k, None)
|
|
else:
|
|
answers[k] = str(v)
|
|
out.append({**run, "answers": answers, "checked": str(run["id"]) in fix or run in (manual.get("add") or [])})
|
|
return out
|
|
|
|
|
|
def reddit_comments(name, spec):
|
|
"""Runs riders described in the megathread comments, read into the "Report a run" answers (automatic: every
|
|
row is flagged, and those not checked by hand say so)."""
|
|
rows = []
|
|
for run in _comment_runs(spec):
|
|
rows.append(answers_row(f"comment-{run['id']}", spec["run_source"], run.get("posted"), run["answers"],
|
|
["read from a Reddit comment" + ("" if run["checked"] else " automatically, not checked by hand")]))
|
|
return rows
|
|
|
|
|
|
@functools.lru_cache(maxsize=1)
|
|
def _form_fields():
|
|
return {f["id"]: f for f in report_form.spec()["fields"]}
|
|
|
|
|
|
def answers_row(run_id, source, date, answers, flags=None):
|
|
"""One run from the "Report a run" answers (option labels) -> a coded row."""
|
|
fields = _form_fields()
|
|
vals, flags = {}, list(flags or [])
|
|
for fid, answer in answers.items():
|
|
f = fields.get(fid)
|
|
if not f:
|
|
continue
|
|
if f.get("type") == "integer":
|
|
m = re.search(r"\d+", answer)
|
|
vals[f["column"]] = int(m.group()) if m else None
|
|
elif answer in f["options"]:
|
|
vals[f["column"]] = f["options"][answer]
|
|
else:
|
|
flags.append(f"unrecognised answer to {fid}")
|
|
row = blank_row(run_id=run_id, source=source, date=date, **vals)
|
|
row["attempt_group"] = attempt_group(row["attempt"])
|
|
if row["outcome"] == "Died before the hour" and row["hour"] == "Unknown":
|
|
row["hour"] = "DIED before the hour"
|
|
branch = row["branch"]
|
|
if branch:
|
|
tree = branch in ("Shadow (rare)", "Descend quickly: Red", "Descend quickly: let the Red pass", "Descend slowly: two Greens")
|
|
if row["hour"] == "Unknown":
|
|
row["hour"] = "Tree" if tree else "Valley"
|
|
elif row["hour"] != ("Tree" if tree else "Valley"):
|
|
flags.append("encounter does not match the tree/valley answer")
|
|
row["saw_shadow"] = branch == "Shadow (rare)" if tree else None
|
|
if branch == "Passed the Red: meadow" and row["outcome_color"] == "Blue":
|
|
row["saw_blue_meadow"] = True
|
|
if row["outcome"] == "Bonded" and not row["outcome_color"]:
|
|
flags.append("bonded, colour not given")
|
|
flags += rule_flags(row)
|
|
encounter_columns(row)
|
|
row["data_flags"] = "; ".join(flags)
|
|
return row
|
|
|
|
|
|
# the scream answer each cadet outcome follows (a run cannot end the encounter on another path)
|
|
CADET_PATH = {
|
|
"Killed the bullies yourself": "Investigate & help", "Squadmate's revenge": "Investigate & help",
|
|
"Left bullies to the dragons": "Investigate & help", "DIED in combat": "Investigate & help",
|
|
"Tourniquet": "Ignore the scream", "Left injured cadet": "Ignore the scream", "Took their weapons": "Ignore the scream",
|
|
"Left them to it": "Investigate & leave",
|
|
}
|
|
REACHED = ("Tree", "Valley")
|
|
|
|
|
|
@functools.lru_cache(maxsize=1)
|
|
def _form_rules():
|
|
from atlas import spec as atlas_spec
|
|
f = atlas_spec.form()
|
|
return f["sighting_pool"], {k: set(v) for k, v in f["encounter_colours"].items()}
|
|
|
|
|
|
def rule_flags(row):
|
|
"""The game's own rules, for runs answered through "Report a run": the checks core/atlas/coding.py makes on
|
|
the community form (first-dragon pools, Greens in the valley) plus the encounter's dragons and the cadet path.
|
|
Each broken rule is a flag; the run is kept."""
|
|
pools, enc_colours = _form_rules()
|
|
flags = []
|
|
company, direction, first = row["company"], row["direction"], row["first_sighting"]
|
|
if company in pools and direction in pools[company] and first in COLOURS:
|
|
pool = list(pools[company][direction]) + (list(pools["Alone"][direction]) if row["left_squadmates"] is True else [])
|
|
if first not in pool:
|
|
flags.append(f"first sighting {first} not in documented pool for {company}/{direction}")
|
|
colour, branch = row["outcome_color"], row["branch"]
|
|
if row["outcome"] in ("Bonded", "Incinerated") and colour:
|
|
if branch in enc_colours and colour not in enc_colours[branch]:
|
|
flags.append(f"{colour} at the {branch} encounter, which has only {'/'.join(sorted(enc_colours[branch]))}")
|
|
if row["hour"] == "Valley" and colour == "Green":
|
|
flags.append("Green recorded on a Valley run (no Greens in the valley)")
|
|
scream, res = row["scream"], row["resolution"]
|
|
if scream in ("Investigate & help", "Ignore the scream", "Investigate & leave"):
|
|
if res in CADET_PATH and CADET_PATH[res] != scream:
|
|
flags.append(f"cadet outcome {res!r} does not follow {scream!r}")
|
|
if scream != "Investigate & help" and (row["combat_first"] not in (None, "(not recorded)")
|
|
or row["combat_second"] not in (None, "(not recorded)")):
|
|
flags.append("fight answers on a run that did not help")
|
|
if scream != "Investigate & help" and row["post_path"] in ("Forest", "West", "East"):
|
|
flags.append("a path after the fight on a run without a fight")
|
|
if row["combat_second"] not in (None, "(not recorded)") and row["combat_first"] != "Assess the situation":
|
|
flags.append("a second fight move without pausing to assess")
|
|
died_early = (scream or "").startswith("DIED") or (res or "").startswith("DIED")
|
|
if died_early and (row["hour"] in REACHED or row["outcome"] in ("Bonded", "Incinerated")):
|
|
flags.append("died before the hour, yet reached the tree, the valley or a bonding encounter")
|
|
return flags
|
|
|
|
|
|
def _norm(text):
|
|
from .transform.bot import normalize
|
|
return normalize(text)
|
|
|
|
|
|
def bot_trajectories(spec):
|
|
"""Every published trajectory (data/bot/trajectories-*.jsonl), each attempt once."""
|
|
seen, out = set(), []
|
|
for f in sorted(Path(path(spec["load"])).glob("trajectories-*.jsonl")):
|
|
for t in read_jsonl(f):
|
|
if t["id"] not in seen:
|
|
seen.add(t["id"])
|
|
out.append(t)
|
|
return out
|
|
|
|
|
|
def bot_variables(traj, crosswalk):
|
|
"""A bot attempt -> columns, through the crosswalk in collection/specs/bot.yaml. First match wins."""
|
|
v, unmapped = {}, []
|
|
for s in played(traj["steps"]):
|
|
t = _norm(s["optionText"])
|
|
hit = next((c for c in crosswalk if c[0].search(t)), None)
|
|
if hit:
|
|
v.setdefault(hit[1], hit[2])
|
|
else:
|
|
unmapped.append(s["optionText"])
|
|
# the first dragon is the first tinted scene once the direction is chosen
|
|
if v.get("direction") and not v.get("first_sighting") and s.get("tint"):
|
|
v["first_sighting"] = s["tint"].capitalize()
|
|
return v, unmapped
|
|
|
|
|
|
BURN_REASONS = { # why the bot chose to die, in words (the bot's own reason codes)
|
|
"TREE_NO_SHADOW_AUTO_BURN": "no shadow at the tree",
|
|
"MEADOW_NO_BLUE_AUTO_BURN": "no Blue in the meadow",
|
|
}
|
|
|
|
|
|
def played(steps):
|
|
"""The steps that were actually played. The bot logs a pick it then overrode (its own rules against the atlas's
|
|
advice) as a step whose `to` is not where the run went, followed by the pick it made on the same screen. Those
|
|
superseded picks are dropped: counting them codes a run with choices it never made."""
|
|
return [s for i, s in enumerate(steps)
|
|
if not (i + 1 < len(steps) and steps[i + 1]["scene"] == s["scene"] and s.get("to") != steps[i + 1]["scene"])]
|
|
|
|
|
|
def _seen_after(steps, start, key):
|
|
"""Values the bot saw (step["seen"][key]) from step index `start` on."""
|
|
out = []
|
|
for st in steps[start:]:
|
|
v = (st.get("seen") or {}).get(key)
|
|
if v is not None:
|
|
out += v if isinstance(v, list) else [v]
|
|
return out
|
|
|
|
|
|
def bot(name, spec):
|
|
cw = [(re.compile(rx), col, val) for rx, col, val in read_yaml(spec["transform"]["spec"])["crosswalk"]]
|
|
rows = []
|
|
for traj in bot_trajectories(spec):
|
|
traj = {**traj, "steps": played(traj["steps"])}
|
|
steps = traj["steps"]
|
|
v, _ = bot_variables(traj, cw)
|
|
red = v.pop("red", None)
|
|
v.pop("scream_investigate", None)
|
|
if red == "Let it pass (two Greens)":
|
|
v["branch"] = "Descend quickly: let the Red pass"
|
|
o = traj["outcome"]
|
|
meta = traj.get("meta") or {}
|
|
flags = []
|
|
if o["type"] == "BONDED":
|
|
outcome, colour = "Bonded", (o.get("colour") or "").capitalize() or None
|
|
elif o["type"] == "BURNED":
|
|
last_tint = steps[-1].get("tint") if steps else None
|
|
outcome = "Incinerated" if v.get("hour") else "Died before the hour"
|
|
colour = last_tint.capitalize() if last_tint and v.get("hour") else None
|
|
if o.get("intentional"):
|
|
why = BURN_REASONS.get(o.get("reason") or "", "")
|
|
flags.append("deliberate burn (the bot chose to die" + (f": {why}" if why else "") + ")")
|
|
else: # ABORTED, ENDED_NO_BOND: the run stopped without an ending the bot saw
|
|
outcome, colour = "No outcome recorded", None
|
|
if not steps:
|
|
flags.append("bonded dragon only (the bot logged no steps)")
|
|
target = (meta.get("target") or "").capitalize() or None
|
|
row = blank_row(run_id=traj["id"], source=spec["run_source"], bot_mode=meta.get("decisionMode"),
|
|
date=(meta.get("startedAt") or "")[:10] or None, attempt=meta.get("attempt"),
|
|
goal=target if target in COLOURS else None,
|
|
outcome=outcome, outcome_color=colour, data_flags="; ".join(flags), **v)
|
|
row["attempt_group"] = attempt_group(row["attempt"])
|
|
if target: # the bot never bonds a dragon it was not sent for: it dies on purpose instead (Malek)
|
|
row["malek_intent"] = "Yes"
|
|
if outcome == "Died before the hour" and not o.get("intentional"): # where the game killed it
|
|
if row["scream"] == "Unknown":
|
|
row["scream"] = "DIED at first sighting"
|
|
elif row["scream"] == "Investigate & help" and row["resolution"] == "Unknown":
|
|
row["resolution"] = "DIED in combat"
|
|
else:
|
|
row["hour"] = "DIED before the hour"
|
|
# what the bot saw: the shadow at the treetop, and which dragons waited in the meadow
|
|
shadow = _seen_after(steps, 0, "shadow")
|
|
if row["hour"] == "Tree" and shadow:
|
|
row["saw_shadow"] = any(shadow)
|
|
if row["branch"] == "Passed the Red: meadow":
|
|
i = next((k for k, st in enumerate(steps) if re.search(r"no way this is your dragon", _norm(st["optionText"]))), len(steps))
|
|
meadow = _seen_after(steps, i + 1, "dragons")
|
|
if meadow or o.get("reason") == "MEADOW_NO_BLUE_AUTO_BURN":
|
|
row["saw_blue_meadow"] = "blue" in meadow or colour == "Blue"
|
|
encounter_columns(row, intentional=bool(o.get("intentional")))
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
CODERS = {"spreadsheet": spreadsheet, "reddit_results": reddit_results, "reddit_comments": reddit_comments,
|
|
"reports": reports, "bot": bot, "register": register}
|
|
|
|
|
|
def build_runs(pipe):
|
|
rows = []
|
|
for name in pipe["runs"]["order"]:
|
|
spec = pipe["sources"][name]
|
|
if spec.get("runs"):
|
|
rows += CODERS[spec["runs"]](name, spec)
|
|
return rows
|