CI / test (pull_request) Canceled after 0s
- analysis/tree/build_tree.py -> data/public/tree.json, analysis/reports/TREE.md: every screen the bot has seen, every option, where each led (fixed or dice, with ranges), best known picks per colour chosen cautiously, where any picks end, and the untested options. solver.js screenPick(): least-tried and Thompson picks from the tree. - The bot logs a pick it then overrode as a step: 37 runs were coded with choices they never made. Importer and graph.js now use the played steps only. - runs_clean.csv: the Unofficial Dragon Register's 288 bonded dragons are in the master list as outcome-only rows, flagged (no analysis uses them). - Cycle, refresh and the bot kit rebuild and carry the tree; tests check every generated report is current. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
275 lines
12 KiB
Python
275 lines
12 KiB
Python
"""Build: every public source file -> one coded run per row, for data/public/runs_clean.csv.
|
|
|
|
One coder per source (pipeline.yaml `runs:`). Every coder returns rows keyed by the columns of data/dictionary.yaml;
|
|
missing answers get the same "not known" values the community form's coding uses, so every source can be filtered
|
|
the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label.
|
|
"""
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
from .common import path, read_jsonl, read_yaml
|
|
from . import report_form
|
|
|
|
DICT = read_yaml("data/dictionary.yaml")
|
|
COLUMNS = list(DICT["columns"])
|
|
COLOURS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
|
|
|
|
# what "not known" is, per column, for sources that did not ask (the community form's coding uses the same)
|
|
NOT_KNOWN = {
|
|
"wing": "Unknown", "squad": "Unknown", "attempt_group": "Unknown", "goal": "(not asked/blank)",
|
|
"company": "Unknown", "direction": "Unknown", "first_sighting": "Unknown", "sighting_action": "Unknown",
|
|
"scream": "Unknown", "resolution": "Unknown", "combat_first": "(not recorded)", "combat_second": "(not recorded)",
|
|
"post_path": "Unknown", "hour": "Unknown", "rare_reported": "(not asked)", "malek_intent": "(not asked)",
|
|
"outcome": "No outcome recorded", "data_flags": "",
|
|
}
|
|
|
|
|
|
def blank_row(**values):
|
|
row = {c: None for c in COLUMNS}
|
|
row.update(values)
|
|
for c, v in NOT_KNOWN.items():
|
|
if row[c] is None:
|
|
row[c] = v
|
|
return row
|
|
|
|
|
|
def attempt_group(n):
|
|
return "First attempt" if n == 1 else "Later attempt" if n and n > 1 else "Unknown"
|
|
|
|
|
|
def encounter_columns(row, intentional=False):
|
|
"""The bonding-grid column of the dragon that ended the run."""
|
|
col = row["outcome_color"]
|
|
if col and not intentional:
|
|
if row["outcome"] == "Bonded":
|
|
row["enc_" + col] = "Bonded!"
|
|
elif row["outcome"] == "Incinerated":
|
|
row["enc_" + col] = "Incinerated"
|
|
|
|
|
|
# ---------------------------------------------------------------- coders, by pipeline.yaml `runs:`
|
|
|
|
def spreadsheet(name, spec):
|
|
"""The community form: core/spec/form.yaml + core/atlas/coding.py, unchanged."""
|
|
from atlas import coding
|
|
df, missing = coding.code_frame(coding.read_export(path(spec["load"])))
|
|
if missing:
|
|
print(f"WARNING: {name}: questions not found in the sheet: {missing}")
|
|
df["attempt"] = df["attempt"].astype("Int64")
|
|
rows = []
|
|
for rec in df.astype(object).where(df.notna(), None).to_dict("records"):
|
|
n = rec.pop("run_id")
|
|
entry = rec.pop("source")
|
|
rows.append({**{c: None for c in COLUMNS}, **rec, "run_id": f"sheet-{n}", "source": spec["run_source"], "entry": entry})
|
|
return rows
|
|
|
|
|
|
def reddit_results(name, spec):
|
|
"""Dragons read from result pages linked in the megathreads: bonded, colour and tail known, route unknown."""
|
|
rows = []
|
|
for r in read_yaml(spec["transform"]["manual"])["results"]:
|
|
row = blank_row(run_id=f"thread-{r['id']}", source=spec["run_source"], outcome="Bonded",
|
|
outcome_color=r["colour"], tail=r["tail"])
|
|
encounter_columns(row)
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def register(name, spec):
|
|
"""The Unofficial Dragon Register: one bonded dragon per row, with the rider's assignment, no route. A rider may
|
|
have logged the same run in the community form, so every row is flagged; nothing can match the two."""
|
|
p = path(spec["load"])
|
|
data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"headers": [], "rows": []}
|
|
|
|
def col(*words):
|
|
return next((h for h in data["headers"] if all(w in h.lower() for w in words)), None)
|
|
|
|
h = {k: col(*w) for k, w in {"when": ("timestamp",), "wing": ("wing",), "section": ("section",), "squad": ("squad",),
|
|
"colour": ("colour",), "tail": ("tail",)}.items()}
|
|
values = {c: DICT["columns"][c].get("values") or [] for c in ("wing", "section", "squad", "outcome_color", "tail")}
|
|
rows = []
|
|
for n, r in enumerate(data["rows"], 1):
|
|
get = lambda k: (str(r.get(h[k])).strip() if h[k] and r.get(h[k]) is not None else None) # noqa: E731
|
|
flags = ["outcome only (register): the same run may also be in the spreadsheet"]
|
|
colour = (get("colour") or "").title() or None
|
|
if colour not in values["outcome_color"]:
|
|
flags.append("colour not recognised")
|
|
colour = None
|
|
tail = (get("tail") or "").replace(" ", "").capitalize() or None
|
|
row = blank_row(run_id=f"register-{n}", source=spec["run_source"], date=(get("when") or "")[:10] or None,
|
|
outcome="Bonded", outcome_color=colour, tail=tail if tail in values["tail"] else None)
|
|
for c in ("wing", "section", "squad"):
|
|
if get(c) and (not values[c] or get(c) in values[c]): # section is free text in the dictionary
|
|
row[c] = get(c)
|
|
encounter_columns(row)
|
|
row["data_flags"] = "; ".join(flags)
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def reports(name, spec):
|
|
"""Runs reported through the "Report a run" issue form (collection/specs/report_form.yaml)."""
|
|
form = report_form.spec()
|
|
fields = {f["id"]: f for f in form["fields"]}
|
|
p = path(spec["load"])
|
|
data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"reports": []}
|
|
rows = []
|
|
for rep in data["reports"]:
|
|
vals, flags = {}, []
|
|
for fid, answer in rep["answers"].items():
|
|
f = fields.get(fid)
|
|
if not f:
|
|
continue
|
|
if f.get("type") == "integer":
|
|
m = re.search(r"\d+", answer)
|
|
vals[f["column"]] = int(m.group()) if m else None
|
|
elif answer in f["options"]:
|
|
vals[f["column"]] = f["options"][answer]
|
|
else:
|
|
flags.append(f"unrecognised answer to {fid}")
|
|
row = blank_row(run_id=f"reported-{rep['number']}", source=spec["run_source"], date=rep.get("opened"), **vals)
|
|
row["attempt_group"] = attempt_group(row["attempt"])
|
|
if row["outcome"] == "Died before the hour" and row["hour"] == "Unknown":
|
|
row["hour"] = "DIED before the hour"
|
|
branch = row["branch"]
|
|
if branch:
|
|
tree = branch in ("Shadow (rare)", "Descend quickly: Red", "Descend quickly: let the Red pass", "Descend slowly: two Greens")
|
|
if row["hour"] == "Unknown":
|
|
row["hour"] = "Tree" if tree else "Valley"
|
|
elif row["hour"] != ("Tree" if tree else "Valley"):
|
|
flags.append("encounter does not match the tree/valley answer")
|
|
row["saw_shadow"] = branch == "Shadow (rare)" if tree else None
|
|
if branch == "Passed the Red: meadow" and row["outcome_color"] == "Blue":
|
|
row["saw_blue_meadow"] = True
|
|
if row["outcome"] == "Bonded" and not row["outcome_color"]:
|
|
flags.append("bonded, colour not given")
|
|
encounter_columns(row)
|
|
row["data_flags"] = "; ".join(flags)
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def _norm(text):
|
|
from .transform.bot import normalize
|
|
return normalize(text)
|
|
|
|
|
|
def bot_trajectories(spec):
|
|
"""Every published trajectory (data/bot/trajectories-*.jsonl), each attempt once."""
|
|
seen, out = set(), []
|
|
for f in sorted(Path(path(spec["load"])).glob("trajectories-*.jsonl")):
|
|
for t in read_jsonl(f):
|
|
if t["id"] not in seen:
|
|
seen.add(t["id"])
|
|
out.append(t)
|
|
return out
|
|
|
|
|
|
def bot_variables(traj, crosswalk):
|
|
"""A bot attempt -> columns, through the crosswalk in collection/specs/bot.yaml. First match wins."""
|
|
v, unmapped = {}, []
|
|
for s in played(traj["steps"]):
|
|
t = _norm(s["optionText"])
|
|
hit = next((c for c in crosswalk if c[0].search(t)), None)
|
|
if hit:
|
|
v.setdefault(hit[1], hit[2])
|
|
else:
|
|
unmapped.append(s["optionText"])
|
|
# the first dragon is the first tinted scene once the direction is chosen
|
|
if v.get("direction") and not v.get("first_sighting") and s.get("tint"):
|
|
v["first_sighting"] = s["tint"].capitalize()
|
|
return v, unmapped
|
|
|
|
|
|
BURN_REASONS = { # why the bot chose to die, in words (the bot's own reason codes)
|
|
"TREE_NO_SHADOW_AUTO_BURN": "no shadow at the tree",
|
|
"MEADOW_NO_BLUE_AUTO_BURN": "no Blue in the meadow",
|
|
}
|
|
|
|
|
|
def played(steps):
|
|
"""The steps that were actually played. The bot logs a pick it then overrode (its own rules against the atlas's
|
|
advice) as a step whose `to` is not where the run went, followed by the pick it made on the same screen. Those
|
|
superseded picks are dropped: counting them codes a run with choices it never made."""
|
|
return [s for i, s in enumerate(steps)
|
|
if not (i + 1 < len(steps) and steps[i + 1]["scene"] == s["scene"] and s.get("to") != steps[i + 1]["scene"])]
|
|
|
|
|
|
def _seen_after(steps, start, key):
|
|
"""Values the bot saw (step["seen"][key]) from step index `start` on."""
|
|
out = []
|
|
for st in steps[start:]:
|
|
v = (st.get("seen") or {}).get(key)
|
|
if v is not None:
|
|
out += v if isinstance(v, list) else [v]
|
|
return out
|
|
|
|
|
|
def bot(name, spec):
|
|
cw = [(re.compile(rx), col, val) for rx, col, val in read_yaml(spec["transform"]["spec"])["crosswalk"]]
|
|
rows = []
|
|
for traj in bot_trajectories(spec):
|
|
traj = {**traj, "steps": played(traj["steps"])}
|
|
steps = traj["steps"]
|
|
v, _ = bot_variables(traj, cw)
|
|
red = v.pop("red", None)
|
|
v.pop("scream_investigate", None)
|
|
if red == "Let it pass (two Greens)":
|
|
v["branch"] = "Descend quickly: let the Red pass"
|
|
o = traj["outcome"]
|
|
meta = traj.get("meta") or {}
|
|
flags = []
|
|
if o["type"] == "BONDED":
|
|
outcome, colour = "Bonded", (o.get("colour") or "").capitalize() or None
|
|
elif o["type"] == "BURNED":
|
|
last_tint = steps[-1].get("tint") if steps else None
|
|
outcome = "Incinerated" if v.get("hour") else "Died before the hour"
|
|
colour = last_tint.capitalize() if last_tint and v.get("hour") else None
|
|
if o.get("intentional"):
|
|
why = BURN_REASONS.get(o.get("reason") or "", "")
|
|
flags.append("deliberate burn (the bot chose to die" + (f": {why}" if why else "") + ")")
|
|
else: # ABORTED, ENDED_NO_BOND: the run stopped without an ending the bot saw
|
|
outcome, colour = "No outcome recorded", None
|
|
if not steps:
|
|
flags.append("bonded dragon only (the bot logged no steps)")
|
|
target = (meta.get("target") or "").capitalize() or None
|
|
row = blank_row(run_id=traj["id"], source=spec["run_source"], bot_mode=meta.get("decisionMode"),
|
|
date=(meta.get("startedAt") or "")[:10] or None, attempt=meta.get("attempt"),
|
|
goal=target if target in COLOURS else None,
|
|
outcome=outcome, outcome_color=colour, data_flags="; ".join(flags), **v)
|
|
row["attempt_group"] = attempt_group(row["attempt"])
|
|
if target: # the bot never bonds a dragon it was not sent for: it dies on purpose instead (Malek)
|
|
row["malek_intent"] = "Yes"
|
|
if outcome == "Died before the hour" and not o.get("intentional"): # where the game killed it
|
|
if row["scream"] == "Unknown":
|
|
row["scream"] = "DIED at first sighting"
|
|
elif row["scream"] == "Investigate & help" and row["resolution"] == "Unknown":
|
|
row["resolution"] = "DIED in combat"
|
|
else:
|
|
row["hour"] = "DIED before the hour"
|
|
# what the bot saw: the shadow at the treetop, and which dragons waited in the meadow
|
|
shadow = _seen_after(steps, 0, "shadow")
|
|
if row["hour"] == "Tree" and shadow:
|
|
row["saw_shadow"] = any(shadow)
|
|
if row["branch"] == "Passed the Red: meadow":
|
|
i = next((k for k, st in enumerate(steps) if re.search(r"no way this is your dragon", _norm(st["optionText"]))), len(steps))
|
|
meadow = _seen_after(steps, i + 1, "dragons")
|
|
if meadow or o.get("reason") == "MEADOW_NO_BLUE_AUTO_BURN":
|
|
row["saw_blue_meadow"] = "blue" in meadow or colour == "Blue"
|
|
encounter_columns(row, intentional=bool(o.get("intentional")))
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
CODERS = {"spreadsheet": spreadsheet, "reddit_results": reddit_results, "reports": reports, "bot": bot, "register": register}
|
|
|
|
|
|
def build_runs(pipe):
|
|
rows = []
|
|
for name in pipe["runs"]["order"]:
|
|
spec = pipe["sources"][name]
|
|
if spec.get("runs"):
|
|
rows += CODERS[spec["runs"]](name, spec)
|
|
return rows
|