Files
threshing-atlas/collection/etl/runs.py
T
ekcSchianoandClaude Opus 5.5 3b24a11f5a Collection as an ETL pipeline; runs_clean.csv from every source; Report a run
- collection/pipeline.yaml declares every source (extract kind, transform,
  privacy rules, load target, run source); `python -m collection update|build`
- extract keeps data/raw/<source>/current + previous only (git-ignored);
  data/manifest.json records the current raw checksum and change date
- transforms are pure; validate checks runs against data/dictionary.yaml
  before load; load writes byte-stable files only when they change
- data/public/runs_clean.csv: every run from every source with a new
  `source` column (reddit spreadsheet, reddit thread, reported, bot,
  reddit pdf); the sheet's own row label moves to `entry`; run_id is text
- bot importer ported to Python (same output as bot.js on the fixture);
  labels and crosswalk in collection/specs/bot.yaml, written into graph.js
- Reddit through the official API (result links to add by hand); the 12
  result pages move to collection/manual/reddit_results.yaml
- "Report a run": a GitHub issue form generated from
  collection/specs/report_form.yaml, collected as source "reported", plus a
  site tab pointing to it, the community spreadsheet and the megathread
- analysis and the route model read runs_clean.csv (sources in
  measures.yaml); stats.json and policy.json unchanged apart from the
  factor's new name
- daily workflow runs the pipeline; the raw store is kept between runs in
  the Actions cache, encrypted with RAW_ARCHIVE_KEY

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-04 02:46:48 +00:00

195 lines
8.1 KiB
Python

"""Build: every public source file -> one coded run per row, for data/public/runs_clean.csv.
One coder per source (pipeline.yaml `runs:`). Every coder returns rows keyed by the columns of data/dictionary.yaml;
missing answers get the same "not known" values the community form's coding uses, so every source can be filtered
the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label.
"""
import json
import re
from pathlib import Path
from .common import path, read_jsonl, read_yaml
from . import report_form
DICT = read_yaml("data/dictionary.yaml")
COLUMNS = list(DICT["columns"])
COLOURS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
# what "not known" is, per column, for sources that did not ask (the community form's coding uses the same)
NOT_KNOWN = {
"wing": "Unknown", "squad": "Unknown", "attempt_group": "Unknown", "goal": "(not asked/blank)",
"company": "Unknown", "direction": "Unknown", "first_sighting": "Unknown", "sighting_action": "Unknown",
"scream": "Unknown", "resolution": "Unknown", "combat_first": "(not recorded)", "combat_second": "(not recorded)",
"post_path": "Unknown", "hour": "Unknown", "rare_reported": "(not asked)", "malek_intent": "(not asked)",
"outcome": "No outcome recorded", "data_flags": "",
}
def blank_row(**values):
row = {c: None for c in COLUMNS}
row.update(values)
for c, v in NOT_KNOWN.items():
if row[c] is None:
row[c] = v
return row
def attempt_group(n):
return "First attempt" if n == 1 else "Later attempt" if n and n > 1 else "Unknown"
def encounter_columns(row, intentional=False):
"""The bonding-grid column of the dragon that ended the run."""
col = row["outcome_color"]
if col and not intentional:
if row["outcome"] == "Bonded":
row["enc_" + col] = "Bonded!"
elif row["outcome"] == "Incinerated":
row["enc_" + col] = "Incinerated"
# ---------------------------------------------------------------- coders, by pipeline.yaml `runs:`
def spreadsheet(name, spec):
"""The community form: core/spec/form.yaml + core/atlas/coding.py, unchanged."""
from atlas import coding
df, missing = coding.code_frame(coding.read_export(path(spec["load"])))
if missing:
print(f"WARNING: {name}: questions not found in the sheet: {missing}")
df["attempt"] = df["attempt"].astype("Int64")
rows = []
for rec in df.astype(object).where(df.notna(), None).to_dict("records"):
n = rec.pop("run_id")
entry = rec.pop("source")
rows.append({**{c: None for c in COLUMNS}, **rec, "run_id": f"sheet-{n}", "source": spec["run_source"], "entry": entry})
return rows
def reddit_results(name, spec):
"""Dragons read from result pages linked in the megathreads: bonded, colour and tail known, route unknown."""
rows = []
for r in read_yaml(spec["transform"]["manual"])["results"]:
row = blank_row(run_id=f"thread-{r['id']}", source=spec["run_source"], outcome="Bonded",
outcome_color=r["colour"], tail=r["tail"])
encounter_columns(row)
rows.append(row)
return rows
def reports(name, spec):
"""Runs reported through the "Report a run" issue form (collection/specs/report_form.yaml)."""
form = report_form.spec()
fields = {f["id"]: f for f in form["fields"]}
p = path(spec["load"])
data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"reports": []}
rows = []
for rep in data["reports"]:
vals, flags = {}, []
for fid, answer in rep["answers"].items():
f = fields.get(fid)
if not f:
continue
if f.get("type") == "integer":
m = re.search(r"\d+", answer)
vals[f["column"]] = int(m.group()) if m else None
elif answer in f["options"]:
vals[f["column"]] = f["options"][answer]
else:
flags.append(f"unrecognised answer to {fid}")
row = blank_row(run_id=f"reported-{rep['number']}", source=spec["run_source"], date=rep.get("opened"), **vals)
row["attempt_group"] = attempt_group(row["attempt"])
if row["outcome"] == "Died before the hour" and row["hour"] == "Unknown":
row["hour"] = "DIED before the hour"
branch = row["branch"]
if branch:
tree = branch in ("Shadow (rare)", "Descend quickly: Red", "Descend quickly: let the Red pass", "Descend slowly: two Greens")
if row["hour"] == "Unknown":
row["hour"] = "Tree" if tree else "Valley"
elif row["hour"] != ("Tree" if tree else "Valley"):
flags.append("encounter does not match the tree/valley answer")
row["saw_shadow"] = branch == "Shadow (rare)" if tree else None
if branch == "Passed the Red: meadow" and row["outcome_color"] == "Blue":
row["saw_blue_meadow"] = True
if row["outcome"] == "Bonded" and not row["outcome_color"]:
flags.append("bonded, colour not given")
encounter_columns(row)
row["data_flags"] = "; ".join(flags)
rows.append(row)
return rows
def _norm(text):
from .transform.bot import normalize
return normalize(text)
def bot_trajectories(spec):
"""Every published trajectory (data/bot/trajectories-*.jsonl), each attempt once."""
seen, out = set(), []
for f in sorted(Path(path(spec["load"])).glob("trajectories-*.jsonl")):
for t in read_jsonl(f):
if t["id"] not in seen:
seen.add(t["id"])
out.append(t)
return out
def bot_variables(traj, crosswalk):
"""A bot attempt -> columns, through the crosswalk in collection/specs/bot.yaml. First match wins."""
v, unmapped = {}, []
for s in traj["steps"]:
t = _norm(s["optionText"])
hit = next((c for c in crosswalk if c[0].search(t)), None)
if hit:
v.setdefault(hit[1], hit[2])
else:
unmapped.append(s["optionText"])
# the first dragon is the first tinted scene once the direction is chosen
if v.get("direction") and not v.get("first_sighting") and s.get("tint"):
v["first_sighting"] = s["tint"].capitalize()
return v, unmapped
def bot(name, spec):
cw = [(re.compile(rx), col, val) for rx, col, val in read_yaml(spec["transform"]["spec"])["crosswalk"]]
rows = []
for traj in bot_trajectories(spec):
v, _ = bot_variables(traj, cw)
red = v.pop("red", None)
v.pop("scream_investigate", None)
if red == "Let it pass (two Greens)":
v["branch"] = "Descend quickly: let the Red pass"
o = traj["outcome"]
meta = traj.get("meta") or {}
flags = []
if o["type"] == "BONDED":
outcome, colour = "Bonded", (o.get("colour") or "").capitalize() or None
elif o["type"] == "BURNED":
last_tint = traj["steps"][-1].get("tint")
outcome = "Incinerated" if v.get("hour") else "Died before the hour"
colour = last_tint.capitalize() if last_tint and v.get("hour") else None
if o.get("intentional"):
flags.append("deliberate burn (the bot chose to die)")
else:
outcome, colour = "No outcome recorded", None
target = (meta.get("target") or "").capitalize() or None
row = blank_row(run_id=traj["id"], source=spec["run_source"], date=(meta.get("startedAt") or "")[:10] or None,
attempt=meta.get("attempt"), goal=target if target in COLOURS else None,
outcome=outcome, outcome_color=colour, data_flags="; ".join(flags), **v)
row["attempt_group"] = attempt_group(row["attempt"])
encounter_columns(row, intentional=bool(o.get("intentional")))
rows.append(row)
return rows
CODERS = {"spreadsheet": spreadsheet, "reddit_results": reddit_results, "reports": reports, "bot": bot}
def build_runs(pipe):
rows = []
for name in pipe["runs"]["order"]:
spec = pipe["sources"][name]
if spec.get("runs"):
rows += CODERS[spec["runs"]](name, spec)
return rows