Collection as an ETL pipeline; runs_clean.csv from every source; Report a run

- collection/pipeline.yaml declares every source (extract kind, transform,
  privacy rules, load target, run source); `python -m collection update|build`
- extract keeps data/raw/<source>/current + previous only (git-ignored);
  data/manifest.json records the current raw checksum and change date
- transforms are pure; validate checks runs against data/dictionary.yaml
  before load; load writes byte-stable files only when they change
- data/public/runs_clean.csv: every run from every source with a new
  `source` column (reddit spreadsheet, reddit thread, reported, bot,
  reddit pdf); the sheet's own row label moves to `entry`; run_id is text
- bot importer ported to Python (same output as bot.js on the fixture);
  labels and crosswalk in collection/specs/bot.yaml, written into graph.js
- Reddit through the official API (result links to add by hand); the 12
  result pages move to collection/manual/reddit_results.yaml
- "Report a run": a GitHub issue form generated from
  collection/specs/report_form.yaml, collected as source "reported", plus a
  site tab pointing to it, the community spreadsheet and the megathread
- analysis and the route model read runs_clean.csv (sources in
  measures.yaml); stats.json and policy.json unchanged apart from the
  factor's new name
- daily workflow runs the pipeline; the raw store is kept between runs in
  the Actions cache, encrypted with RAW_ARCHIVE_KEY

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-04 02:46:48 +00:00
co-authored by Claude Opus 5.5
parent 61bc50583c
commit 3b24a11f5a
64 changed files with 8626 additions and 6909 deletions
View File
+26
View File
@@ -0,0 +1,26 @@
"""Files written from the declarative specs, so each rule has one source of truth:
.github/ISSUE_TEMPLATE/report-a-run.yml from collection/specs/report_form.yaml (report_form.render)
the CROSSWALK block of core/graph.js from collection/specs/bot.yaml (Ken's bot uses it at run time)
"""
import json
from .common import path, read_yaml
GRAPH_JS = "core/graph.js"
BEGIN = " // BEGIN CROSSWALK"
END = " // END CROSSWALK\n"
def graph_js():
"""core/graph.js with its CROSSWALK block rewritten from the bot spec."""
text = path(GRAPH_JS).read_text(encoding="utf-8")
a = text.index(BEGIN)
a = text.index("\n", a) + 1
b = text.index(END, a)
rules = read_yaml("collection/specs/bot.yaml")["crosswalk"]
lines = [" const CROSSWALK = ["]
for i, (rx, col, val) in enumerate(rules):
sep = "," if i < len(rules) - 1 else ""
lines.append(f" [/{rx.replace('/', chr(92) + '/')}/, {json.dumps(col)}, {json.dumps(val, ensure_ascii=False)}]{sep}")
lines.append(" ];")
return text[:a] + "\n".join(lines) + "\n" + text[b:]
+76
View File
@@ -0,0 +1,76 @@
"""Paths, the pipeline spec, and byte-stable writers shared by every stage."""
import hashlib
import json
import sys
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[2]
if str(ROOT / "core") not in sys.path:
sys.path.insert(0, str(ROOT / "core"))
def path(p):
return ROOT / p
def read_yaml(p):
with open(path(p), encoding="utf-8") as f:
return yaml.safe_load(f)
def pipeline():
return read_yaml("collection/pipeline.yaml")
def sha256_bytes(data):
return hashlib.sha256(data).hexdigest()
def sha256_file(p):
h = hashlib.sha256()
with open(p, "rb") as f:
for block in iter(lambda: f.read(1 << 20), b""):
h.update(block)
return h.hexdigest()
def json_bytes(obj, compact=True):
if compact:
return (json.dumps(obj, ensure_ascii=False, separators=(",", ":"))).encode("utf-8")
return (json.dumps(obj, ensure_ascii=False, indent=1) + "\n").encode("utf-8")
def jsonl_bytes(rows):
return "".join(json.dumps(r, ensure_ascii=False, separators=(",", ":")) + "\n" for r in rows).encode("utf-8")
def read_jsonl(p):
p = Path(p)
if not p.exists():
return []
out = []
for line in p.read_text(encoding="utf-8").splitlines():
if line.strip():
try:
out.append(json.loads(line))
except json.JSONDecodeError:
pass
return out
class Report:
"""What a pipeline run did, as Markdown lines (the daily pull request's body)."""
def __init__(self):
self.lines = []
self.changed = False
self.failed = []
def add(self, line):
self.lines.append(line)
print(line)
def text(self, title):
return "\n".join([f"## {title}", ""] + self.lines) + "\n"
+132
View File
@@ -0,0 +1,132 @@
"""Extract: fetch each source as it is now into the raw store. No transformation happens here.
The raw store is data/raw/<source>/ (git-ignored, never published):
current.<ext> the latest version
previous.<ext> the version before it (kept only in case it is needed; the daily workflow keeps it in an
encrypted, private cache). Nothing older is kept and old versions are not hashed.
data/manifest.json records, for each source, the sha256 of the current raw file and the date it last changed.
Each extractor returns the raw bytes, or raises Skip when the source cannot be reached from here (a missing
secret, the bot's private repo not on this machine).
"""
import base64
import json
import os
import shutil
import tempfile
import urllib.parse
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from .common import path, sha256_bytes
USER_AGENT = "threshing-atlas-collector/1.0 (+https://github.com/ekcSchiano/threshing-atlas)"
RAW_DIR = path("data/raw")
class Skip(Exception):
"""The source cannot be extracted here; not an error."""
def _get(url, headers=None, data=None, timeout=60):
req = urllib.request.Request(url, data=data, headers={"User-Agent": USER_AGENT, **(headers or {})})
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.read()
# ---------------------------------------------------------------- extractors, by `kind`
def http(spec, env):
data = _get(spec["url"])
head = data[:15].lower()
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
raise RuntimeError(f"{spec['url']} returned a web page, not a file (is the link still shared publicly?)")
return data
def reddit(spec, env):
"""The threads' comments through the official Reddit API (app-only OAuth: a "script" app's id and secret)."""
cid, secret = (env.get(k) for k in spec["secrets"])
if not (cid and secret):
raise Skip(f"set the {' and '.join(spec['secrets'])} secrets to read the threads through the Reddit API")
auth = base64.b64encode(f"{cid}:{secret}".encode()).decode()
token = json.loads(_get("https://www.reddit.com/api/v1/access_token",
headers={"Authorization": f"Basic {auth}"},
data=b"grant_type=client_credentials"))["access_token"]
threads = {}
for tid in spec["threads"]:
threads[tid] = json.loads(_get(f"https://oauth.reddit.com/comments/{tid}?limit=500&depth=10&raw_json=1",
headers={"Authorization": f"Bearer {token}"}))
return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8")
def github_issues(spec, env):
"""Every issue with the label (open and closed), through the GitHub REST API."""
headers = {"Accept": "application/vnd.github+json"}
token = env.get("GITHUB_TOKEN") or env.get("GH_TOKEN")
if token:
headers["Authorization"] = f"Bearer {token}"
label = urllib.parse.quote(spec["label"])
issues, page = [], 1
while True:
url = f"https://api.github.com/repos/{spec['repo']}/issues?labels={label}&state=all&per_page=100&page={page}"
batch = json.loads(_get(url, headers=headers))
issues += [i for i in batch if "pull_request" not in i]
if len(batch) < 100:
break
page += 1
issues.sort(key=lambda i: i["number"])
return json.dumps(issues, ensure_ascii=False, sort_keys=True).encode("utf-8")
def local_dir(spec, env):
"""The bot's files, from a local copy of its private repo (DRAGONKIND_BOT_DIR), packed as one JSON."""
root = env.get(spec["env"])
if not root:
raise Skip(f"the bot's repo is private: set {spec['env']} to a local copy to import it")
files = {}
for rel in spec["files"]:
p = os.path.join(root, rel)
files[rel] = Path(p).read_text(encoding="utf-8") if os.path.exists(p) else None
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir}
# ---------------------------------------------------------------- the raw store
def raw_paths(name, ext):
d = RAW_DIR / name
return d / f"current.{ext}", d / f"previous.{ext}"
def store(name, ext, data, manifest, today=None):
"""Keep `data` as the current raw file. Returns True when it differs from the last one seen."""
current, previous = raw_paths(name, ext)
digest = sha256_bytes(data)
entry = manifest.get(name) or {}
changed = entry.get("sha256") != digest
current.parent.mkdir(parents=True, exist_ok=True)
on_disk = sha256_bytes(current.read_bytes()) if current.exists() else None
if on_disk and on_disk != digest:
shutil.copyfile(current, previous) # keep one previous version; anything older is gone
if on_disk != digest:
fd, tmp = tempfile.mkstemp(dir=current.parent)
with os.fdopen(fd, "wb") as f:
f.write(data)
os.replace(tmp, current)
if changed:
manifest[name] = {"sha256": digest, "bytes": len(data),
"changed": today or datetime.now(timezone.utc).strftime("%Y-%m-%d")}
return changed
def extract(name, spec, manifest, env=None, local_file=None):
"""Fetch one source (or take a local file) into the raw store. Returns (raw path, changed)."""
env = os.environ if env is None else env
ext = spec["extract"].get("ext") or "json"
data = Path(local_file).read_bytes() if local_file else EXTRACTORS[spec["extract"]["kind"]](spec["extract"], env)
changed = store(name, ext, data, manifest)
return raw_paths(name, ext)[0], changed
+54
View File
@@ -0,0 +1,54 @@
"""Load: the only stage that writes public files. Writes are byte-stable (the same table gives the same bytes)
and only happen when the bytes change, so git shows exactly what changed."""
import csv
import io
import json
from .common import json_bytes, jsonl_bytes, path
def public(obj):
"""Drop transient keys (those starting with "_"), which are for the report only."""
if isinstance(obj, dict):
return {k: public(v) for k, v in obj.items() if not str(k).startswith("_")}
if isinstance(obj, list):
return [public(v) for v in obj]
return obj
def read_json(rel):
p = path(rel)
return json.loads(p.read_text(encoding="utf-8")) if p.exists() else None
def write_bytes(rel, data, check=False):
"""Write when different. Returns True when the file changed (or, with check, would change)."""
p = path(rel)
if p.exists() and p.read_bytes() == data:
return False
if not check:
p.parent.mkdir(parents=True, exist_ok=True)
p.write_bytes(data)
return True
def json_file(rel, obj, check=False, compact=True):
return write_bytes(rel, json_bytes(public(obj), compact), check)
def jsonl_file(rel, rows, check=False):
return write_bytes(rel, jsonl_bytes(rows), check)
def csv_bytes(rows, columns):
"""True/False for booleans, empty for missing, everything else as text."""
buf = io.StringIO()
w = csv.writer(buf, lineterminator="\n")
w.writerow(columns)
for r in rows:
w.writerow(["" if r.get(c) is None else r[c] for c in columns])
return buf.getvalue().encode("utf-8")
def csv_file(rel, rows, columns, check=False):
return write_bytes(rel, csv_bytes(rows, columns), check)
+133
View File
@@ -0,0 +1,133 @@
"""Write data/croissant.json (Croissant 1.0 JSON-LD) from data/dictionary.yaml.
Croissant is the MLCommons metadata format for datasets: it says what each file is, its checksum, and what each
column means, so tools (and people) can load and cite the data without reading this repo's code.
Written by `python -m collection build` (a test checks it is up to date).
"""
import json
from .common import ROOT, pipeline, read_yaml, sha256_file
OUT = "data/croissant.json"
CONTEXT = {
"@language": "en",
"@vocab": "https://schema.org/",
"citeAs": "cr:citeAs",
"column": "cr:column",
"conformsTo": "dct:conformsTo",
"cr": "http://mlcommons.org/croissant/",
"data": {"@id": "cr:data", "@type": "@json"},
"dataType": {"@id": "cr:dataType", "@type": "@vocab"},
"dct": "http://purl.org/dc/terms/",
"examples": {"@id": "cr:examples", "@type": "@json"},
"extract": "cr:extract",
"field": "cr:field",
"fileObject": "cr:fileObject",
"fileProperty": "cr:fileProperty",
"format": "cr:format",
"includes": "cr:includes",
"isLiveDataset": "cr:isLiveDataset",
"jsonPath": "cr:jsonPath",
"key": "cr:key",
"md5": "cr:md5",
"parentField": "cr:parentField",
"path": "cr:path",
"recordSet": "cr:recordSet",
"references": "cr:references",
"regex": "cr:regex",
"repeated": "cr:repeated",
"replace": "cr:replace",
"sc": "https://schema.org/",
"separator": "cr:separator",
"source": "cr:source",
"subField": "cr:subField",
"transform": "cr:transform",
}
# Croissant has no boolean-as-text type; the CSV writes True/False, which sc:Boolean parses.
DATA_TYPES = {"text": "sc:Text", "integer": "sc:Integer", "boolean": "sc:Boolean", "date": "sc:Date"}
sha256 = sha256_file
def load_dictionary():
return read_yaml("data/dictionary.yaml")
def field_description(spec):
text = " ".join(spec["description"].split())
if spec.get("values"):
text += " Values: " + "; ".join(str(v) for v in spec["values"]) + "."
return text
def build(dictionary=None):
d = dictionary or load_dictionary()
ds = d["dataset"]
sources = pipeline()["sources"]
distribution = []
for key, spec in d["files"].items():
path = ROOT / spec["path"]
entry = {
"@type": "cr:FileObject",
"@id": key,
"name": key,
"description": " ".join(spec["description"].split()),
"contentUrl": spec.get("url") or f"{ds['repository']}/blob/main/{spec['path']}",
"encodingFormat": spec["format"],
"sha256": sha256(path),
"contentSize": f"{path.stat().st_size} B",
}
distribution.append(entry)
fields = [
{
"@type": "cr:Field",
"@id": f"runs/{name}",
"name": name,
"description": field_description(spec),
"dataType": DATA_TYPES[spec["type"]],
"source": {"fileObject": {"@id": "runs_clean"}, "extract": {"column": name}},
}
for name, spec in d["columns"].items()
]
return {
"@context": CONTEXT,
"@type": "sc:Dataset",
"conformsTo": "http://mlcommons.org/croissant/1.0",
"name": ds["name"],
"description": " ".join(ds["description"].split()),
"url": ds["url"],
"sameAs": ds["repository"],
"license": ds["license"],
"creator": {"@type": "sc:Person", "name": ds["creator"], "url": f"https://github.com/{ds['creator']}"},
"keywords": ds["keywords"],
"citeAs": ds["citation"],
"isLiveDataset": True,
"isBasedOn": [
{"@type": "sc:CreativeWork", "name": s["label"], "url": next(iter(s["links"].values()))}
for s in sources.values()
# the route PDF is only fingerprinted, never redistributed; the bot's repo is private
if (s.get("privacy") or {}).get("redistribute", True) and s.get("links")
],
"distribution": distribution,
"recordSet": [
{
"@type": "cr:RecordSet",
"@id": "runs",
"name": "runs",
"description": "One coded threshing run per row, from every source (data/public/runs_clean.csv).",
"key": {"@id": "runs/run_id"},
"field": fields,
}
],
}
def render(doc):
return json.dumps(doc, indent=2, ensure_ascii=False) + "\n"
+60
View File
@@ -0,0 +1,60 @@
"""The "Report a run" GitHub issue form, written from collection/specs/report_form.yaml, and the parser for the
issues it creates (GitHub renders each answer as "### <question>" followed by the answer)."""
import re
import yaml
from .common import read_yaml
class _Dumper(yaml.SafeDumper):
pass
def _str(dumper, s):
# multi-line text (the intro) as a readable block
return dumper.represent_scalar("tag:yaml.org,2002:str", s, style="|" if "\n" in s else None)
_Dumper.add_representer(str, _str)
SPEC_PATH = "collection/specs/report_form.yaml"
TEMPLATE = ".github/ISSUE_TEMPLATE/report-a-run.yml"
NOT_SURE = "Not sure"
NO_RESPONSE = {"", "_No response_", "None", NOT_SURE}
def spec():
return read_yaml(SPEC_PATH)
def render(s=None):
s = s or spec()
body = [{"type": "markdown", "attributes": {"value": s["intro"]}}]
for f in s["fields"]:
if f.get("type") == "integer":
item = {"type": "input", "id": f["id"], "attributes": {"label": f["label"], "placeholder": "e.g. 3"}}
else:
opts = list(f["options"])
if not f.get("required"):
opts.append(NOT_SURE)
item = {"type": "dropdown", "id": f["id"], "attributes": {"label": f["label"], "options": opts}}
item["validations"] = {"required": bool(f.get("required"))}
body.append(item)
form = {"name": s["name"], "description": s["description"], "title": s["title"], "labels": [s["label"]], "body": body}
head = f"# Written by collection/etl/report_form.py from {SPEC_PATH}: edit that file, then run `python -m collection build`.\n"
return head + yaml.dump(form, Dumper=_Dumper, sort_keys=False, allow_unicode=True, width=1000)
def parse_body(body, s=None):
"""Issue body -> {field id: chosen option label or text}. Unanswered questions are left out."""
s = s or spec()
by_label = {f["label"].strip(): f["id"] for f in s["fields"]}
out = {}
for block in re.split(r"^### ", body or "", flags=re.M)[1:]:
q, _, a = block.partition("\n")
fid = by_label.get(q.strip())
a = a.strip()
if fid and a not in NO_RESPONSE:
out[fid] = a
return out
+194
View File
@@ -0,0 +1,194 @@
"""Build: every public source file -> one coded run per row, for data/public/runs_clean.csv.
One coder per source (pipeline.yaml `runs:`). Every coder returns rows keyed by the columns of data/dictionary.yaml;
missing answers get the same "not known" values the community form's coding uses, so every source can be filtered
the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label.
"""
import json
import re
from pathlib import Path
from .common import path, read_jsonl, read_yaml
from . import report_form
DICT = read_yaml("data/dictionary.yaml")
COLUMNS = list(DICT["columns"])
COLOURS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
# what "not known" is, per column, for sources that did not ask (the community form's coding uses the same)
NOT_KNOWN = {
"wing": "Unknown", "squad": "Unknown", "attempt_group": "Unknown", "goal": "(not asked/blank)",
"company": "Unknown", "direction": "Unknown", "first_sighting": "Unknown", "sighting_action": "Unknown",
"scream": "Unknown", "resolution": "Unknown", "combat_first": "(not recorded)", "combat_second": "(not recorded)",
"post_path": "Unknown", "hour": "Unknown", "rare_reported": "(not asked)", "malek_intent": "(not asked)",
"outcome": "No outcome recorded", "data_flags": "",
}
def blank_row(**values):
row = {c: None for c in COLUMNS}
row.update(values)
for c, v in NOT_KNOWN.items():
if row[c] is None:
row[c] = v
return row
def attempt_group(n):
return "First attempt" if n == 1 else "Later attempt" if n and n > 1 else "Unknown"
def encounter_columns(row, intentional=False):
"""The bonding-grid column of the dragon that ended the run."""
col = row["outcome_color"]
if col and not intentional:
if row["outcome"] == "Bonded":
row["enc_" + col] = "Bonded!"
elif row["outcome"] == "Incinerated":
row["enc_" + col] = "Incinerated"
# ---------------------------------------------------------------- coders, by pipeline.yaml `runs:`
def spreadsheet(name, spec):
"""The community form: core/spec/form.yaml + core/atlas/coding.py, unchanged."""
from atlas import coding
df, missing = coding.code_frame(coding.read_export(path(spec["load"])))
if missing:
print(f"WARNING: {name}: questions not found in the sheet: {missing}")
df["attempt"] = df["attempt"].astype("Int64")
rows = []
for rec in df.astype(object).where(df.notna(), None).to_dict("records"):
n = rec.pop("run_id")
entry = rec.pop("source")
rows.append({**{c: None for c in COLUMNS}, **rec, "run_id": f"sheet-{n}", "source": spec["run_source"], "entry": entry})
return rows
def reddit_results(name, spec):
"""Dragons read from result pages linked in the megathreads: bonded, colour and tail known, route unknown."""
rows = []
for r in read_yaml(spec["transform"]["manual"])["results"]:
row = blank_row(run_id=f"thread-{r['id']}", source=spec["run_source"], outcome="Bonded",
outcome_color=r["colour"], tail=r["tail"])
encounter_columns(row)
rows.append(row)
return rows
def reports(name, spec):
"""Runs reported through the "Report a run" issue form (collection/specs/report_form.yaml)."""
form = report_form.spec()
fields = {f["id"]: f for f in form["fields"]}
p = path(spec["load"])
data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"reports": []}
rows = []
for rep in data["reports"]:
vals, flags = {}, []
for fid, answer in rep["answers"].items():
f = fields.get(fid)
if not f:
continue
if f.get("type") == "integer":
m = re.search(r"\d+", answer)
vals[f["column"]] = int(m.group()) if m else None
elif answer in f["options"]:
vals[f["column"]] = f["options"][answer]
else:
flags.append(f"unrecognised answer to {fid}")
row = blank_row(run_id=f"reported-{rep['number']}", source=spec["run_source"], date=rep.get("opened"), **vals)
row["attempt_group"] = attempt_group(row["attempt"])
if row["outcome"] == "Died before the hour" and row["hour"] == "Unknown":
row["hour"] = "DIED before the hour"
branch = row["branch"]
if branch:
tree = branch in ("Shadow (rare)", "Descend quickly: Red", "Descend quickly: let the Red pass", "Descend slowly: two Greens")
if row["hour"] == "Unknown":
row["hour"] = "Tree" if tree else "Valley"
elif row["hour"] != ("Tree" if tree else "Valley"):
flags.append("encounter does not match the tree/valley answer")
row["saw_shadow"] = branch == "Shadow (rare)" if tree else None
if branch == "Passed the Red: meadow" and row["outcome_color"] == "Blue":
row["saw_blue_meadow"] = True
if row["outcome"] == "Bonded" and not row["outcome_color"]:
flags.append("bonded, colour not given")
encounter_columns(row)
row["data_flags"] = "; ".join(flags)
rows.append(row)
return rows
def _norm(text):
from .transform.bot import normalize
return normalize(text)
def bot_trajectories(spec):
"""Every published trajectory (data/bot/trajectories-*.jsonl), each attempt once."""
seen, out = set(), []
for f in sorted(Path(path(spec["load"])).glob("trajectories-*.jsonl")):
for t in read_jsonl(f):
if t["id"] not in seen:
seen.add(t["id"])
out.append(t)
return out
def bot_variables(traj, crosswalk):
"""A bot attempt -> columns, through the crosswalk in collection/specs/bot.yaml. First match wins."""
v, unmapped = {}, []
for s in traj["steps"]:
t = _norm(s["optionText"])
hit = next((c for c in crosswalk if c[0].search(t)), None)
if hit:
v.setdefault(hit[1], hit[2])
else:
unmapped.append(s["optionText"])
# the first dragon is the first tinted scene once the direction is chosen
if v.get("direction") and not v.get("first_sighting") and s.get("tint"):
v["first_sighting"] = s["tint"].capitalize()
return v, unmapped
def bot(name, spec):
cw = [(re.compile(rx), col, val) for rx, col, val in read_yaml(spec["transform"]["spec"])["crosswalk"]]
rows = []
for traj in bot_trajectories(spec):
v, _ = bot_variables(traj, cw)
red = v.pop("red", None)
v.pop("scream_investigate", None)
if red == "Let it pass (two Greens)":
v["branch"] = "Descend quickly: let the Red pass"
o = traj["outcome"]
meta = traj.get("meta") or {}
flags = []
if o["type"] == "BONDED":
outcome, colour = "Bonded", (o.get("colour") or "").capitalize() or None
elif o["type"] == "BURNED":
last_tint = traj["steps"][-1].get("tint")
outcome = "Incinerated" if v.get("hour") else "Died before the hour"
colour = last_tint.capitalize() if last_tint and v.get("hour") else None
if o.get("intentional"):
flags.append("deliberate burn (the bot chose to die)")
else:
outcome, colour = "No outcome recorded", None
target = (meta.get("target") or "").capitalize() or None
row = blank_row(run_id=traj["id"], source=spec["run_source"], date=(meta.get("startedAt") or "")[:10] or None,
attempt=meta.get("attempt"), goal=target if target in COLOURS else None,
outcome=outcome, outcome_color=colour, data_flags="; ".join(flags), **v)
row["attempt_group"] = attempt_group(row["attempt"])
encounter_columns(row, intentional=bool(o.get("intentional")))
rows.append(row)
return rows
CODERS = {"spreadsheet": spreadsheet, "reddit_results": reddit_results, "reports": reports, "bot": bot}
def build_runs(pipe):
rows = []
for name in pipe["runs"]["order"]:
spec = pipe["sources"][name]
if spec.get("runs"):
rows += CODERS[spec["runs"]](name, spec)
return rows
+7
View File
@@ -0,0 +1,7 @@
"""Named transforms (pipeline.yaml `transform: {name: ...}`). Each module has
transform(raw_path, spec, ctx) -> the public source file's content (pure: no network, no clock)
describe(old, new) -> report lines comparing the previous public file with the new one
"""
from . import bot, pdf, reddit, reports, sheet
TRANSFORMS = {"sheet": sheet, "pdf_fingerprint": pdf, "bot": bot, "reddit_results": reddit, "reports": reports}
+239
View File
@@ -0,0 +1,239 @@
"""The DragonKind bot's private data -> anonymised trajectories and a scene catalogue (Python port of bot.js).
Raw input (from the private bot repo, packed by extract.local_dir): data/knowledge.json (scene atlas: prose,
choices, tint, backdrop), data/runs.jsonl (one record per attempt) and logs/encounters.jsonl (every decision the
bot made, with the options on screen). Nothing of the prose or the account ids leaves this module: the output
holds scene fingerprints, short labels (collection/specs/bot.yaml), choice text and outcomes, and check_public()
refuses anything else.
Output: {"scenes": {sceneId: {stage, tint, backdrop, label, options: [{id, text}]}},
"trajectories": [{id, source, meta, steps, outcome}], "unlabeled": [...]}
"""
import hashlib
import json
import re
from datetime import datetime
from ..common import read_yaml
SPEC = read_yaml("collection/specs/bot.yaml")
_NORMALIZE = [(re.compile(a), b) for a, b in SPEC["normalize"]]
CHOICE_EVENTS = set(SPEC["choice_events"])
def sha1(s, n=12):
return hashlib.sha1(str(s).encode("utf-8")).hexdigest()[:n]
def normalize(text):
"""Must match normalize() in the bot's src/strategy.js, so fingerprints line up."""
t = str(text or "").lower()
for rx, rep in _NORMALIZE:
t = rx.sub(rep, t)
return t.strip()
def scene_id(prose, choices):
"""Must match computeSceneFingerprint() in the bot's src/learner.js."""
prose_norm = normalize(prose)[:500]
opts = sorted(c for c in (normalize(c) for c in choices) if c)
return sha1(f"{prose_norm} ### {' || '.join(opts)}")
def option_id(text):
return sha1(normalize(text), 10)
def label_for(sid, node):
if sid in SPEC["labels"]:
return SPEC["labels"][sid]
bits = [node.get("stage") or "Scene"]
if node.get("tint"):
bits.append(node["tint"])
return " · ".join(bits)
def _jsonl(text):
out = []
for line in (text or "").splitlines():
if line.strip():
try:
out.append(json.loads(line))
except json.JSONDecodeError:
pass
return out
def build_scenes(knowledge, encounters):
scenes = {}
for sid, n in ((knowledge or {}).get("nodes") or {}).items():
scenes[sid] = {
"stage": n.get("stage") or "UNKNOWN", "tint": n.get("tint"), "backdrop": n.get("backdrop"),
"label": label_for(sid, n),
"options": [{"id": option_id(c.get("label") or c.get("choiceNorm")), "text": c.get("label") or c.get("choiceNorm")}
for c in (n.get("choices") or {}).values()],
}
for e in encounters:
if not isinstance(e.get("choices"), list) or not e.get("prose"):
continue
sid = e.get("nodeId") or scene_id(e["prose"], e["choices"])
if sid in scenes:
continue
stage = (e.get("decision") or {}).get("stage") or e.get("stage") or "UNKNOWN"
scenes[sid] = {"stage": stage, "tint": None, "backdrop": None, "label": label_for(sid, {"stage": stage}),
"options": [{"id": option_id(t), "text": t} for t in e["choices"]]}
return scenes
def trajectories_from_encounters(encounters, scenes):
"""Group the log by attempt, keep the decisions acted on, and close each attempt at its first BURNED or
DRAGON_BONDED event. Repeated BURNED events with no decision between them are ignored."""
by_attempt = {}
for e in encounters:
key = f"{e['account']}:{e.get('attempt')}" if e.get("account") else str(e.get("attempt"))
by_attempt.setdefault(key, []).append(e)
out = []
for key, events in by_attempt.items():
events = sorted(events, key=lambda e: str(e.get("ts")))
steps, outcome = [], None
for e in events:
if outcome:
break
if e.get("event") in CHOICE_EVENTS:
d = e.get("decision") or {}
text = d.get("choiceText") or e.get("choiceText")
if d.get("action") and d["action"] != "choose":
continue # paused, nothing happened
if not text or not isinstance(e.get("choices"), list):
continue
scene = e.get("nodeId") or scene_id(e.get("prose") or "", e["choices"])
manual = e["event"] != "DECISION"
sc = scenes.get(scene) or {}
steps.append({
"scene": scene, "stage": d.get("stage") or sc.get("stage") or "UNKNOWN", "tint": sc.get("tint"),
"option": option_id(text), "optionText": text, "offered": [option_id(c) for c in e["choices"]],
"rule": d.get("rule"), "decisionMode": "manual" if manual else d.get("source"),
"manual": manual, "at": e.get("ts"),
})
elif e.get("event") == "BURNED" and steps:
outcome = {"type": "BURNED", "colour": None, "intentional": bool(e.get("intentional"))}
elif e.get("event") == "DRAGON_BONDED" and steps:
outcome = {"type": "BONDED", "colour": str(e["dragon"]).lower() if e.get("dragon") else None, "intentional": False}
# a decision repeated on the same scene (the bot re-detecting the screen) counts once
dedup = [s for i, s in enumerate(steps)
if not (i > 0 and steps[i - 1]["scene"] == s["scene"] and steps[i - 1]["option"] == s["option"])]
for i, s in enumerate(dedup):
s["to"] = dedup[i + 1]["scene"] if i + 1 < len(dedup) else None
if not dedup:
continue
if outcome:
dedup[-1]["to"] = "BURNED" if outcome["type"] == "BURNED" else f"BONDED:{outcome['colour'] or 'unknown'}"
try:
attempt = int(float(key.split(":")[-1])) or None
except ValueError:
attempt = None
started = dedup[0]["at"]
out.append({
"id": "bot-" + sha1(f"{key}:{started}", 10),
"source": "bot",
"meta": {"attempt": attempt, "startedAt": dedup[0]["at"], "accountKey": None, "target": None, "decisionMode": None},
"steps": dedup,
"outcome": outcome or {"type": "ABORTED", "colour": None, "intentional": False},
})
return out
def _ms(ts):
try:
return datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp() * 1000
except ValueError:
return float("nan")
def attach_run_meta(trajs, runs, salt):
"""Run-level metadata (target colour, mode) from runs.jsonl, where an attempt is in both."""
window = SPEC["match_hours"] * 3600e3
for r in runs:
t = next((x for x in trajs if x["meta"]["attempt"] == r.get("attempt")
and abs(_ms(x["meta"]["startedAt"]) - _ms(r.get("startedAt"))) < window), None)
if not t:
continue
m = t["meta"]
m["target"] = r.get("targetDragon")
m["decisionMode"] = r.get("decisionMode")
m["policyId"] = r.get("policyId")
m["experimentId"] = r.get("experimentId")
m["arm"] = r.get("arm")
if r.get("accountKey"):
m["accountKey"] = r["accountKey"]
elif r.get("accountId") and salt:
m["accountKey"] = sha1(f"{salt}:{r['accountId']}", 10)
if r.get("outcome") == "BONDED" and r.get("dragonColor"):
t["outcome"] = {"type": "BONDED", "colour": str(r["dragonColor"]).lower(), "intentional": False}
if r.get("intentionalBurn"):
t["outcome"]["intentional"] = True
return trajs
def import_bot(knowledge, encounters, runs, salt=None):
scenes = build_scenes(knowledge, encounters)
trajectories = attach_run_meta(trajectories_from_encounters(encounters, scenes), runs, salt)
unlabeled = [f"{sid} ({s['stage']}{', ' + s['tint'] if s['tint'] else ''})" for sid, s in scenes.items() if sid not in SPEC["labels"]]
return scenes, trajectories, unlabeled
def check_public(obj, refuse, max_string):
"""Nothing that looks like game prose or an account identifier may leave this module."""
def walk(v, p):
if isinstance(v, list):
for i, x in enumerate(v):
walk(x, f"{p}[{i}]")
elif isinstance(v, dict):
for k, x in v.items():
if k in refuse:
raise ValueError(f'refusing to export field "{k}" at {p}')
walk(x, f"{p}.{k}")
elif isinstance(v, str) and len(v) > max_string:
raise ValueError(f"refusing to export a {len(v)}-character string at {p} (looks like prose)")
walk(obj, "$")
def merge_scenes(existing, scenes):
"""Add new scenes and new options to the published catalogue; never drop what is already there."""
out = json.loads(json.dumps(existing or {}))
for sid, s in scenes.items():
old = out.get(sid)
if not old:
out[sid] = s
continue
ids = {o["id"] for o in old["options"]}
old["options"] += [o for o in s["options"] if o["id"] not in ids]
old["tint"] = old.get("tint") or s["tint"]
old["backdrop"] = old.get("backdrop") or s["backdrop"]
if "·" in old["label"] and "·" not in s["label"]:
old["label"] = s["label"]
return dict(sorted(out.items()))
def transform(raw_path, spec, ctx):
with open(raw_path, encoding="utf-8") as f:
files = json.load(f)
k = files.get("data/knowledge.json")
knowledge = json.loads(k) if k else {"nodes": {}}
encounters = _jsonl(files.get("logs/encounters.jsonl"))
runs = _jsonl(files.get("data/runs.jsonl"))
privacy = spec.get("privacy") or {}
scenes, trajectories, unlabeled = import_bot(knowledge, encounters, runs, ctx.get("env", {}).get(privacy.get("account_salt_env", "")))
for obj in (scenes, trajectories):
check_public(obj, set(privacy.get("refuse_fields", [])), privacy.get("max_string", 200))
return {"scenes": scenes, "trajectories": trajectories, "_unlabeled": unlabeled}
def describe(old, new):
outcomes = {}
for t in new["trajectories"]:
outcomes[t["outcome"]["type"]] = outcomes.get(t["outcome"]["type"], 0) + 1
lines = [f"{len(new['scenes'])} scenes, {len(new['trajectories'])} attempts {json.dumps(outcomes)}"]
if new.get("_unlabeled"):
lines.append(" - Scenes without a short label (add them to collection/specs/bot.yaml): " + ", ".join(new["_unlabeled"]))
return lines
+51
View File
@@ -0,0 +1,51 @@
"""The route PDF -> its fingerprint only. The PDF is never redistributed: the public file holds hashes of the file,
of each page's text and of each line, so a change can be detected and located without keeping the text."""
import hashlib
def _sha(text, n):
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:n]
def pdf_lines(path):
from pypdf import PdfReader
pages = []
for page in PdfReader(str(path)).pages:
text = page.extract_text() or ""
pages.append([" ".join(line.split()) for line in text.splitlines() if line.strip()])
return pages
def transform(raw_path, spec, ctx):
with open(raw_path, "rb") as f:
data = f.read()
pages = pdf_lines(raw_path)
return {
"sha256": hashlib.sha256(data).hexdigest(),
"bytes": len(data),
"pages": len(pages),
"page_hashes": [_sha("\n".join(p), 16) for p in pages],
"line_hashes": sorted({_sha(line.lower(), 12) for p in pages for line in p}),
"checked": ctx.get("changed"),
# transient: the new lines' text, for the report only (load drops every key starting with "_")
"_lines": [line for p in pages for line in p],
}
def describe(old, new):
if old is None:
return [f"fingerprint recorded for the first time ({new['pages']} pages)"]
if old.get("page_hashes") == new["page_hashes"]:
return ["the file changed but its text did not (layout or images only). Worth a look."]
changed = [i + 1 for i, h in enumerate(new["page_hashes"])
if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
old_lines = set(old["line_hashes"])
new_lines = [line for line in new.get("_lines", []) if _sha(line.lower(), 12) not in old_lines]
lines = [f"changed. {old['pages']} → {new['pages']} pages; pages with new text: {', '.join(map(str, changed)) or 'none'}; "
f"{len(new_lines)} new lines, {len(old_lines - set(new['line_hashes']))} lines gone."]
if new_lines:
lines.append(" - New lines (update core/spec/game.yaml or the crosswalks if they add options):")
lines += [f" - {line}" for line in new_lines[:60]]
if len(new_lines) > 60:
lines.append(f" - … and {len(new_lines) - 60} more")
return lines
+45
View File
@@ -0,0 +1,45 @@
"""Reddit megathreads (raw: the threads' comment trees from the official API) -> dragonkind.com result links that
are not yet in collection/manual/reddit_results.yaml. Usernames and comment text are never kept: only the result
ids and the comment permalinks, for the report, so a person can read the result page and add it by hand."""
import json
import re
from pathlib import Path
from ..common import read_yaml
RESULT_RE = re.compile(r"dragonkind\.com/results/(THRESH-[A-Z0-9]+)", re.I)
def _comments(node):
"""Every comment in a listing tree."""
if isinstance(node, list):
for x in node:
yield from _comments(x)
elif isinstance(node, dict):
data = node.get("data") or {}
if node.get("kind") in ("t1", "t3"):
yield data
for key in ("children", "replies"):
if isinstance(data.get(key), (dict, list)):
yield from _comments(data[key])
def transform(raw_path, spec, ctx):
threads = json.loads(Path(raw_path).read_text(encoding="utf-8"))
known = {r["id"].upper() for r in read_yaml(spec["transform"]["manual"])["results"]}
found = {}
for tid, tree in sorted(threads.items()):
for c in _comments(tree):
for m in RESULT_RE.finditer((c.get("body") or "") + " " + (c.get("selftext") or "") + " " + (c.get("url") or "")):
rid = m.group(1).upper()
if rid not in known and rid not in found:
found[rid] = "https://www.reddit.com" + (c.get("permalink") or f"/comments/{tid}/")
return {"new_results": [{"id": k, "comment": v} for k, v in sorted(found.items())]}
def describe(old, new):
rs = new["new_results"]
if not rs:
return ["no new result links in the threads"]
return [f"{len(rs)} result link(s) not yet in collection/manual/reddit_results.yaml (read each page, then add it):"] + \
[f" - https://dragonkind.com/results/{r['id']} (posted at {r['comment']})" for r in rs[:40]]
+31
View File
@@ -0,0 +1,31 @@
"""Run reports (raw: the "run report" issues from the GitHub API) -> data/public/reports.json.
Kept per report: the issue number, the date it was opened, and the answers (the option labels chosen). GitHub
usernames and any other text are dropped. Reports closed as "not planned" or carrying a reject label (e.g.
invalid, duplicate) are left out, so a maintainer removes a bad report by closing it that way."""
import json
from pathlib import Path
from .. import report_form
def transform(raw_path, spec, ctx):
issues = json.loads(Path(raw_path).read_text(encoding="utf-8"))
reject = set(spec["transform"].get("reject_labels", []))
form = report_form.spec()
reports = []
for i in sorted(issues, key=lambda i: i["number"]):
labels = {lb["name"] if isinstance(lb, dict) else lb for lb in i.get("labels") or []}
if labels & reject or i.get("state_reason") == "not_planned":
continue
answers = report_form.parse_body(i.get("body"), form)
if not answers:
continue
reports.append({"number": i["number"], "opened": (i.get("created_at") or "")[:10] or None, "answers": answers})
return {"source": spec["label"], "reports": reports}
def describe(old, new):
o = {r["number"] for r in (old or {}).get("reports", [])}
n = {r["number"] for r in new["reports"]}
return [f"{len(n)} report(s) ({len(n - o)} new, {len(o - n)} removed)"]
+75
View File
@@ -0,0 +1,75 @@
"""A Google Sheets export (.xlsx or .csv) -> the public source file: the sheet's own questions and answers, with the
source's privacy rules applied. No recoding happens here (that is core/atlas/coding.py, at build time)."""
import re
from datetime import datetime
import pandas as pd
from ..common import read_yaml
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
def load_sheet(path, sheet_contains="form responses"):
if str(path).lower().endswith(".csv"):
return pd.read_csv(path)
xl = pd.ExcelFile(path)
sheet = next((n for n in xl.sheet_names if sheet_contains in n.lower()), xl.sheet_names[0])
return pd.read_excel(xl, sheet)
def cell(v, redact=True):
if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)):
return None
if isinstance(v, (datetime, pd.Timestamp)):
return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S")
if isinstance(v, float) and v.is_integer():
return int(v)
if isinstance(v, str) and redact:
return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text
return v
def extra_results(manual):
"""Dragons read from result pages linked on Reddit (collection/manual/reddit_results.yaml)."""
return [{"source": "dragonkind.com result page linked on Reddit", "url": f"https://dragonkind.com/results/{r['id']}",
"name": r["name"], "colour": r["colour"], "tail": r["tail"], "sex": r["sex"], "age": r["age"],
"lineage": r["lineage"]} for r in read_yaml(manual)["results"]]
def transform(raw_path, spec, ctx):
t, privacy = spec["transform"], spec.get("privacy") or {}
raw = load_sheet(raw_path, t.get("sheet_contains", "form responses"))
drop = [d.lower() for d in privacy.get("drop_columns_matching", [])]
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
redact = privacy.get("redact_emails", True)
rows = []
for _, r in raw.iterrows():
row = {}
for k, v in r.items():
c = cell(v, redact)
if c is not None:
row[str(k)] = c
rows.append(row)
out = {"source": spec["label"], "exported": ctx.get("changed"),
# every header, even one blank in every row, so the coding rules see every question
"headers": [str(c) for c in raw.columns], "rows": rows}
if t.get("extra"):
out["extra"] = extra_results(t["extra"])
return out
def describe(old, new):
"""One report line (plus details) comparing the previous public file with the new one."""
if old is None:
return [f"first export: {len(new['rows'])} rows"]
key = lambda r: repr(sorted(r.items())) # noqa: E731
o, n = {key(r) for r in old["rows"]}, {key(r) for r in new["rows"]}
lines = [f"{len(old['rows'])} → {len(new['rows'])} rows ({len(n - o)} new or edited, {len(o - n)} removed or edited)"]
new_q = [h for h in new["headers"] if h not in old["headers"]]
gone_q = [h for h in old["headers"] if h not in new["headers"]]
if new_q:
lines.append(f" - New questions (add them to `core/spec/form.yaml`): {', '.join(new_q)}")
if gone_q:
lines.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
return lines
+44
View File
@@ -0,0 +1,44 @@
"""Validate: a table is checked against data/dictionary.yaml before it is loaded. A failure stops the load."""
import re
CHECKS = {
"integer": lambda v: isinstance(v, int) and not isinstance(v, bool) or bool(re.fullmatch(r"-?\d+", str(v))),
"boolean": lambda v: isinstance(v, bool) or v in ("True", "False"),
"date": lambda v: bool(re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(v))),
"text": lambda v: isinstance(v, str),
}
class Invalid(Exception):
pass
def problems(rows, dictionary, header=None):
"""Every way the rows break the dictionary, as readable lines (empty when the table is valid)."""
cols = dictionary["columns"]
out = []
if header is not None and list(header) != list(cols):
out.append(f"columns differ from the dictionary: {list(header)} vs {list(cols)}")
ids = [r.get("run_id") for r in rows]
if len(set(ids)) != len(ids):
out.append("run_id is not unique")
for name, col in cols.items():
vals = [r.get(name) for r in rows]
present = [v for v in vals if v is not None and v != ""]
bad_type = sorted({str(v) for v in present if not CHECKS[col["type"]](v)})
if bad_type:
out.append(f"{name}: not {col['type']}: {bad_type[:5]}")
if "values" in col:
allowed = {str(v) for v in col["values"]}
bad = sorted({str(v) for v in present if str(v) not in allowed})
if bad:
out.append(f"{name}: values not in the dictionary: {bad[:10]}")
if col.get("required") and len(present) != len(vals):
out.append(f"{name}: required, but {len(vals) - len(present)} rows are empty")
return out
def check(rows, dictionary, what="table"):
errs = problems(rows, dictionary)
if errs:
raise Invalid(f"{what} does not match data/dictionary.yaml:\n " + "\n ".join(errs))