Collection as an ETL pipeline; runs_clean.csv from every source; Report a run

- collection/pipeline.yaml declares every source (extract kind, transform,
  privacy rules, load target, run source); `python -m collection update|build`
- extract keeps data/raw/<source>/current + previous only (git-ignored);
  data/manifest.json records the current raw checksum and change date
- transforms are pure; validate checks runs against data/dictionary.yaml
  before load; load writes byte-stable files only when they change
- data/public/runs_clean.csv: every run from every source with a new
  `source` column (reddit spreadsheet, reddit thread, reported, bot,
  reddit pdf); the sheet's own row label moves to `entry`; run_id is text
- bot importer ported to Python (same output as bot.js on the fixture);
  labels and crosswalk in collection/specs/bot.yaml, written into graph.js
- Reddit through the official API (result links to add by hand); the 12
  result pages move to collection/manual/reddit_results.yaml
- "Report a run": a GitHub issue form generated from
  collection/specs/report_form.yaml, collected as source "reported", plus a
  site tab pointing to it, the community spreadsheet and the megathread
- analysis and the route model read runs_clean.csv (sources in
  measures.yaml); stats.json and policy.json unchanged apart from the
  factor's new name
- daily workflow runs the pipeline; the raw store is kept between runs in
  the Actions cache, encrypted with RAW_ARCHIVE_KEY

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-04 02:46:48 +00:00
co-authored by Claude Opus 5.5
parent 61bc50583c
commit 3b24a11f5a
64 changed files with 8626 additions and 6909 deletions
+7
View File
@@ -0,0 +1,7 @@
"""Named transforms (pipeline.yaml `transform: {name: ...}`). Each module has
transform(raw_path, spec, ctx) -> the public source file's content (pure: no network, no clock)
describe(old, new) -> report lines comparing the previous public file with the new one
"""
from . import bot, pdf, reddit, reports, sheet
TRANSFORMS = {"sheet": sheet, "pdf_fingerprint": pdf, "bot": bot, "reddit_results": reddit, "reports": reports}
+239
View File
@@ -0,0 +1,239 @@
"""The DragonKind bot's private data -> anonymised trajectories and a scene catalogue (Python port of bot.js).
Raw input (from the private bot repo, packed by extract.local_dir): data/knowledge.json (scene atlas: prose,
choices, tint, backdrop), data/runs.jsonl (one record per attempt) and logs/encounters.jsonl (every decision the
bot made, with the options on screen). Nothing of the prose or the account ids leaves this module: the output
holds scene fingerprints, short labels (collection/specs/bot.yaml), choice text and outcomes, and check_public()
refuses anything else.
Output: {"scenes": {sceneId: {stage, tint, backdrop, label, options: [{id, text}]}},
"trajectories": [{id, source, meta, steps, outcome}], "unlabeled": [...]}
"""
import hashlib
import json
import re
from datetime import datetime
from ..common import read_yaml
SPEC = read_yaml("collection/specs/bot.yaml")
_NORMALIZE = [(re.compile(a), b) for a, b in SPEC["normalize"]]
CHOICE_EVENTS = set(SPEC["choice_events"])
def sha1(s, n=12):
return hashlib.sha1(str(s).encode("utf-8")).hexdigest()[:n]
def normalize(text):
"""Must match normalize() in the bot's src/strategy.js, so fingerprints line up."""
t = str(text or "").lower()
for rx, rep in _NORMALIZE:
t = rx.sub(rep, t)
return t.strip()
def scene_id(prose, choices):
"""Must match computeSceneFingerprint() in the bot's src/learner.js."""
prose_norm = normalize(prose)[:500]
opts = sorted(c for c in (normalize(c) for c in choices) if c)
return sha1(f"{prose_norm} ### {' || '.join(opts)}")
def option_id(text):
return sha1(normalize(text), 10)
def label_for(sid, node):
if sid in SPEC["labels"]:
return SPEC["labels"][sid]
bits = [node.get("stage") or "Scene"]
if node.get("tint"):
bits.append(node["tint"])
return " · ".join(bits)
def _jsonl(text):
out = []
for line in (text or "").splitlines():
if line.strip():
try:
out.append(json.loads(line))
except json.JSONDecodeError:
pass
return out
def build_scenes(knowledge, encounters):
scenes = {}
for sid, n in ((knowledge or {}).get("nodes") or {}).items():
scenes[sid] = {
"stage": n.get("stage") or "UNKNOWN", "tint": n.get("tint"), "backdrop": n.get("backdrop"),
"label": label_for(sid, n),
"options": [{"id": option_id(c.get("label") or c.get("choiceNorm")), "text": c.get("label") or c.get("choiceNorm")}
for c in (n.get("choices") or {}).values()],
}
for e in encounters:
if not isinstance(e.get("choices"), list) or not e.get("prose"):
continue
sid = e.get("nodeId") or scene_id(e["prose"], e["choices"])
if sid in scenes:
continue
stage = (e.get("decision") or {}).get("stage") or e.get("stage") or "UNKNOWN"
scenes[sid] = {"stage": stage, "tint": None, "backdrop": None, "label": label_for(sid, {"stage": stage}),
"options": [{"id": option_id(t), "text": t} for t in e["choices"]]}
return scenes
def trajectories_from_encounters(encounters, scenes):
"""Group the log by attempt, keep the decisions acted on, and close each attempt at its first BURNED or
DRAGON_BONDED event. Repeated BURNED events with no decision between them are ignored."""
by_attempt = {}
for e in encounters:
key = f"{e['account']}:{e.get('attempt')}" if e.get("account") else str(e.get("attempt"))
by_attempt.setdefault(key, []).append(e)
out = []
for key, events in by_attempt.items():
events = sorted(events, key=lambda e: str(e.get("ts")))
steps, outcome = [], None
for e in events:
if outcome:
break
if e.get("event") in CHOICE_EVENTS:
d = e.get("decision") or {}
text = d.get("choiceText") or e.get("choiceText")
if d.get("action") and d["action"] != "choose":
continue # paused, nothing happened
if not text or not isinstance(e.get("choices"), list):
continue
scene = e.get("nodeId") or scene_id(e.get("prose") or "", e["choices"])
manual = e["event"] != "DECISION"
sc = scenes.get(scene) or {}
steps.append({
"scene": scene, "stage": d.get("stage") or sc.get("stage") or "UNKNOWN", "tint": sc.get("tint"),
"option": option_id(text), "optionText": text, "offered": [option_id(c) for c in e["choices"]],
"rule": d.get("rule"), "decisionMode": "manual" if manual else d.get("source"),
"manual": manual, "at": e.get("ts"),
})
elif e.get("event") == "BURNED" and steps:
outcome = {"type": "BURNED", "colour": None, "intentional": bool(e.get("intentional"))}
elif e.get("event") == "DRAGON_BONDED" and steps:
outcome = {"type": "BONDED", "colour": str(e["dragon"]).lower() if e.get("dragon") else None, "intentional": False}
# a decision repeated on the same scene (the bot re-detecting the screen) counts once
dedup = [s for i, s in enumerate(steps)
if not (i > 0 and steps[i - 1]["scene"] == s["scene"] and steps[i - 1]["option"] == s["option"])]
for i, s in enumerate(dedup):
s["to"] = dedup[i + 1]["scene"] if i + 1 < len(dedup) else None
if not dedup:
continue
if outcome:
dedup[-1]["to"] = "BURNED" if outcome["type"] == "BURNED" else f"BONDED:{outcome['colour'] or 'unknown'}"
try:
attempt = int(float(key.split(":")[-1])) or None
except ValueError:
attempt = None
started = dedup[0]["at"]
out.append({
"id": "bot-" + sha1(f"{key}:{started}", 10),
"source": "bot",
"meta": {"attempt": attempt, "startedAt": dedup[0]["at"], "accountKey": None, "target": None, "decisionMode": None},
"steps": dedup,
"outcome": outcome or {"type": "ABORTED", "colour": None, "intentional": False},
})
return out
def _ms(ts):
try:
return datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp() * 1000
except ValueError:
return float("nan")
def attach_run_meta(trajs, runs, salt):
"""Run-level metadata (target colour, mode) from runs.jsonl, where an attempt is in both."""
window = SPEC["match_hours"] * 3600e3
for r in runs:
t = next((x for x in trajs if x["meta"]["attempt"] == r.get("attempt")
and abs(_ms(x["meta"]["startedAt"]) - _ms(r.get("startedAt"))) < window), None)
if not t:
continue
m = t["meta"]
m["target"] = r.get("targetDragon")
m["decisionMode"] = r.get("decisionMode")
m["policyId"] = r.get("policyId")
m["experimentId"] = r.get("experimentId")
m["arm"] = r.get("arm")
if r.get("accountKey"):
m["accountKey"] = r["accountKey"]
elif r.get("accountId") and salt:
m["accountKey"] = sha1(f"{salt}:{r['accountId']}", 10)
if r.get("outcome") == "BONDED" and r.get("dragonColor"):
t["outcome"] = {"type": "BONDED", "colour": str(r["dragonColor"]).lower(), "intentional": False}
if r.get("intentionalBurn"):
t["outcome"]["intentional"] = True
return trajs
def import_bot(knowledge, encounters, runs, salt=None):
scenes = build_scenes(knowledge, encounters)
trajectories = attach_run_meta(trajectories_from_encounters(encounters, scenes), runs, salt)
unlabeled = [f"{sid} ({s['stage']}{', ' + s['tint'] if s['tint'] else ''})" for sid, s in scenes.items() if sid not in SPEC["labels"]]
return scenes, trajectories, unlabeled
def check_public(obj, refuse, max_string):
"""Nothing that looks like game prose or an account identifier may leave this module."""
def walk(v, p):
if isinstance(v, list):
for i, x in enumerate(v):
walk(x, f"{p}[{i}]")
elif isinstance(v, dict):
for k, x in v.items():
if k in refuse:
raise ValueError(f'refusing to export field "{k}" at {p}')
walk(x, f"{p}.{k}")
elif isinstance(v, str) and len(v) > max_string:
raise ValueError(f"refusing to export a {len(v)}-character string at {p} (looks like prose)")
walk(obj, "$")
def merge_scenes(existing, scenes):
"""Add new scenes and new options to the published catalogue; never drop what is already there."""
out = json.loads(json.dumps(existing or {}))
for sid, s in scenes.items():
old = out.get(sid)
if not old:
out[sid] = s
continue
ids = {o["id"] for o in old["options"]}
old["options"] += [o for o in s["options"] if o["id"] not in ids]
old["tint"] = old.get("tint") or s["tint"]
old["backdrop"] = old.get("backdrop") or s["backdrop"]
if "·" in old["label"] and "·" not in s["label"]:
old["label"] = s["label"]
return dict(sorted(out.items()))
def transform(raw_path, spec, ctx):
with open(raw_path, encoding="utf-8") as f:
files = json.load(f)
k = files.get("data/knowledge.json")
knowledge = json.loads(k) if k else {"nodes": {}}
encounters = _jsonl(files.get("logs/encounters.jsonl"))
runs = _jsonl(files.get("data/runs.jsonl"))
privacy = spec.get("privacy") or {}
scenes, trajectories, unlabeled = import_bot(knowledge, encounters, runs, ctx.get("env", {}).get(privacy.get("account_salt_env", "")))
for obj in (scenes, trajectories):
check_public(obj, set(privacy.get("refuse_fields", [])), privacy.get("max_string", 200))
return {"scenes": scenes, "trajectories": trajectories, "_unlabeled": unlabeled}
def describe(old, new):
outcomes = {}
for t in new["trajectories"]:
outcomes[t["outcome"]["type"]] = outcomes.get(t["outcome"]["type"], 0) + 1
lines = [f"{len(new['scenes'])} scenes, {len(new['trajectories'])} attempts {json.dumps(outcomes)}"]
if new.get("_unlabeled"):
lines.append(" - Scenes without a short label (add them to collection/specs/bot.yaml): " + ", ".join(new["_unlabeled"]))
return lines
+51
View File
@@ -0,0 +1,51 @@
"""The route PDF -> its fingerprint only. The PDF is never redistributed: the public file holds hashes of the file,
of each page's text and of each line, so a change can be detected and located without keeping the text."""
import hashlib
def _sha(text, n):
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:n]
def pdf_lines(path):
from pypdf import PdfReader
pages = []
for page in PdfReader(str(path)).pages:
text = page.extract_text() or ""
pages.append([" ".join(line.split()) for line in text.splitlines() if line.strip()])
return pages
def transform(raw_path, spec, ctx):
with open(raw_path, "rb") as f:
data = f.read()
pages = pdf_lines(raw_path)
return {
"sha256": hashlib.sha256(data).hexdigest(),
"bytes": len(data),
"pages": len(pages),
"page_hashes": [_sha("\n".join(p), 16) for p in pages],
"line_hashes": sorted({_sha(line.lower(), 12) for p in pages for line in p}),
"checked": ctx.get("changed"),
# transient: the new lines' text, for the report only (load drops every key starting with "_")
"_lines": [line for p in pages for line in p],
}
def describe(old, new):
if old is None:
return [f"fingerprint recorded for the first time ({new['pages']} pages)"]
if old.get("page_hashes") == new["page_hashes"]:
return ["the file changed but its text did not (layout or images only). Worth a look."]
changed = [i + 1 for i, h in enumerate(new["page_hashes"])
if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
old_lines = set(old["line_hashes"])
new_lines = [line for line in new.get("_lines", []) if _sha(line.lower(), 12) not in old_lines]
lines = [f"changed. {old['pages']} → {new['pages']} pages; pages with new text: {', '.join(map(str, changed)) or 'none'}; "
f"{len(new_lines)} new lines, {len(old_lines - set(new['line_hashes']))} lines gone."]
if new_lines:
lines.append(" - New lines (update core/spec/game.yaml or the crosswalks if they add options):")
lines += [f" - {line}" for line in new_lines[:60]]
if len(new_lines) > 60:
lines.append(f" - … and {len(new_lines) - 60} more")
return lines
+45
View File
@@ -0,0 +1,45 @@
"""Reddit megathreads (raw: the threads' comment trees from the official API) -> dragonkind.com result links that
are not yet in collection/manual/reddit_results.yaml. Usernames and comment text are never kept: only the result
ids and the comment permalinks, for the report, so a person can read the result page and add it by hand."""
import json
import re
from pathlib import Path
from ..common import read_yaml
RESULT_RE = re.compile(r"dragonkind\.com/results/(THRESH-[A-Z0-9]+)", re.I)
def _comments(node):
"""Every comment in a listing tree."""
if isinstance(node, list):
for x in node:
yield from _comments(x)
elif isinstance(node, dict):
data = node.get("data") or {}
if node.get("kind") in ("t1", "t3"):
yield data
for key in ("children", "replies"):
if isinstance(data.get(key), (dict, list)):
yield from _comments(data[key])
def transform(raw_path, spec, ctx):
threads = json.loads(Path(raw_path).read_text(encoding="utf-8"))
known = {r["id"].upper() for r in read_yaml(spec["transform"]["manual"])["results"]}
found = {}
for tid, tree in sorted(threads.items()):
for c in _comments(tree):
for m in RESULT_RE.finditer((c.get("body") or "") + " " + (c.get("selftext") or "") + " " + (c.get("url") or "")):
rid = m.group(1).upper()
if rid not in known and rid not in found:
found[rid] = "https://www.reddit.com" + (c.get("permalink") or f"/comments/{tid}/")
return {"new_results": [{"id": k, "comment": v} for k, v in sorted(found.items())]}
def describe(old, new):
rs = new["new_results"]
if not rs:
return ["no new result links in the threads"]
return [f"{len(rs)} result link(s) not yet in collection/manual/reddit_results.yaml (read each page, then add it):"] + \
[f" - https://dragonkind.com/results/{r['id']} (posted at {r['comment']})" for r in rs[:40]]
+31
View File
@@ -0,0 +1,31 @@
"""Run reports (raw: the "run report" issues from the GitHub API) -> data/public/reports.json.
Kept per report: the issue number, the date it was opened, and the answers (the option labels chosen). GitHub
usernames and any other text are dropped. Reports closed as "not planned" or carrying a reject label (e.g.
invalid, duplicate) are left out, so a maintainer removes a bad report by closing it that way."""
import json
from pathlib import Path
from .. import report_form
def transform(raw_path, spec, ctx):
issues = json.loads(Path(raw_path).read_text(encoding="utf-8"))
reject = set(spec["transform"].get("reject_labels", []))
form = report_form.spec()
reports = []
for i in sorted(issues, key=lambda i: i["number"]):
labels = {lb["name"] if isinstance(lb, dict) else lb for lb in i.get("labels") or []}
if labels & reject or i.get("state_reason") == "not_planned":
continue
answers = report_form.parse_body(i.get("body"), form)
if not answers:
continue
reports.append({"number": i["number"], "opened": (i.get("created_at") or "")[:10] or None, "answers": answers})
return {"source": spec["label"], "reports": reports}
def describe(old, new):
o = {r["number"] for r in (old or {}).get("reports", [])}
n = {r["number"] for r in new["reports"]}
return [f"{len(n)} report(s) ({len(n - o)} new, {len(o - n)} removed)"]
+75
View File
@@ -0,0 +1,75 @@
"""A Google Sheets export (.xlsx or .csv) -> the public source file: the sheet's own questions and answers, with the
source's privacy rules applied. No recoding happens here (that is core/atlas/coding.py, at build time)."""
import re
from datetime import datetime
import pandas as pd
from ..common import read_yaml
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
def load_sheet(path, sheet_contains="form responses"):
if str(path).lower().endswith(".csv"):
return pd.read_csv(path)
xl = pd.ExcelFile(path)
sheet = next((n for n in xl.sheet_names if sheet_contains in n.lower()), xl.sheet_names[0])
return pd.read_excel(xl, sheet)
def cell(v, redact=True):
if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)):
return None
if isinstance(v, (datetime, pd.Timestamp)):
return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S")
if isinstance(v, float) and v.is_integer():
return int(v)
if isinstance(v, str) and redact:
return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text
return v
def extra_results(manual):
"""Dragons read from result pages linked on Reddit (collection/manual/reddit_results.yaml)."""
return [{"source": "dragonkind.com result page linked on Reddit", "url": f"https://dragonkind.com/results/{r['id']}",
"name": r["name"], "colour": r["colour"], "tail": r["tail"], "sex": r["sex"], "age": r["age"],
"lineage": r["lineage"]} for r in read_yaml(manual)["results"]]
def transform(raw_path, spec, ctx):
t, privacy = spec["transform"], spec.get("privacy") or {}
raw = load_sheet(raw_path, t.get("sheet_contains", "form responses"))
drop = [d.lower() for d in privacy.get("drop_columns_matching", [])]
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
redact = privacy.get("redact_emails", True)
rows = []
for _, r in raw.iterrows():
row = {}
for k, v in r.items():
c = cell(v, redact)
if c is not None:
row[str(k)] = c
rows.append(row)
out = {"source": spec["label"], "exported": ctx.get("changed"),
# every header, even one blank in every row, so the coding rules see every question
"headers": [str(c) for c in raw.columns], "rows": rows}
if t.get("extra"):
out["extra"] = extra_results(t["extra"])
return out
def describe(old, new):
"""One report line (plus details) comparing the previous public file with the new one."""
if old is None:
return [f"first export: {len(new['rows'])} rows"]
key = lambda r: repr(sorted(r.items())) # noqa: E731
o, n = {key(r) for r in old["rows"]}, {key(r) for r in new["rows"]}
lines = [f"{len(old['rows'])} → {len(new['rows'])} rows ({len(n - o)} new or edited, {len(o - n)} removed or edited)"]
new_q = [h for h in new["headers"] if h not in old["headers"]]
gone_q = [h for h in old["headers"] if h not in new["headers"]]
if new_q:
lines.append(f" - New questions (add them to `core/spec/form.yaml`): {', '.join(new_q)}")
if gone_q:
lines.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
return lines