- the importer reads data/accounts/*/runs.jsonl and logs/accounts/*/encounters.jsonl; with account folders, the top-level files (a mirror of the first account) are skipped - run records that carry their steps (schema 2) become trajectories directly, keeping what the bot saw (shadow, meadow dragons), propensities, decision mode and policy id; older runs are still rebuilt from the decision log; bonded runs without steps are kept - account names become salted hashes (DK_ACCOUNT_SALT, or a private salt in data/raw) - runs_clean.csv: new bot_mode column; saw_shadow and saw_blue_meadow from what the bot saw; deliberate burns say why - crosswalk: the valley Red, leaving squadmates, squad direction, hold your ground, hide/approach reactions, the wounded cadet's other two choices Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
368 lines
17 KiB
Python
368 lines
17 KiB
Python
"""The DragonKind bot's private data -> anonymised trajectories and a scene catalogue (Python port of bot.js).
|
|
|
|
Raw input (from the private bot repo, packed by extract.local_dir): data/knowledge.json (scene atlas: prose,
|
|
choices, tint, backdrop), data/runs.jsonl (one record per attempt) and logs/encounters.jsonl (every decision the
|
|
bot made, with the options on screen). Nothing of the prose or the account ids leaves this module: the output
|
|
holds scene fingerprints, short labels (collection/specs/bot.yaml), choice text and outcomes, and check_public()
|
|
refuses anything else.
|
|
|
|
Output: {"scenes": {sceneId: {stage, tint, backdrop, label, options: [{id, text}]}},
|
|
"trajectories": [{id, source, meta, steps, outcome}], "unlabeled": [...]}
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from datetime import datetime
|
|
|
|
from ..common import read_yaml
|
|
|
|
SPEC = read_yaml("collection/specs/bot.yaml")
|
|
_NORMALIZE = [(re.compile(a), b) for a, b in SPEC["normalize"]]
|
|
CHOICE_EVENTS = set(SPEC["choice_events"])
|
|
|
|
|
|
def sha1(s, n=12):
|
|
return hashlib.sha1(str(s).encode("utf-8")).hexdigest()[:n]
|
|
|
|
|
|
def normalize(text):
|
|
"""Must match normalize() in the bot's src/strategy.js, so fingerprints line up."""
|
|
t = str(text or "").lower()
|
|
for rx, rep in _NORMALIZE:
|
|
t = rx.sub(rep, t)
|
|
return t.strip()
|
|
|
|
|
|
def scene_id(prose, choices):
|
|
"""Must match computeSceneFingerprint() in the bot's src/learner.js."""
|
|
prose_norm = normalize(prose)[:500]
|
|
opts = sorted(c for c in (normalize(c) for c in choices) if c)
|
|
return sha1(f"{prose_norm} ### {' || '.join(opts)}")
|
|
|
|
|
|
def option_id(text):
|
|
return sha1(normalize(text), 10)
|
|
|
|
|
|
def label_for(sid, node):
|
|
if sid in SPEC["labels"]:
|
|
return SPEC["labels"][sid]
|
|
bits = [node.get("stage") or "Scene"]
|
|
if node.get("tint"):
|
|
bits.append(node["tint"])
|
|
return " · ".join(bits)
|
|
|
|
|
|
def _jsonl(text):
|
|
out = []
|
|
for line in (text or "").splitlines():
|
|
if line.strip():
|
|
try:
|
|
out.append(json.loads(line))
|
|
except json.JSONDecodeError:
|
|
pass
|
|
return out
|
|
|
|
|
|
def build_scenes(knowledge, encounters):
|
|
scenes = {}
|
|
for sid, n in ((knowledge or {}).get("nodes") or {}).items():
|
|
scenes[sid] = {
|
|
"stage": n.get("stage") or "UNKNOWN", "tint": n.get("tint"), "backdrop": n.get("backdrop"),
|
|
"label": label_for(sid, n),
|
|
"options": [{"id": option_id(c.get("label") or c.get("choiceNorm")), "text": c.get("label") or c.get("choiceNorm")}
|
|
for c in (n.get("choices") or {}).values()],
|
|
}
|
|
for e in encounters:
|
|
if not isinstance(e.get("choices"), list) or not e.get("prose"):
|
|
continue
|
|
sid = e.get("nodeId") or scene_id(e["prose"], e["choices"])
|
|
if sid in scenes:
|
|
continue
|
|
stage = (e.get("decision") or {}).get("stage") or e.get("stage") or "UNKNOWN"
|
|
scenes[sid] = {"stage": stage, "tint": None, "backdrop": None, "label": label_for(sid, {"stage": stage}),
|
|
"options": [{"id": option_id(t), "text": t} for t in e["choices"]]}
|
|
return scenes
|
|
|
|
|
|
def trajectories_from_encounters(encounters, scenes):
|
|
"""Group the log by attempt, keep the decisions acted on, and close each attempt at its first BURNED or
|
|
DRAGON_BONDED event. Repeated BURNED events with no decision between them are ignored."""
|
|
by_attempt = {}
|
|
for e in encounters:
|
|
key = f"{e['account']}:{e.get('attempt')}" if e.get("account") else str(e.get("attempt"))
|
|
by_attempt.setdefault(key, []).append(e)
|
|
out = []
|
|
for key, events in by_attempt.items():
|
|
events = sorted(events, key=lambda e: str(e.get("ts")))
|
|
steps, outcome = [], None
|
|
for e in events:
|
|
if outcome:
|
|
break
|
|
if e.get("event") in CHOICE_EVENTS:
|
|
d = e.get("decision") or {}
|
|
text = d.get("choiceText") or e.get("choiceText")
|
|
if d.get("action") and d["action"] != "choose":
|
|
continue # paused, nothing happened
|
|
if not text or not isinstance(e.get("choices"), list):
|
|
continue
|
|
scene = e.get("nodeId") or scene_id(e.get("prose") or "", e["choices"])
|
|
manual = e["event"] != "DECISION"
|
|
sc = scenes.get(scene) or {}
|
|
steps.append({
|
|
"scene": scene, "stage": d.get("stage") or sc.get("stage") or "UNKNOWN", "tint": sc.get("tint"),
|
|
"option": option_id(text), "optionText": text, "offered": [option_id(c) for c in e["choices"]],
|
|
"rule": d.get("rule"), "decisionMode": "manual" if manual else d.get("source"),
|
|
"manual": manual, "at": e.get("ts"),
|
|
})
|
|
elif e.get("event") == "BURNED" and steps:
|
|
outcome = {"type": "BURNED", "colour": None, "intentional": bool(e.get("intentional"))}
|
|
elif e.get("event") == "DRAGON_BONDED" and steps:
|
|
outcome = {"type": "BONDED", "colour": str(e["dragon"]).lower() if e.get("dragon") else None, "intentional": False}
|
|
# a decision repeated on the same scene (the bot re-detecting the screen) counts once
|
|
dedup = [s for i, s in enumerate(steps)
|
|
if not (i > 0 and steps[i - 1]["scene"] == s["scene"] and steps[i - 1]["option"] == s["option"])]
|
|
for i, s in enumerate(dedup):
|
|
s["to"] = dedup[i + 1]["scene"] if i + 1 < len(dedup) else None
|
|
if not dedup:
|
|
continue
|
|
if outcome:
|
|
dedup[-1]["to"] = "BURNED" if outcome["type"] == "BURNED" else f"BONDED:{outcome['colour'] or 'unknown'}"
|
|
try:
|
|
attempt = int(float(key.split(":")[-1])) or None
|
|
except ValueError:
|
|
attempt = None
|
|
started = dedup[0]["at"]
|
|
out.append({
|
|
"id": "bot-" + sha1(f"{key}:{started}", 10),
|
|
"source": "bot",
|
|
"meta": {"attempt": attempt, "startedAt": dedup[0]["at"], "accountKey": None, "target": None, "decisionMode": None},
|
|
"steps": dedup,
|
|
"outcome": outcome or {"type": "ABORTED", "colour": None, "intentional": False},
|
|
})
|
|
return out
|
|
|
|
|
|
def _ms(ts):
|
|
try:
|
|
return datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp() * 1000
|
|
except ValueError:
|
|
return float("nan")
|
|
|
|
|
|
def attach_run_meta(trajs, runs, salt):
|
|
"""Run-level metadata (target colour, mode) from runs.jsonl, where an attempt is in both."""
|
|
window = SPEC["match_hours"] * 3600e3
|
|
for r in runs:
|
|
t = next((x for x in trajs if x["meta"]["attempt"] == r.get("attempt")
|
|
and abs(_ms(x["meta"]["startedAt"]) - _ms(r.get("startedAt"))) < window), None)
|
|
if not t:
|
|
continue
|
|
m = t["meta"]
|
|
m["target"] = r.get("targetDragon")
|
|
m["decisionMode"] = r.get("decisionMode")
|
|
m["policyId"] = r.get("policyId")
|
|
m["experimentId"] = r.get("experimentId")
|
|
m["arm"] = r.get("arm")
|
|
if r.get("accountKey"):
|
|
m["accountKey"] = r["accountKey"]
|
|
elif r.get("accountId") and salt:
|
|
m["accountKey"] = sha1(f"{salt}:{r['accountId']}", 10)
|
|
if r.get("outcome") == "BONDED" and r.get("dragonColor"):
|
|
t["outcome"] = {"type": "BONDED", "colour": str(r["dragonColor"]).lower(), "intentional": False}
|
|
if r.get("intentionalBurn"):
|
|
t["outcome"]["intentional"] = True
|
|
return trajs
|
|
|
|
|
|
def import_bot(knowledge, encounters, runs, salt=None):
|
|
scenes = build_scenes(knowledge, encounters)
|
|
trajectories = attach_run_meta(trajectories_from_encounters(encounters, scenes), runs, salt)
|
|
unlabeled = [f"{sid} ({s['stage']}{', ' + s['tint'] if s['tint'] else ''})" for sid, s in scenes.items() if sid not in SPEC["labels"]]
|
|
return scenes, trajectories, unlabeled
|
|
|
|
|
|
def check_public(obj, refuse, max_string):
|
|
"""Nothing that looks like game prose or an account identifier may leave this module."""
|
|
def walk(v, p):
|
|
if isinstance(v, list):
|
|
for i, x in enumerate(v):
|
|
walk(x, f"{p}[{i}]")
|
|
elif isinstance(v, dict):
|
|
for k, x in v.items():
|
|
if k in refuse:
|
|
raise ValueError(f'refusing to export field "{k}" at {p}')
|
|
walk(x, f"{p}.{k}")
|
|
elif isinstance(v, str) and len(v) > max_string:
|
|
raise ValueError(f"refusing to export a {len(v)}-character string at {p} (looks like prose)")
|
|
walk(obj, "$")
|
|
|
|
|
|
def merge_scenes(existing, scenes):
|
|
"""Add new scenes and new options to the published catalogue; never drop what is already there."""
|
|
out = json.loads(json.dumps(existing or {}))
|
|
for sid, s in scenes.items():
|
|
old = out.get(sid)
|
|
if not old:
|
|
out[sid] = s
|
|
continue
|
|
ids = {o["id"] for o in old["options"]}
|
|
old["options"] += [o for o in s["options"] if o["id"] not in ids]
|
|
old["tint"] = old.get("tint") or s["tint"]
|
|
old["backdrop"] = old.get("backdrop") or s["backdrop"]
|
|
if "·" in old["label"] and "·" not in s["label"]:
|
|
old["label"] = s["label"]
|
|
return dict(sorted(out.items()))
|
|
|
|
|
|
# ---------------------------------------------------------------- run records with steps (the bot's schema 2)
|
|
|
|
def _to(key):
|
|
"""The bot's toKey -> ours: a scene id, BURNED or BONDED:<colour>."""
|
|
key = str(key or "")
|
|
if key == "TERMINAL:BURNED":
|
|
return "BURNED"
|
|
if key.startswith("TERMINAL:BONDED"):
|
|
return "BONDED:" + (key.split("_", 1)[1].lower() if "_" in key else "unknown")
|
|
return key or None
|
|
|
|
|
|
def _seen(observed):
|
|
"""What the bot saw on a screen that the route model needs: the dragons (from the tint) and the shadow."""
|
|
o = observed or {}
|
|
out = {}
|
|
if o.get("dragons"):
|
|
out["dragons"] = [str(d).lower() for d in o["dragons"]]
|
|
if "shadow" in o:
|
|
out["shadow"] = bool(o["shadow"])
|
|
return out
|
|
|
|
|
|
def trajectory_from_run(r, account_key):
|
|
"""A run record that carries its own steps -> a trajectory. Nothing but choice text, ids and outcomes is kept."""
|
|
steps = []
|
|
for st in r.get("steps") or []:
|
|
text = st.get("choiceText") or st.get("choiceNorm")
|
|
if not text:
|
|
continue
|
|
steps.append({
|
|
"scene": st.get("nodeId"), "stage": st.get("stage") or "UNKNOWN", "tint": st.get("tint"),
|
|
"option": option_id(text), "optionText": text,
|
|
"offered": [option_id(o.get("text") or o.get("norm")) for o in st.get("offered") or [] if o.get("text") or o.get("norm")],
|
|
"rule": st.get("rule"), "decisionMode": "manual" if st.get("manual") else st.get("decisionMode"),
|
|
"manual": bool(st.get("manual")), "at": st.get("at"), "seen": _seen(st.get("observed")),
|
|
"propensity": st.get("propensity"), "to": _to(st.get("toKey")),
|
|
})
|
|
colour = str(r["dragonColor"]).lower() if r.get("dragonColor") else None
|
|
return {
|
|
"id": "bot-" + sha1(r["runId"], 10),
|
|
"source": "bot",
|
|
"meta": {"attempt": r.get("accountAttempt") or r.get("attempt"), "startedAt": r.get("startedAt"),
|
|
"accountKey": account_key, "accountRole": r.get("accountRole"), "target": r.get("targetDragon"),
|
|
"decisionMode": r.get("decisionMode"), "policyId": r.get("policyId"),
|
|
"experimentId": r.get("experimentId"), "arm": r.get("arm")},
|
|
"steps": steps,
|
|
"outcome": {"type": r.get("outcome") or "ABORTED", "colour": colour, "intentional": bool(r.get("intentionalBurn")),
|
|
"reason": (r.get("burn") or {}).get("reason")},
|
|
}
|
|
|
|
|
|
def _account_files(files):
|
|
"""{account: {"runs": [...], "encounters": [...]}} from the packed files. The top-level files are the bot's
|
|
original single account; data/accounts/<name>/ and logs/accounts/<name>/ hold one account each."""
|
|
out = {}
|
|
for rel, text in files.items():
|
|
if text is None:
|
|
continue
|
|
parts = rel.split("/")
|
|
acct = parts[2] if len(parts) == 4 and parts[1] == "accounts" else "(default)"
|
|
kind = "runs" if rel.endswith("runs.jsonl") else "encounters" if rel.endswith("encounters.jsonl") else None
|
|
if kind:
|
|
out.setdefault(acct, {"runs": [], "encounters": []})[kind] += _jsonl(text)
|
|
# since the bot keeps one folder per account, its top-level files mirror its first account (the same runs,
|
|
# logged a moment apart): counting both would count those runs twice
|
|
if len(out) > 1:
|
|
out.pop("(default)", None)
|
|
return out
|
|
|
|
|
|
def account_salt(env, privacy):
|
|
"""The salt that turns account names into anonymous keys: the environment variable if set, otherwise one made
|
|
once and kept in the private raw store (data/raw/bot/account-salt.txt, never committed)."""
|
|
salt = env.get(privacy.get("account_salt_env", ""))
|
|
if salt:
|
|
return salt
|
|
from ..common import path
|
|
import secrets
|
|
p = path("data/raw/bot/account-salt.txt")
|
|
if not p.exists():
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
p.write_text(secrets.token_hex(16), encoding="utf-8")
|
|
return p.read_text(encoding="utf-8").strip()
|
|
|
|
|
|
def transform(raw_path, spec, ctx):
|
|
with open(raw_path, encoding="utf-8") as f:
|
|
files = json.load(f)
|
|
k = files.get("data/knowledge.json")
|
|
knowledge = json.loads(k) if k else {"nodes": {}}
|
|
privacy = spec.get("privacy") or {}
|
|
salt = account_salt(ctx.get("env", {}), privacy)
|
|
accounts = _account_files(files)
|
|
all_encounters = [e for a in accounts.values() for e in a["encounters"]]
|
|
scenes = build_scenes(knowledge, all_encounters)
|
|
trajectories, seen_ids = [], set()
|
|
for acct, data in sorted(accounts.items()):
|
|
key = sha1(f"{salt}:{acct}", 10)
|
|
with_steps = [r for r in data["runs"] if r.get("steps") and r.get("runId")]
|
|
for r in with_steps:
|
|
t = trajectory_from_run(r, key)
|
|
if t["id"] not in seen_ids:
|
|
seen_ids.add(t["id"])
|
|
trajectories.append(t)
|
|
# older runs without steps: rebuild them from the decision log, as before
|
|
legacy = [r for r in data["runs"] if not r.get("steps")]
|
|
window = SPEC["match_hours"] * 3600e3
|
|
same_run = lambda t, r: (t["meta"]["attempt"] == (r.get("accountAttempt") or r.get("attempt")) # noqa: E731
|
|
and abs(_ms(t["meta"]["startedAt"]) - _ms(r.get("startedAt"))) < window)
|
|
old = attach_run_meta(trajectories_from_encounters(data["encounters"], scenes), legacy, salt)
|
|
kept = []
|
|
for t in old: # the decision log also holds the runs that have steps: keep only the others
|
|
if any(same_run(t, r) for r in with_steps) or t["id"] in seen_ids:
|
|
continue
|
|
t["meta"]["accountKey"] = key
|
|
seen_ids.add(t["id"])
|
|
trajectories.append(t)
|
|
kept.append(t)
|
|
# bonded runs logged without any steps still say which dragon was bonded
|
|
for r in legacy:
|
|
if r.get("outcome") == "BONDED" and r.get("runId") and not any(same_run(t, r) for t in kept):
|
|
t = trajectory_from_run({**r, "steps": []}, key)
|
|
if t["id"] not in seen_ids:
|
|
seen_ids.add(t["id"])
|
|
trajectories.append(t)
|
|
# scenes the run records show but the knowledge file does not have yet
|
|
for t in trajectories:
|
|
for st in t["steps"]:
|
|
if st["scene"] and st["scene"] not in scenes:
|
|
scenes[st["scene"]] = {"stage": st["stage"], "tint": st["tint"], "backdrop": None,
|
|
"label": label_for(st["scene"], {"stage": st["stage"], "tint": st["tint"]}), "options": []}
|
|
sc = scenes.get(st["scene"])
|
|
if sc is not None and st["option"] not in {o["id"] for o in sc["options"]}:
|
|
sc["options"].append({"id": st["option"], "text": st["optionText"]})
|
|
unlabeled = [f"{sid} ({s['stage']}{', ' + s['tint'] if s['tint'] else ''})" for sid, s in scenes.items() if sid not in SPEC["labels"]]
|
|
for obj in (scenes, trajectories):
|
|
check_public(obj, set(privacy.get("refuse_fields", [])), privacy.get("max_string", 200))
|
|
trajectories.sort(key=lambda t: (str(t["meta"].get("startedAt")), t["id"]))
|
|
return {"scenes": scenes, "trajectories": trajectories, "_unlabeled": unlabeled}
|
|
|
|
|
|
def describe(old, new):
|
|
outcomes = {}
|
|
for t in new["trajectories"]:
|
|
outcomes[t["outcome"]["type"]] = outcomes.get(t["outcome"]["type"], 0) + 1
|
|
lines = [f"{len(new['scenes'])} scenes, {len(new['trajectories'])} attempts {json.dumps(outcomes)}"]
|
|
if new.get("_unlabeled"):
|
|
lines.append(" - Scenes without a short label (add them to collection/specs/bot.yaml): " + ", ".join(new["_unlabeled"]))
|
|
return lines
|