"""The DragonKind bot's private data -> anonymised trajectories and a scene catalogue (Python port of bot.js). Raw input (from the private bot repo, packed by extract.local_dir): data/knowledge.json (scene atlas: prose, choices, tint, backdrop), data/runs.jsonl (one record per attempt) and logs/encounters.jsonl (every decision the bot made, with the options on screen). Nothing of the prose or the account ids leaves this module: the output holds scene fingerprints, short labels (collection/specs/bot.yaml), choice text and outcomes, and check_public() refuses anything else. Output: {"scenes": {sceneId: {stage, tint, backdrop, label, options: [{id, text}]}}, "trajectories": [{id, source, meta, steps, outcome}], "unlabeled": [...]} """ import hashlib import json import re from datetime import datetime from ..common import read_yaml SPEC = read_yaml("collection/specs/bot.yaml") _NORMALIZE = [(re.compile(a), b) for a, b in SPEC["normalize"]] CHOICE_EVENTS = set(SPEC["choice_events"]) def sha1(s, n=12): return hashlib.sha1(str(s).encode("utf-8")).hexdigest()[:n] def normalize(text): """Must match normalize() in the bot's src/strategy.js, so fingerprints line up.""" t = str(text or "").lower() for rx, rep in _NORMALIZE: t = rx.sub(rep, t) return t.strip() def scene_id(prose, choices): """Must match computeSceneFingerprint() in the bot's src/learner.js.""" prose_norm = normalize(prose)[:500] opts = sorted(c for c in (normalize(c) for c in choices) if c) return sha1(f"{prose_norm} ### {' || '.join(opts)}") def option_id(text): return sha1(normalize(text), 10) def label_for(sid, node): if sid in SPEC["labels"]: return SPEC["labels"][sid] bits = [node.get("stage") or "Scene"] if node.get("tint"): bits.append(node["tint"]) return " · ".join(bits) def _jsonl(text): out = [] for line in (text or "").splitlines(): if line.strip(): try: out.append(json.loads(line)) except json.JSONDecodeError: pass return out def build_scenes(knowledge, encounters): scenes = {} for sid, n in ((knowledge or {}).get("nodes") or {}).items(): scenes[sid] = { "stage": n.get("stage") or "UNKNOWN", "tint": n.get("tint"), "backdrop": n.get("backdrop"), "label": label_for(sid, n), "options": [{"id": option_id(c.get("label") or c.get("choiceNorm")), "text": c.get("label") or c.get("choiceNorm")} for c in (n.get("choices") or {}).values()], } for e in encounters: if not isinstance(e.get("choices"), list) or not e.get("prose"): continue sid = e.get("nodeId") or scene_id(e["prose"], e["choices"]) if sid in scenes: continue stage = (e.get("decision") or {}).get("stage") or e.get("stage") or "UNKNOWN" scenes[sid] = {"stage": stage, "tint": None, "backdrop": None, "label": label_for(sid, {"stage": stage}), "options": [{"id": option_id(t), "text": t} for t in e["choices"]]} return scenes def trajectories_from_encounters(encounters, scenes): """Group the log by attempt, keep the decisions acted on, and close each attempt at its first BURNED or DRAGON_BONDED event. Repeated BURNED events with no decision between them are ignored.""" by_attempt = {} for e in encounters: key = f"{e['account']}:{e.get('attempt')}" if e.get("account") else str(e.get("attempt")) by_attempt.setdefault(key, []).append(e) out = [] for key, events in by_attempt.items(): events = sorted(events, key=lambda e: str(e.get("ts"))) steps, outcome = [], None for e in events: if outcome: break if e.get("event") in CHOICE_EVENTS: d = e.get("decision") or {} text = d.get("choiceText") or e.get("choiceText") if d.get("action") and d["action"] != "choose": continue # paused, nothing happened if not text or not isinstance(e.get("choices"), list): continue scene = e.get("nodeId") or scene_id(e.get("prose") or "", e["choices"]) manual = e["event"] != "DECISION" sc = scenes.get(scene) or {} steps.append({ "scene": scene, "stage": d.get("stage") or sc.get("stage") or "UNKNOWN", "tint": sc.get("tint"), "option": option_id(text), "optionText": text, "offered": [option_id(c) for c in e["choices"]], "rule": d.get("rule"), "decisionMode": "manual" if manual else d.get("source"), "manual": manual, "at": e.get("ts"), }) elif e.get("event") == "BURNED" and steps: outcome = {"type": "BURNED", "colour": None, "intentional": bool(e.get("intentional"))} elif e.get("event") == "DRAGON_BONDED" and steps: outcome = {"type": "BONDED", "colour": str(e["dragon"]).lower() if e.get("dragon") else None, "intentional": False} # a decision repeated on the same scene (the bot re-detecting the screen) counts once dedup = [s for i, s in enumerate(steps) if not (i > 0 and steps[i - 1]["scene"] == s["scene"] and steps[i - 1]["option"] == s["option"])] for i, s in enumerate(dedup): s["to"] = dedup[i + 1]["scene"] if i + 1 < len(dedup) else None if not dedup: continue if outcome: dedup[-1]["to"] = "BURNED" if outcome["type"] == "BURNED" else f"BONDED:{outcome['colour'] or 'unknown'}" try: attempt = int(float(key.split(":")[-1])) or None except ValueError: attempt = None started = dedup[0]["at"] out.append({ "id": "bot-" + sha1(f"{key}:{started}", 10), "source": "bot", "meta": {"attempt": attempt, "startedAt": dedup[0]["at"], "accountKey": None, "target": None, "decisionMode": None}, "steps": dedup, "outcome": outcome or {"type": "ABORTED", "colour": None, "intentional": False}, }) return out def _ms(ts): try: return datetime.fromisoformat(str(ts).replace("Z", "+00:00")).timestamp() * 1000 except ValueError: return float("nan") def attach_run_meta(trajs, runs, salt): """Run-level metadata (target colour, mode) from runs.jsonl, where an attempt is in both.""" window = SPEC["match_hours"] * 3600e3 for r in runs: t = next((x for x in trajs if x["meta"]["attempt"] == r.get("attempt") and abs(_ms(x["meta"]["startedAt"]) - _ms(r.get("startedAt"))) < window), None) if not t: continue m = t["meta"] m["target"] = r.get("targetDragon") m["decisionMode"] = r.get("decisionMode") m["policyId"] = r.get("policyId") m["experimentId"] = r.get("experimentId") m["arm"] = r.get("arm") if r.get("accountKey"): m["accountKey"] = r["accountKey"] elif r.get("accountId") and salt: m["accountKey"] = sha1(f"{salt}:{r['accountId']}", 10) if r.get("outcome") == "BONDED" and r.get("dragonColor"): t["outcome"] = {"type": "BONDED", "colour": str(r["dragonColor"]).lower(), "intentional": False} if r.get("intentionalBurn"): t["outcome"]["intentional"] = True return trajs def import_bot(knowledge, encounters, runs, salt=None): scenes = build_scenes(knowledge, encounters) trajectories = attach_run_meta(trajectories_from_encounters(encounters, scenes), runs, salt) unlabeled = [f"{sid} ({s['stage']}{', ' + s['tint'] if s['tint'] else ''})" for sid, s in scenes.items() if sid not in SPEC["labels"]] return scenes, trajectories, unlabeled def check_public(obj, refuse, max_string): """Nothing that looks like game prose or an account identifier may leave this module.""" def walk(v, p): if isinstance(v, list): for i, x in enumerate(v): walk(x, f"{p}[{i}]") elif isinstance(v, dict): for k, x in v.items(): if k in refuse: raise ValueError(f'refusing to export field "{k}" at {p}') walk(x, f"{p}.{k}") elif isinstance(v, str) and len(v) > max_string: raise ValueError(f"refusing to export a {len(v)}-character string at {p} (looks like prose)") walk(obj, "$") def merge_scenes(existing, scenes): """Add new scenes and new options to the published catalogue; never drop what is already there.""" out = json.loads(json.dumps(existing or {})) for sid, s in scenes.items(): old = out.get(sid) if not old: out[sid] = s continue ids = {o["id"] for o in old["options"]} old["options"] += [o for o in s["options"] if o["id"] not in ids] old["tint"] = old.get("tint") or s["tint"] old["backdrop"] = old.get("backdrop") or s["backdrop"] if "·" in old["label"] and "·" not in s["label"]: old["label"] = s["label"] return dict(sorted(out.items())) # ---------------------------------------------------------------- run records with steps (the bot's schema 2) def _to(key): """The bot's toKey -> ours: a scene id, BURNED or BONDED:.""" key = str(key or "") if key == "TERMINAL:BURNED": return "BURNED" if key.startswith("TERMINAL:BONDED"): return "BONDED:" + (key.split("_", 1)[1].lower() if "_" in key else "unknown") return key or None def _seen(observed): """What the bot saw on a screen that the route model needs: the dragons (from the tint) and the shadow.""" o = observed or {} out = {} if o.get("dragons"): out["dragons"] = [str(d).lower() for d in o["dragons"]] if "shadow" in o: out["shadow"] = bool(o["shadow"]) return out def trajectory_from_run(r, account_key): """A run record that carries its own steps -> a trajectory. Nothing but choice text, ids and outcomes is kept.""" steps = [] for st in r.get("steps") or []: text = st.get("choiceText") or st.get("choiceNorm") if not text: continue steps.append({ "scene": st.get("nodeId"), "stage": st.get("stage") or "UNKNOWN", "tint": st.get("tint"), "option": option_id(text), "optionText": text, "offered": [option_id(o.get("text") or o.get("norm")) for o in st.get("offered") or [] if o.get("text") or o.get("norm")], "rule": st.get("rule"), "decisionMode": "manual" if st.get("manual") else st.get("decisionMode"), "manual": bool(st.get("manual")), "at": st.get("at"), "seen": _seen(st.get("observed")), "propensity": st.get("propensity"), "to": _to(st.get("toKey")), }) colour = str(r["dragonColor"]).lower() if r.get("dragonColor") else None return { "id": "bot-" + sha1(r["runId"], 10), "source": "bot", "meta": {"attempt": r.get("accountAttempt") or r.get("attempt"), "startedAt": r.get("startedAt"), "accountKey": account_key, "accountRole": r.get("accountRole"), "target": r.get("targetDragon"), "decisionMode": r.get("decisionMode"), "policyId": r.get("policyId"), "experimentId": r.get("experimentId"), "arm": r.get("arm")}, "steps": steps, "outcome": {"type": r.get("outcome") or "ABORTED", "colour": colour, "intentional": bool(r.get("intentionalBurn")), "reason": (r.get("burn") or {}).get("reason")}, } def _account_files(files): """{account: {"runs": [...], "encounters": [...]}} from the packed files. The top-level files are the bot's original single account; data/accounts// and logs/accounts// hold one account each.""" out = {} for rel, text in files.items(): if text is None: continue parts = rel.split("/") acct = parts[2] if len(parts) == 4 and parts[1] == "accounts" else "(default)" kind = "runs" if rel.endswith("runs.jsonl") else "encounters" if rel.endswith("encounters.jsonl") else None if kind: out.setdefault(acct, {"runs": [], "encounters": []})[kind] += _jsonl(text) # since the bot keeps one folder per account, its top-level files mirror its first account (the same runs, # logged a moment apart): counting both would count those runs twice if len(out) > 1: out.pop("(default)", None) return out def account_salt(env, privacy): """The salt that turns account names into anonymous keys: the environment variable if set, otherwise one made once and kept in the private raw store (data/raw/bot/account-salt.txt, never committed).""" salt = env.get(privacy.get("account_salt_env", "")) if salt: return salt from ..common import path import secrets p = path("data/raw/bot/account-salt.txt") if not p.exists(): p.parent.mkdir(parents=True, exist_ok=True) p.write_text(secrets.token_hex(16), encoding="utf-8") return p.read_text(encoding="utf-8").strip() def transform(raw_path, spec, ctx): with open(raw_path, encoding="utf-8") as f: files = json.load(f) k = files.get("data/knowledge.json") knowledge = json.loads(k) if k else {"nodes": {}} privacy = spec.get("privacy") or {} salt = account_salt(ctx.get("env", {}), privacy) accounts = _account_files(files) all_encounters = [e for a in accounts.values() for e in a["encounters"]] scenes = build_scenes(knowledge, all_encounters) trajectories, seen_ids = [], set() for acct, data in sorted(accounts.items()): key = sha1(f"{salt}:{acct}", 10) with_steps = [r for r in data["runs"] if r.get("steps") and r.get("runId")] for r in with_steps: t = trajectory_from_run(r, key) if t["id"] not in seen_ids: seen_ids.add(t["id"]) trajectories.append(t) # older runs without steps: rebuild them from the decision log, as before legacy = [r for r in data["runs"] if not r.get("steps")] window = SPEC["match_hours"] * 3600e3 same_run = lambda t, r: (t["meta"]["attempt"] == (r.get("accountAttempt") or r.get("attempt")) # noqa: E731 and abs(_ms(t["meta"]["startedAt"]) - _ms(r.get("startedAt"))) < window) old = attach_run_meta(trajectories_from_encounters(data["encounters"], scenes), legacy, salt) kept = [] for t in old: # the decision log also holds the runs that have steps: keep only the others if any(same_run(t, r) for r in with_steps) or t["id"] in seen_ids: continue t["meta"]["accountKey"] = key seen_ids.add(t["id"]) trajectories.append(t) kept.append(t) # bonded runs logged without any steps still say which dragon was bonded for r in legacy: if r.get("outcome") == "BONDED" and r.get("runId") and not any(same_run(t, r) for t in kept): t = trajectory_from_run({**r, "steps": []}, key) if t["id"] not in seen_ids: seen_ids.add(t["id"]) trajectories.append(t) # scenes the run records show but the knowledge file does not have yet for t in trajectories: for st in t["steps"]: if st["scene"] and st["scene"] not in scenes: scenes[st["scene"]] = {"stage": st["stage"], "tint": st["tint"], "backdrop": None, "label": label_for(st["scene"], {"stage": st["stage"], "tint": st["tint"]}), "options": []} sc = scenes.get(st["scene"]) if sc is not None and st["option"] not in {o["id"] for o in sc["options"]}: sc["options"].append({"id": st["option"], "text": st["optionText"]}) unlabeled = [f"{sid} ({s['stage']}{', ' + s['tint'] if s['tint'] else ''})" for sid, s in scenes.items() if sid not in SPEC["labels"]] for obj in (scenes, trajectories): check_public(obj, set(privacy.get("refuse_fields", [])), privacy.get("max_string", 200)) trajectories.sort(key=lambda t: (str(t["meta"].get("startedAt")), t["id"])) return {"scenes": scenes, "trajectories": trajectories, "_unlabeled": unlabeled} def describe(old, new): outcomes = {} for t in new["trajectories"]: outcomes[t["outcome"]["type"]] = outcomes.get(t["outcome"]["type"], 0) + 1 lines = [f"{len(new['scenes'])} scenes, {len(new['trajectories'])} attempts {json.dumps(outcomes)}"] if new.get("_unlabeled"): lines.append(" - Scenes without a short label (add them to collection/specs/bot.yaml): " + ", ".join(new["_unlabeled"])) return lines