Bot import: one folder per account, run records with steps

- the importer reads data/accounts/*/runs.jsonl and logs/accounts/*/encounters.jsonl;
  with account folders, the top-level files (a mirror of the first account) are skipped
- run records that carry their steps (schema 2) become trajectories directly, keeping
  what the bot saw (shadow, meadow dragons), propensities, decision mode and policy id;
  older runs are still rebuilt from the decision log; bonded runs without steps are kept
- account names become salted hashes (DK_ACCOUNT_SALT, or a private salt in data/raw)
- runs_clean.csv: new bot_mode column; saw_shadow and saw_blue_meadow from what the bot
  saw; deliberate burns say why
- crosswalk: the valley Red, leaving squadmates, squad direction, hold your ground,
  hide/approach reactions, the wounded cadet's other two choices

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-10 02:08:57 +00:00
co-authored by Claude Opus 5.5
parent dd25dfaa32
commit 43066ad67f
13 changed files with 6292 additions and 6051 deletions
+131 -3
View File
@@ -215,17 +215,145 @@ def merge_scenes(existing, scenes):
return dict(sorted(out.items()))
# ---------------------------------------------------------------- run records with steps (the bot's schema 2)
def _to(key):
"""The bot's toKey -> ours: a scene id, BURNED or BONDED:<colour>."""
key = str(key or "")
if key == "TERMINAL:BURNED":
return "BURNED"
if key.startswith("TERMINAL:BONDED"):
return "BONDED:" + (key.split("_", 1)[1].lower() if "_" in key else "unknown")
return key or None
def _seen(observed):
"""What the bot saw on a screen that the route model needs: the dragons (from the tint) and the shadow."""
o = observed or {}
out = {}
if o.get("dragons"):
out["dragons"] = [str(d).lower() for d in o["dragons"]]
if "shadow" in o:
out["shadow"] = bool(o["shadow"])
return out
def trajectory_from_run(r, account_key):
"""A run record that carries its own steps -> a trajectory. Nothing but choice text, ids and outcomes is kept."""
steps = []
for st in r.get("steps") or []:
text = st.get("choiceText") or st.get("choiceNorm")
if not text:
continue
steps.append({
"scene": st.get("nodeId"), "stage": st.get("stage") or "UNKNOWN", "tint": st.get("tint"),
"option": option_id(text), "optionText": text,
"offered": [option_id(o.get("text") or o.get("norm")) for o in st.get("offered") or [] if o.get("text") or o.get("norm")],
"rule": st.get("rule"), "decisionMode": "manual" if st.get("manual") else st.get("decisionMode"),
"manual": bool(st.get("manual")), "at": st.get("at"), "seen": _seen(st.get("observed")),
"propensity": st.get("propensity"), "to": _to(st.get("toKey")),
})
colour = str(r["dragonColor"]).lower() if r.get("dragonColor") else None
return {
"id": "bot-" + sha1(r["runId"], 10),
"source": "bot",
"meta": {"attempt": r.get("accountAttempt") or r.get("attempt"), "startedAt": r.get("startedAt"),
"accountKey": account_key, "accountRole": r.get("accountRole"), "target": r.get("targetDragon"),
"decisionMode": r.get("decisionMode"), "policyId": r.get("policyId"),
"experimentId": r.get("experimentId"), "arm": r.get("arm")},
"steps": steps,
"outcome": {"type": r.get("outcome") or "ABORTED", "colour": colour, "intentional": bool(r.get("intentionalBurn")),
"reason": (r.get("burn") or {}).get("reason")},
}
def _account_files(files):
"""{account: {"runs": [...], "encounters": [...]}} from the packed files. The top-level files are the bot's
original single account; data/accounts/<name>/ and logs/accounts/<name>/ hold one account each."""
out = {}
for rel, text in files.items():
if text is None:
continue
parts = rel.split("/")
acct = parts[2] if len(parts) == 4 and parts[1] == "accounts" else "(default)"
kind = "runs" if rel.endswith("runs.jsonl") else "encounters" if rel.endswith("encounters.jsonl") else None
if kind:
out.setdefault(acct, {"runs": [], "encounters": []})[kind] += _jsonl(text)
# since the bot keeps one folder per account, its top-level files mirror its first account (the same runs,
# logged a moment apart): counting both would count those runs twice
if len(out) > 1:
out.pop("(default)", None)
return out
def account_salt(env, privacy):
"""The salt that turns account names into anonymous keys: the environment variable if set, otherwise one made
once and kept in the private raw store (data/raw/bot/account-salt.txt, never committed)."""
salt = env.get(privacy.get("account_salt_env", ""))
if salt:
return salt
from ..common import path
import secrets
p = path("data/raw/bot/account-salt.txt")
if not p.exists():
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(secrets.token_hex(16), encoding="utf-8")
return p.read_text(encoding="utf-8").strip()
def transform(raw_path, spec, ctx):
with open(raw_path, encoding="utf-8") as f:
files = json.load(f)
k = files.get("data/knowledge.json")
knowledge = json.loads(k) if k else {"nodes": {}}
encounters = _jsonl(files.get("logs/encounters.jsonl"))
runs = _jsonl(files.get("data/runs.jsonl"))
privacy = spec.get("privacy") or {}
scenes, trajectories, unlabeled = import_bot(knowledge, encounters, runs, ctx.get("env", {}).get(privacy.get("account_salt_env", "")))
salt = account_salt(ctx.get("env", {}), privacy)
accounts = _account_files(files)
all_encounters = [e for a in accounts.values() for e in a["encounters"]]
scenes = build_scenes(knowledge, all_encounters)
trajectories, seen_ids = [], set()
for acct, data in sorted(accounts.items()):
key = sha1(f"{salt}:{acct}", 10)
with_steps = [r for r in data["runs"] if r.get("steps") and r.get("runId")]
for r in with_steps:
t = trajectory_from_run(r, key)
if t["id"] not in seen_ids:
seen_ids.add(t["id"])
trajectories.append(t)
# older runs without steps: rebuild them from the decision log, as before
legacy = [r for r in data["runs"] if not r.get("steps")]
window = SPEC["match_hours"] * 3600e3
same_run = lambda t, r: (t["meta"]["attempt"] == (r.get("accountAttempt") or r.get("attempt")) # noqa: E731
and abs(_ms(t["meta"]["startedAt"]) - _ms(r.get("startedAt"))) < window)
old = attach_run_meta(trajectories_from_encounters(data["encounters"], scenes), legacy, salt)
kept = []
for t in old: # the decision log also holds the runs that have steps: keep only the others
if any(same_run(t, r) for r in with_steps) or t["id"] in seen_ids:
continue
t["meta"]["accountKey"] = key
seen_ids.add(t["id"])
trajectories.append(t)
kept.append(t)
# bonded runs logged without any steps still say which dragon was bonded
for r in legacy:
if r.get("outcome") == "BONDED" and r.get("runId") and not any(same_run(t, r) for t in kept):
t = trajectory_from_run({**r, "steps": []}, key)
if t["id"] not in seen_ids:
seen_ids.add(t["id"])
trajectories.append(t)
# scenes the run records show but the knowledge file does not have yet
for t in trajectories:
for st in t["steps"]:
if st["scene"] and st["scene"] not in scenes:
scenes[st["scene"]] = {"stage": st["stage"], "tint": st["tint"], "backdrop": None,
"label": label_for(st["scene"], {"stage": st["stage"], "tint": st["tint"]}), "options": []}
sc = scenes.get(st["scene"])
if sc is not None and st["option"] not in {o["id"] for o in sc["options"]}:
sc["options"].append({"id": st["option"], "text": st["optionText"]})
unlabeled = [f"{sid} ({s['stage']}{', ' + s['tint'] if s['tint'] else ''})" for sid, s in scenes.items() if sid not in SPEC["labels"]]
for obj in (scenes, trajectories):
check_public(obj, set(privacy.get("refuse_fields", [])), privacy.get("max_string", 200))
trajectories.sort(key=lambda t: (str(t["meta"].get("startedAt")), t["id"]))
return {"scenes": scenes, "trajectories": trajectories, "_unlabeled": unlabeled}