Bot import: one folder per account, run records with steps
- the importer reads data/accounts/*/runs.jsonl and logs/accounts/*/encounters.jsonl; with account folders, the top-level files (a mirror of the first account) are skipped - run records that carry their steps (schema 2) become trajectories directly, keeping what the bot saw (shadow, meadow dragons), propensities, decision mode and policy id; older runs are still rebuilt from the decision log; bonded runs without steps are kept - account names become salted hashes (DK_ACCOUNT_SALT, or a private salt in data/raw) - runs_clean.csv: new bot_mode column; saw_shadow and saw_blue_meadow from what the bot saw; deliberate burns say why - crosswalk: the valley Red, leaving squadmates, squad direction, hold your ground, hide/approach reactions, the wounded cadet's other two choices Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
dd25dfaa32
commit
43066ad67f
@@ -10,6 +10,8 @@ Each extractor returns the raw bytes, or raises Skip when the source cannot be r
|
||||
secret, the bot's private repo not on this machine).
|
||||
"""
|
||||
import base64
|
||||
import fnmatch
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
@@ -87,9 +89,12 @@ def local_dir(spec, env):
|
||||
if not root:
|
||||
raise Skip(f"the bot's repo is private: set {spec['env']} to a local copy to import it")
|
||||
files = {}
|
||||
for rel in spec["files"]:
|
||||
p = os.path.join(root, rel)
|
||||
files[rel] = Path(p).read_text(encoding="utf-8") if os.path.exists(p) else None
|
||||
for pattern in spec["files"]: # a pattern may have * for one folder level (one folder per bot account)
|
||||
hits = sorted(glob.glob(os.path.join(root, pattern)))
|
||||
for p in hits:
|
||||
files[os.path.relpath(p, root).replace(os.sep, "/")] = Path(p).read_text(encoding="utf-8")
|
||||
if not hits and "*" not in pattern:
|
||||
files[pattern] = None
|
||||
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
@@ -99,13 +104,16 @@ def pack_local(spec, src):
|
||||
src = Path(src).expanduser()
|
||||
if src.is_dir():
|
||||
return local_dir(spec, {spec["env"]: str(src)})
|
||||
files = {rel: None for rel in spec["files"]}
|
||||
files = {rel: None for rel in spec["files"] if "*" not in rel}
|
||||
with zipfile.ZipFile(src) as z:
|
||||
names = [n for n in z.namelist() if not n.endswith("/") and "__MACOSX" not in n]
|
||||
for rel in spec["files"]:
|
||||
hits = sorted((n for n in names if n == rel or n.endswith("/" + rel)), key=len)
|
||||
if hits:
|
||||
files[rel] = z.read(hits[0]).decode("utf-8")
|
||||
for pattern in spec["files"]:
|
||||
for n in names:
|
||||
# the files may sit under a top folder (a zipped repo): match the pattern on the end of the name
|
||||
parts = n.split("/")
|
||||
rel = next(("/".join(parts[i:]) for i in range(len(parts)) if fnmatch.fnmatchcase("/".join(parts[i:]), pattern)), None)
|
||||
if rel and files.get(rel) is None:
|
||||
files[rel] = z.read(n).decode("utf-8")
|
||||
if not any(files.values()):
|
||||
raise RuntimeError(f"{src.name}: none of {', '.join(spec['files'])} found inside")
|
||||
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
+35
-5
@@ -150,10 +150,27 @@ def bot_variables(traj, crosswalk):
|
||||
return v, unmapped
|
||||
|
||||
|
||||
BURN_REASONS = { # why the bot chose to die, in words (the bot's own reason codes)
|
||||
"TREE_NO_SHADOW_AUTO_BURN": "no shadow at the tree",
|
||||
"MEADOW_NO_BLUE_AUTO_BURN": "no Blue in the meadow",
|
||||
}
|
||||
|
||||
|
||||
def _seen_after(steps, start, key):
|
||||
"""Values the bot saw (step["seen"][key]) from step index `start` on."""
|
||||
out = []
|
||||
for st in steps[start:]:
|
||||
v = (st.get("seen") or {}).get(key)
|
||||
if v is not None:
|
||||
out += v if isinstance(v, list) else [v]
|
||||
return out
|
||||
|
||||
|
||||
def bot(name, spec):
|
||||
cw = [(re.compile(rx), col, val) for rx, col, val in read_yaml(spec["transform"]["spec"])["crosswalk"]]
|
||||
rows = []
|
||||
for traj in bot_trajectories(spec):
|
||||
steps = traj["steps"]
|
||||
v, _ = bot_variables(traj, cw)
|
||||
red = v.pop("red", None)
|
||||
v.pop("scream_investigate", None)
|
||||
@@ -165,18 +182,31 @@ def bot(name, spec):
|
||||
if o["type"] == "BONDED":
|
||||
outcome, colour = "Bonded", (o.get("colour") or "").capitalize() or None
|
||||
elif o["type"] == "BURNED":
|
||||
last_tint = traj["steps"][-1].get("tint")
|
||||
last_tint = steps[-1].get("tint") if steps else None
|
||||
outcome = "Incinerated" if v.get("hour") else "Died before the hour"
|
||||
colour = last_tint.capitalize() if last_tint and v.get("hour") else None
|
||||
if o.get("intentional"):
|
||||
flags.append("deliberate burn (the bot chose to die)")
|
||||
else:
|
||||
why = BURN_REASONS.get(o.get("reason") or "", "")
|
||||
flags.append("deliberate burn (the bot chose to die" + (f": {why}" if why else "") + ")")
|
||||
else: # ABORTED, ENDED_NO_BOND: the run stopped without an ending the bot saw
|
||||
outcome, colour = "No outcome recorded", None
|
||||
if not steps:
|
||||
flags.append("bonded dragon only (the bot logged no steps)")
|
||||
target = (meta.get("target") or "").capitalize() or None
|
||||
row = blank_row(run_id=traj["id"], source=spec["run_source"], date=(meta.get("startedAt") or "")[:10] or None,
|
||||
attempt=meta.get("attempt"), goal=target if target in COLOURS else None,
|
||||
row = blank_row(run_id=traj["id"], source=spec["run_source"], bot_mode=meta.get("decisionMode"),
|
||||
date=(meta.get("startedAt") or "")[:10] or None, attempt=meta.get("attempt"),
|
||||
goal=target if target in COLOURS else None,
|
||||
outcome=outcome, outcome_color=colour, data_flags="; ".join(flags), **v)
|
||||
row["attempt_group"] = attempt_group(row["attempt"])
|
||||
# what the bot saw: the shadow at the treetop, and which dragons waited in the meadow
|
||||
shadow = _seen_after(steps, 0, "shadow")
|
||||
if row["hour"] == "Tree" and shadow:
|
||||
row["saw_shadow"] = any(shadow)
|
||||
if row["branch"] == "Passed the Red: meadow":
|
||||
i = next((k for k, st in enumerate(steps) if re.search(r"no way this is your dragon", _norm(st["optionText"]))), len(steps))
|
||||
meadow = _seen_after(steps, i + 1, "dragons")
|
||||
if meadow or o.get("reason") == "MEADOW_NO_BLUE_AUTO_BURN":
|
||||
row["saw_blue_meadow"] = "blue" in meadow or colour == "Blue"
|
||||
encounter_columns(row, intentional=bool(o.get("intentional")))
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
@@ -215,17 +215,145 @@ def merge_scenes(existing, scenes):
|
||||
return dict(sorted(out.items()))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- run records with steps (the bot's schema 2)
|
||||
|
||||
def _to(key):
|
||||
"""The bot's toKey -> ours: a scene id, BURNED or BONDED:<colour>."""
|
||||
key = str(key or "")
|
||||
if key == "TERMINAL:BURNED":
|
||||
return "BURNED"
|
||||
if key.startswith("TERMINAL:BONDED"):
|
||||
return "BONDED:" + (key.split("_", 1)[1].lower() if "_" in key else "unknown")
|
||||
return key or None
|
||||
|
||||
|
||||
def _seen(observed):
|
||||
"""What the bot saw on a screen that the route model needs: the dragons (from the tint) and the shadow."""
|
||||
o = observed or {}
|
||||
out = {}
|
||||
if o.get("dragons"):
|
||||
out["dragons"] = [str(d).lower() for d in o["dragons"]]
|
||||
if "shadow" in o:
|
||||
out["shadow"] = bool(o["shadow"])
|
||||
return out
|
||||
|
||||
|
||||
def trajectory_from_run(r, account_key):
|
||||
"""A run record that carries its own steps -> a trajectory. Nothing but choice text, ids and outcomes is kept."""
|
||||
steps = []
|
||||
for st in r.get("steps") or []:
|
||||
text = st.get("choiceText") or st.get("choiceNorm")
|
||||
if not text:
|
||||
continue
|
||||
steps.append({
|
||||
"scene": st.get("nodeId"), "stage": st.get("stage") or "UNKNOWN", "tint": st.get("tint"),
|
||||
"option": option_id(text), "optionText": text,
|
||||
"offered": [option_id(o.get("text") or o.get("norm")) for o in st.get("offered") or [] if o.get("text") or o.get("norm")],
|
||||
"rule": st.get("rule"), "decisionMode": "manual" if st.get("manual") else st.get("decisionMode"),
|
||||
"manual": bool(st.get("manual")), "at": st.get("at"), "seen": _seen(st.get("observed")),
|
||||
"propensity": st.get("propensity"), "to": _to(st.get("toKey")),
|
||||
})
|
||||
colour = str(r["dragonColor"]).lower() if r.get("dragonColor") else None
|
||||
return {
|
||||
"id": "bot-" + sha1(r["runId"], 10),
|
||||
"source": "bot",
|
||||
"meta": {"attempt": r.get("accountAttempt") or r.get("attempt"), "startedAt": r.get("startedAt"),
|
||||
"accountKey": account_key, "accountRole": r.get("accountRole"), "target": r.get("targetDragon"),
|
||||
"decisionMode": r.get("decisionMode"), "policyId": r.get("policyId"),
|
||||
"experimentId": r.get("experimentId"), "arm": r.get("arm")},
|
||||
"steps": steps,
|
||||
"outcome": {"type": r.get("outcome") or "ABORTED", "colour": colour, "intentional": bool(r.get("intentionalBurn")),
|
||||
"reason": (r.get("burn") or {}).get("reason")},
|
||||
}
|
||||
|
||||
|
||||
def _account_files(files):
|
||||
"""{account: {"runs": [...], "encounters": [...]}} from the packed files. The top-level files are the bot's
|
||||
original single account; data/accounts/<name>/ and logs/accounts/<name>/ hold one account each."""
|
||||
out = {}
|
||||
for rel, text in files.items():
|
||||
if text is None:
|
||||
continue
|
||||
parts = rel.split("/")
|
||||
acct = parts[2] if len(parts) == 4 and parts[1] == "accounts" else "(default)"
|
||||
kind = "runs" if rel.endswith("runs.jsonl") else "encounters" if rel.endswith("encounters.jsonl") else None
|
||||
if kind:
|
||||
out.setdefault(acct, {"runs": [], "encounters": []})[kind] += _jsonl(text)
|
||||
# since the bot keeps one folder per account, its top-level files mirror its first account (the same runs,
|
||||
# logged a moment apart): counting both would count those runs twice
|
||||
if len(out) > 1:
|
||||
out.pop("(default)", None)
|
||||
return out
|
||||
|
||||
|
||||
def account_salt(env, privacy):
|
||||
"""The salt that turns account names into anonymous keys: the environment variable if set, otherwise one made
|
||||
once and kept in the private raw store (data/raw/bot/account-salt.txt, never committed)."""
|
||||
salt = env.get(privacy.get("account_salt_env", ""))
|
||||
if salt:
|
||||
return salt
|
||||
from ..common import path
|
||||
import secrets
|
||||
p = path("data/raw/bot/account-salt.txt")
|
||||
if not p.exists():
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(secrets.token_hex(16), encoding="utf-8")
|
||||
return p.read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def transform(raw_path, spec, ctx):
|
||||
with open(raw_path, encoding="utf-8") as f:
|
||||
files = json.load(f)
|
||||
k = files.get("data/knowledge.json")
|
||||
knowledge = json.loads(k) if k else {"nodes": {}}
|
||||
encounters = _jsonl(files.get("logs/encounters.jsonl"))
|
||||
runs = _jsonl(files.get("data/runs.jsonl"))
|
||||
privacy = spec.get("privacy") or {}
|
||||
scenes, trajectories, unlabeled = import_bot(knowledge, encounters, runs, ctx.get("env", {}).get(privacy.get("account_salt_env", "")))
|
||||
salt = account_salt(ctx.get("env", {}), privacy)
|
||||
accounts = _account_files(files)
|
||||
all_encounters = [e for a in accounts.values() for e in a["encounters"]]
|
||||
scenes = build_scenes(knowledge, all_encounters)
|
||||
trajectories, seen_ids = [], set()
|
||||
for acct, data in sorted(accounts.items()):
|
||||
key = sha1(f"{salt}:{acct}", 10)
|
||||
with_steps = [r for r in data["runs"] if r.get("steps") and r.get("runId")]
|
||||
for r in with_steps:
|
||||
t = trajectory_from_run(r, key)
|
||||
if t["id"] not in seen_ids:
|
||||
seen_ids.add(t["id"])
|
||||
trajectories.append(t)
|
||||
# older runs without steps: rebuild them from the decision log, as before
|
||||
legacy = [r for r in data["runs"] if not r.get("steps")]
|
||||
window = SPEC["match_hours"] * 3600e3
|
||||
same_run = lambda t, r: (t["meta"]["attempt"] == (r.get("accountAttempt") or r.get("attempt")) # noqa: E731
|
||||
and abs(_ms(t["meta"]["startedAt"]) - _ms(r.get("startedAt"))) < window)
|
||||
old = attach_run_meta(trajectories_from_encounters(data["encounters"], scenes), legacy, salt)
|
||||
kept = []
|
||||
for t in old: # the decision log also holds the runs that have steps: keep only the others
|
||||
if any(same_run(t, r) for r in with_steps) or t["id"] in seen_ids:
|
||||
continue
|
||||
t["meta"]["accountKey"] = key
|
||||
seen_ids.add(t["id"])
|
||||
trajectories.append(t)
|
||||
kept.append(t)
|
||||
# bonded runs logged without any steps still say which dragon was bonded
|
||||
for r in legacy:
|
||||
if r.get("outcome") == "BONDED" and r.get("runId") and not any(same_run(t, r) for t in kept):
|
||||
t = trajectory_from_run({**r, "steps": []}, key)
|
||||
if t["id"] not in seen_ids:
|
||||
seen_ids.add(t["id"])
|
||||
trajectories.append(t)
|
||||
# scenes the run records show but the knowledge file does not have yet
|
||||
for t in trajectories:
|
||||
for st in t["steps"]:
|
||||
if st["scene"] and st["scene"] not in scenes:
|
||||
scenes[st["scene"]] = {"stage": st["stage"], "tint": st["tint"], "backdrop": None,
|
||||
"label": label_for(st["scene"], {"stage": st["stage"], "tint": st["tint"]}), "options": []}
|
||||
sc = scenes.get(st["scene"])
|
||||
if sc is not None and st["option"] not in {o["id"] for o in sc["options"]}:
|
||||
sc["options"].append({"id": st["option"], "text": st["optionText"]})
|
||||
unlabeled = [f"{sid} ({s['stage']}{', ' + s['tint'] if s['tint'] else ''})" for sid, s in scenes.items() if sid not in SPEC["labels"]]
|
||||
for obj in (scenes, trajectories):
|
||||
check_public(obj, set(privacy.get("refuse_fields", [])), privacy.get("max_string", 200))
|
||||
trajectories.sort(key=lambda t: (str(t["meta"].get("startedAt")), t["id"]))
|
||||
return {"scenes": scenes, "trajectories": trajectories, "_unlabeled": unlabeled}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user