From 78e069f03ff642f8a6734b3a40dcd7f7b6155fb1 Mon Sep 17 00:00:00 2001 From: kathleen Date: Thu, 19 Nov 2026 02:20:35 +0000 Subject: [PATCH] Bot handoff: the daily check pulls Ken's bot repo; route model updates go to the bot by pull request - update-sources: download DragonKind/DragonKind (read with ATLAS_BOT_TOKEN, or DRAGONKIND_TOKEN if set) and import it as the bot source. Needs DK_ACCOUNT_SALT (the same salt as earlier imports, so accounts keep their keys); without it the bot source is skipped as before. - The bot load keeps one file, data/bot/trajectories-bot.jsonl: the bot's repo holds every account's full history, so each import is merged by attempt id (attempts already published stay; older per-import files are folded in). One copy of each attempt also keeps build-graph.js from counting an attempt twice. - New workflow push-policy: when policy.json, solver.js or graph.js change on main, open (or update) a pull request on DragonKind/DragonKind that replaces vendor/threshing-atlas/ with them. Ken merges it. - forge open-pr: --repo/--base for another repository, and reuse the open pull request for the same branch. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ --- .gitea/workflows/push-policy.yml | 60 +++++++++++++++++++++++++++++ .gitea/workflows/update-sources.yml | 17 ++++++++ collection/etl/forge.py | 21 +++++++--- collection/pipeline.yaml | 2 +- collection/run.py | 29 ++++++++++++-- 5 files changed, 119 insertions(+), 10 deletions(-) create mode 100644 .gitea/workflows/push-policy.yml diff --git a/.gitea/workflows/push-policy.yml b/.gitea/workflows/push-policy.yml new file mode 100644 index 0000000..6145e2c --- /dev/null +++ b/.gitea/workflows/push-policy.yml @@ -0,0 +1,60 @@ +# When the route model changes on main, send it to the bot: open (or update) a pull request on the bot's repo, +# DragonKind/DragonKind, that replaces vendor/threshing-atlas/{policy.json,solver.js,graph.js} with this repo's +# data/public/policy.json, core/solver.js and core/graph.js. The three always travel together, so the policy and the +# solver that reads it never get out of step. Ken reviews and merges it; the running bot reloads the files after a pull. +# +# Secrets: ATLAS_BOT_TOKEN (repository: read and write, with write access to DragonKind/DragonKind). +name: Send the route model to the bot +on: + push: + branches: [main] + paths: + - data/public/policy.json + - core/solver.js + - core/graph.js + workflow_dispatch: +jobs: + send: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.ATLAS_BOT_TOKEN }} + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install pyyaml + - name: Open or update the pull request on the bot's repo + env: + ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }} + run: | + set -e + dst="$RUNNER_TEMP/dragonkind" + export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0="http.https://git.kenschiano.net/.extraheader" \ + GIT_CONFIG_VALUE_0="AUTHORIZATION: token $ATLAS_BOT_TOKEN" + git clone --quiet --depth 1 https://git.kenschiano.net/DragonKind/DragonKind.git "$dst" + mkdir -p "$dst/vendor/threshing-atlas" + cp data/public/policy.json core/solver.js core/graph.js "$dst/vendor/threshing-atlas/" + id=$(python3 -c "import json; print(json.load(open('data/public/policy.json')).get('id', ''))") + sha=$(git rev-parse --short HEAD) + cd "$dst" + if [ -z "$(git status --porcelain -- vendor/threshing-atlas)" ]; then + echo "The bot already has route model $id."; exit 0 + fi + git config user.name "Atlas data bot" + git config user.email "atlas-bot@noreply.localhost" + git checkout -q -b atlas/route-model + git add vendor/threshing-atlas + git commit -q -m "Threshing Atlas route model $id (atlas $sha)" + git push -q --force origin atlas/route-model + cd "$GITHUB_WORKSPACE" + printf '%s\n' \ + "The Threshing Atlas route model changed (policy \`$id\`, atlas commit \`$sha\`)." \ + "" \ + "This replaces \`vendor/threshing-atlas/\` (policy.json, solver.js, graph.js) together, as \`npm run atlas:sync\` would." \ + "This pull request is updated in place each time the model changes. After merging, pull on the bot's machine:" \ + "the running bot reloads the files on its next decision." \ + "" \ + "Opened automatically by the Atlas's push-policy workflow." > "$RUNNER_TEMP/body.md" + python -m collection forge open-pr --repo DragonKind/DragonKind --base main --head atlas/route-model \ + --title "Threshing Atlas route model update" --body-file "$RUNNER_TEMP/body.md" diff --git a/.gitea/workflows/update-sources.yml b/.gitea/workflows/update-sources.yml index d11b00c..676ab63 100644 --- a/.gitea/workflows/update-sources.yml +++ b/.gitea/workflows/update-sources.yml @@ -8,6 +8,9 @@ # update branch and open the pull request; the built-in token cannot start the CI it needs. # RAW_ARCHIVE_KEY optional: encrypts the private raw store kept in the Actions cache between runs # REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET optional: read the megathreads through the Reddit API +# DK_ACCOUNT_SALT the salt that turns bot account names into anonymous keys (keep it the same forever). With it, +# the bot's private repo (DragonKind/DragonKind) is downloaded and imported; without it, skipped. +# DRAGONKIND_TOKEN optional: a read-only token for the bot's repo (ATLAS_BOT_TOKEN is used otherwise) name: Check sources for new data on: schedule: @@ -41,9 +44,23 @@ jobs: if [ -n "$RAW_ARCHIVE_KEY" ] && [ -f "$f" ]; then openssl enc -d -aes-256-cbc -pbkdf2 -pass env:RAW_ARCHIVE_KEY -in "$f" | tar -xz -C data fi + - name: Download the bot's data (DragonKind/DragonKind, private) + env: + DK_TOKEN: ${{ secrets.DRAGONKIND_TOKEN || secrets.ATLAS_BOT_TOKEN }} + DK_ACCOUNT_SALT: ${{ secrets.DK_ACCOUNT_SALT }} + run: | + if [ -n "$DK_TOKEN" ] && [ -n "$DK_ACCOUNT_SALT" ]; then + GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0="http.https://git.kenschiano.net/.extraheader" \ + GIT_CONFIG_VALUE_0="AUTHORIZATION: token $DK_TOKEN" \ + git clone --quiet --depth 1 https://git.kenschiano.net/DragonKind/DragonKind.git "$RUNNER_TEMP/dragonkind" + echo "DRAGONKIND_BOT_DIR=$RUNNER_TEMP/dragonkind" >> "$GITHUB_ENV" + else + echo "DK_ACCOUNT_SALT is not set: the bot source is skipped" + fi - id: check env: ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }} + DK_ACCOUNT_SALT: ${{ secrets.DK_ACCOUNT_SALT }} REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }} REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }} run: python -m collection update --report "$RUNNER_TEMP/report.md" diff --git a/collection/etl/forge.py b/collection/etl/forge.py index 54ab6c7..8a4e812 100644 --- a/collection/etl/forge.py +++ b/collection/etl/forge.py @@ -1,6 +1,6 @@ """The repository host (forge.yaml): links, and the few API calls the pipeline makes, for Gitea and GitHub. - python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge] + python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge] [--repo OWNER/NAME --base B] python -m collection forge open-issue --title T --body-file F (skipped if an open issue has that title) python -m collection forge ensure-label --name N --color HEX @@ -96,10 +96,19 @@ def list_issues(label, env=None, f=None): return sorted(out, key=lambda i: i["number"]) -def open_pr(head, title, body, automerge=False, env=None, f=None): - f = f or config() +def open_pr(head, title, body, automerge=False, env=None, f=None, repo=None, base=None): + """Open a pull request (on this repository, or `repo` on the same host). If one is open for the branch already, + it is reused: pushing to its branch has updated it.""" + f = dict(f or config()) + if repo: + f["repo"] = repo + base = base or f["branch"] tok = token(env) - pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": f["branch"], "title": title, "body": body}, tok, f) + open_prs = _call("GET", f"{api_base(f)}/pulls?state=open&limit=50&per_page=100", tok=tok, f=f) or [] + same = [p for p in open_prs if (p.get("head") or {}).get("ref") == head] + if same: + return same[0]["number"] + pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": base, "title": title, "body": body}, tok, f) n = pr["number"] if automerge: if f["kind"] == "github": @@ -141,6 +150,8 @@ def main(argv): p.add_argument("--title", required=True) p.add_argument("--body-file", required=True) p.add_argument("--automerge", action="store_true") + p.add_argument("--repo", help="another repository on the same host (OWNER/NAME)") + p.add_argument("--base", help="the branch to merge into (default: forge.yaml branch)") i = sub.add_parser("open-issue") i.add_argument("--title", required=True) i.add_argument("--body-file", required=True) @@ -150,7 +161,7 @@ def main(argv): a = ap.parse_args(argv) if a.cmd == "open-pr": body = open(a.body_file, encoding="utf-8").read() - print(f"pull request #{open_pr(a.head, a.title, body, a.automerge)}") + print(f"pull request #{open_pr(a.head, a.title, body, a.automerge, repo=a.repo, base=a.base)}") elif a.cmd == "open-issue": n = open_issue(a.title, open(a.body_file, encoding="utf-8").read()) print(f"issue #{n}" if n else "an open issue with that title exists already") diff --git a/collection/pipeline.yaml b/collection/pipeline.yaml index a94f336..627f67f 100644 --- a/collection/pipeline.yaml +++ b/collection/pipeline.yaml @@ -95,5 +95,5 @@ sources: # data/raw/bot/account-salt.txt); never committed refuse_fields: [prose, proseNorm, proseExcerpt, email, password, accountId, accountName] max_string: 200 # anything longer looks like game prose - load: data/bot # scenes.json (merged) + trajectories-.jsonl + load: data/bot # scenes.json and trajectories-bot.jsonl, both merged with what is published runs: bot diff --git a/collection/run.py b/collection/run.py index cf14237..e766b0e 100644 --- a/collection/run.py +++ b/collection/run.py @@ -28,10 +28,33 @@ from datetime import datetime, timezone from .etl import extract as E from .etl import load as L from .etl import codegen, metadata, report_form, runs, validate -from .etl.common import Report, path, pipeline +from .etl.common import Report, path, pipeline, read_jsonl from .etl.transform import TRANSFORMS +BOT_FILE = "trajectories-bot.jsonl" + + +def merge_trajectories(target, new): + """Every bot attempt ever imported, in one file (data/bot/trajectories-bot.jsonl), sorted by id. The bot's repo + holds each account's full history, so an import is a superset: a re-imported attempt replaces its old copy, and + attempts no longer in the bot's repo (an archived account) stay. Older per-import files are folded in.""" + d = path(target) + olds = sorted(d.glob("trajectories-*.jsonl")) + by_id = {} + for f in olds: + for t in read_jsonl(f): + by_id.setdefault(t["id"], t) + for t in new: + by_id[t["id"]] = t + wrote = L.jsonl_file(f"{target}/{BOT_FILE}", [by_id[k] for k in sorted(by_id)]) + for f in olds: + if f.name != BOT_FILE: + f.unlink() + wrote = True + return wrote + + def update_source(name, spec, manifest, report, local_file=None, env=None): env = os.environ if env is None else env try: @@ -54,9 +77,7 @@ def update_source(name, spec, manifest, report, local_file=None, env=None): old = {"scenes": L.read_json(f"{target}/scenes.json") or {}} lines = mod.describe(old, new) wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False) - # one file per import (date + raw checksum), so two imports on the same day (yours, Ken's) never overwrite - tag = f"{ctx['changed']}-{manifest[name]['sha256'][:8]}" - wrote |= L.jsonl_file(f"{target}/trajectories-{tag}.jsonl", new["trajectories"]) + wrote |= merge_trajectories(target, new["trajectories"]) elif target: old = L.read_json(target) lines = mod.describe(old, new)