Merge pull request 'Bot handoff: pull Ken's bot data daily; send route model updates to the bot by PR' (#12) from bot-handoff into main
CI / test (push) Successful in 38s

This commit was merged in pull request #12.
This commit is contained in:
2026-11-19 02:21:38 +00:00
5 changed files with 119 additions and 10 deletions
+60
View File
@@ -0,0 +1,60 @@
# When the route model changes on main, send it to the bot: open (or update) a pull request on the bot's repo,
# DragonKind/DragonKind, that replaces vendor/threshing-atlas/{policy.json,solver.js,graph.js} with this repo's
# data/public/policy.json, core/solver.js and core/graph.js. The three always travel together, so the policy and the
# solver that reads it never get out of step. Ken reviews and merges it; the running bot reloads the files after a pull.
#
# Secrets: ATLAS_BOT_TOKEN (repository: read and write, with write access to DragonKind/DragonKind).
name: Send the route model to the bot
on:
push:
branches: [main]
paths:
- data/public/policy.json
- core/solver.js
- core/graph.js
workflow_dispatch:
jobs:
send:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
token: ${{ secrets.ATLAS_BOT_TOKEN }}
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- run: pip install pyyaml
- name: Open or update the pull request on the bot's repo
env:
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
run: |
set -e
dst="$RUNNER_TEMP/dragonkind"
export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0="http.https://git.kenschiano.net/.extraheader" \
GIT_CONFIG_VALUE_0="AUTHORIZATION: token $ATLAS_BOT_TOKEN"
git clone --quiet --depth 1 https://git.kenschiano.net/DragonKind/DragonKind.git "$dst"
mkdir -p "$dst/vendor/threshing-atlas"
cp data/public/policy.json core/solver.js core/graph.js "$dst/vendor/threshing-atlas/"
id=$(python3 -c "import json; print(json.load(open('data/public/policy.json')).get('id', ''))")
sha=$(git rev-parse --short HEAD)
cd "$dst"
if [ -z "$(git status --porcelain -- vendor/threshing-atlas)" ]; then
echo "The bot already has route model $id."; exit 0
fi
git config user.name "Atlas data bot"
git config user.email "atlas-bot@noreply.localhost"
git checkout -q -b atlas/route-model
git add vendor/threshing-atlas
git commit -q -m "Threshing Atlas route model $id (atlas $sha)"
git push -q --force origin atlas/route-model
cd "$GITHUB_WORKSPACE"
printf '%s\n' \
"The Threshing Atlas route model changed (policy \`$id\`, atlas commit \`$sha\`)." \
"" \
"This replaces \`vendor/threshing-atlas/\` (policy.json, solver.js, graph.js) together, as \`npm run atlas:sync\` would." \
"This pull request is updated in place each time the model changes. After merging, pull on the bot's machine:" \
"the running bot reloads the files on its next decision." \
"" \
"Opened automatically by the Atlas's push-policy workflow." > "$RUNNER_TEMP/body.md"
python -m collection forge open-pr --repo DragonKind/DragonKind --base main --head atlas/route-model \
--title "Threshing Atlas route model update" --body-file "$RUNNER_TEMP/body.md"
+17
View File
@@ -8,6 +8,9 @@
# update branch and open the pull request; the built-in token cannot start the CI it needs. # update branch and open the pull request; the built-in token cannot start the CI it needs.
# RAW_ARCHIVE_KEY optional: encrypts the private raw store kept in the Actions cache between runs # RAW_ARCHIVE_KEY optional: encrypts the private raw store kept in the Actions cache between runs
# REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET optional: read the megathreads through the Reddit API # REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET optional: read the megathreads through the Reddit API
# DK_ACCOUNT_SALT the salt that turns bot account names into anonymous keys (keep it the same forever). With it,
# the bot's private repo (DragonKind/DragonKind) is downloaded and imported; without it, skipped.
# DRAGONKIND_TOKEN optional: a read-only token for the bot's repo (ATLAS_BOT_TOKEN is used otherwise)
name: Check sources for new data name: Check sources for new data
on: on:
schedule: schedule:
@@ -41,9 +44,23 @@ jobs:
if [ -n "$RAW_ARCHIVE_KEY" ] && [ -f "$f" ]; then if [ -n "$RAW_ARCHIVE_KEY" ] && [ -f "$f" ]; then
openssl enc -d -aes-256-cbc -pbkdf2 -pass env:RAW_ARCHIVE_KEY -in "$f" | tar -xz -C data openssl enc -d -aes-256-cbc -pbkdf2 -pass env:RAW_ARCHIVE_KEY -in "$f" | tar -xz -C data
fi fi
- name: Download the bot's data (DragonKind/DragonKind, private)
env:
DK_TOKEN: ${{ secrets.DRAGONKIND_TOKEN || secrets.ATLAS_BOT_TOKEN }}
DK_ACCOUNT_SALT: ${{ secrets.DK_ACCOUNT_SALT }}
run: |
if [ -n "$DK_TOKEN" ] && [ -n "$DK_ACCOUNT_SALT" ]; then
GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0="http.https://git.kenschiano.net/.extraheader" \
GIT_CONFIG_VALUE_0="AUTHORIZATION: token $DK_TOKEN" \
git clone --quiet --depth 1 https://git.kenschiano.net/DragonKind/DragonKind.git "$RUNNER_TEMP/dragonkind"
echo "DRAGONKIND_BOT_DIR=$RUNNER_TEMP/dragonkind" >> "$GITHUB_ENV"
else
echo "DK_ACCOUNT_SALT is not set: the bot source is skipped"
fi
- id: check - id: check
env: env:
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }} ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
DK_ACCOUNT_SALT: ${{ secrets.DK_ACCOUNT_SALT }}
REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }} REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }}
REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }} REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }}
run: python -m collection update --report "$RUNNER_TEMP/report.md" run: python -m collection update --report "$RUNNER_TEMP/report.md"
+16 -5
View File
@@ -1,6 +1,6 @@
"""The repository host (forge.yaml): links, and the few API calls the pipeline makes, for Gitea and GitHub. """The repository host (forge.yaml): links, and the few API calls the pipeline makes, for Gitea and GitHub.
python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge] python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge] [--repo OWNER/NAME --base B]
python -m collection forge open-issue --title T --body-file F (skipped if an open issue has that title) python -m collection forge open-issue --title T --body-file F (skipped if an open issue has that title)
python -m collection forge ensure-label --name N --color HEX python -m collection forge ensure-label --name N --color HEX
@@ -96,10 +96,19 @@ def list_issues(label, env=None, f=None):
return sorted(out, key=lambda i: i["number"]) return sorted(out, key=lambda i: i["number"])
def open_pr(head, title, body, automerge=False, env=None, f=None): def open_pr(head, title, body, automerge=False, env=None, f=None, repo=None, base=None):
f = f or config() """Open a pull request (on this repository, or `repo` on the same host). If one is open for the branch already,
it is reused: pushing to its branch has updated it."""
f = dict(f or config())
if repo:
f["repo"] = repo
base = base or f["branch"]
tok = token(env) tok = token(env)
pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": f["branch"], "title": title, "body": body}, tok, f) open_prs = _call("GET", f"{api_base(f)}/pulls?state=open&limit=50&per_page=100", tok=tok, f=f) or []
same = [p for p in open_prs if (p.get("head") or {}).get("ref") == head]
if same:
return same[0]["number"]
pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": base, "title": title, "body": body}, tok, f)
n = pr["number"] n = pr["number"]
if automerge: if automerge:
if f["kind"] == "github": if f["kind"] == "github":
@@ -141,6 +150,8 @@ def main(argv):
p.add_argument("--title", required=True) p.add_argument("--title", required=True)
p.add_argument("--body-file", required=True) p.add_argument("--body-file", required=True)
p.add_argument("--automerge", action="store_true") p.add_argument("--automerge", action="store_true")
p.add_argument("--repo", help="another repository on the same host (OWNER/NAME)")
p.add_argument("--base", help="the branch to merge into (default: forge.yaml branch)")
i = sub.add_parser("open-issue") i = sub.add_parser("open-issue")
i.add_argument("--title", required=True) i.add_argument("--title", required=True)
i.add_argument("--body-file", required=True) i.add_argument("--body-file", required=True)
@@ -150,7 +161,7 @@ def main(argv):
a = ap.parse_args(argv) a = ap.parse_args(argv)
if a.cmd == "open-pr": if a.cmd == "open-pr":
body = open(a.body_file, encoding="utf-8").read() body = open(a.body_file, encoding="utf-8").read()
print(f"pull request #{open_pr(a.head, a.title, body, a.automerge)}") print(f"pull request #{open_pr(a.head, a.title, body, a.automerge, repo=a.repo, base=a.base)}")
elif a.cmd == "open-issue": elif a.cmd == "open-issue":
n = open_issue(a.title, open(a.body_file, encoding="utf-8").read()) n = open_issue(a.title, open(a.body_file, encoding="utf-8").read())
print(f"issue #{n}" if n else "an open issue with that title exists already") print(f"issue #{n}" if n else "an open issue with that title exists already")
+1 -1
View File
@@ -95,5 +95,5 @@ sources:
# data/raw/bot/account-salt.txt); never committed # data/raw/bot/account-salt.txt); never committed
refuse_fields: [prose, proseNorm, proseExcerpt, email, password, accountId, accountName] refuse_fields: [prose, proseNorm, proseExcerpt, email, password, accountId, accountName]
max_string: 200 # anything longer looks like game prose max_string: 200 # anything longer looks like game prose
load: data/bot # scenes.json (merged) + trajectories-<date>.jsonl load: data/bot # scenes.json and trajectories-bot.jsonl, both merged with what is published
runs: bot runs: bot
+25 -4
View File
@@ -28,10 +28,33 @@ from datetime import datetime, timezone
from .etl import extract as E from .etl import extract as E
from .etl import load as L from .etl import load as L
from .etl import codegen, metadata, report_form, runs, validate from .etl import codegen, metadata, report_form, runs, validate
from .etl.common import Report, path, pipeline from .etl.common import Report, path, pipeline, read_jsonl
from .etl.transform import TRANSFORMS from .etl.transform import TRANSFORMS
BOT_FILE = "trajectories-bot.jsonl"
def merge_trajectories(target, new):
"""Every bot attempt ever imported, in one file (data/bot/trajectories-bot.jsonl), sorted by id. The bot's repo
holds each account's full history, so an import is a superset: a re-imported attempt replaces its old copy, and
attempts no longer in the bot's repo (an archived account) stay. Older per-import files are folded in."""
d = path(target)
olds = sorted(d.glob("trajectories-*.jsonl"))
by_id = {}
for f in olds:
for t in read_jsonl(f):
by_id.setdefault(t["id"], t)
for t in new:
by_id[t["id"]] = t
wrote = L.jsonl_file(f"{target}/{BOT_FILE}", [by_id[k] for k in sorted(by_id)])
for f in olds:
if f.name != BOT_FILE:
f.unlink()
wrote = True
return wrote
def update_source(name, spec, manifest, report, local_file=None, env=None): def update_source(name, spec, manifest, report, local_file=None, env=None):
env = os.environ if env is None else env env = os.environ if env is None else env
try: try:
@@ -54,9 +77,7 @@ def update_source(name, spec, manifest, report, local_file=None, env=None):
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}} old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
lines = mod.describe(old, new) lines = mod.describe(old, new)
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False) wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
# one file per import (date + raw checksum), so two imports on the same day (yours, Ken's) never overwrite wrote |= merge_trajectories(target, new["trajectories"])
tag = f"{ctx['changed']}-{manifest[name]['sha256'][:8]}"
wrote |= L.jsonl_file(f"{target}/trajectories-{tag}.jsonl", new["trajectories"])
elif target: elif target:
old = L.read_json(target) old = L.read_json(target)
lines = mod.describe(old, new) lines = mod.describe(old, new)