Bot handoff: pull Ken's bot data daily; send route model updates to the bot by PR #12
@@ -0,0 +1,60 @@
|
||||
# When the route model changes on main, send it to the bot: open (or update) a pull request on the bot's repo,
|
||||
# DragonKind/DragonKind, that replaces vendor/threshing-atlas/{policy.json,solver.js,graph.js} with this repo's
|
||||
# data/public/policy.json, core/solver.js and core/graph.js. The three always travel together, so the policy and the
|
||||
# solver that reads it never get out of step. Ken reviews and merges it; the running bot reloads the files after a pull.
|
||||
#
|
||||
# Secrets: ATLAS_BOT_TOKEN (repository: read and write, with write access to DragonKind/DragonKind).
|
||||
name: Send the route model to the bot
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- data/public/policy.json
|
||||
- core/solver.js
|
||||
- core/graph.js
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
send:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install pyyaml
|
||||
- name: Open or update the pull request on the bot's repo
|
||||
env:
|
||||
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||
run: |
|
||||
set -e
|
||||
dst="$RUNNER_TEMP/dragonkind"
|
||||
export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0="http.https://git.kenschiano.net/.extraheader" \
|
||||
GIT_CONFIG_VALUE_0="AUTHORIZATION: token $ATLAS_BOT_TOKEN"
|
||||
git clone --quiet --depth 1 https://git.kenschiano.net/DragonKind/DragonKind.git "$dst"
|
||||
mkdir -p "$dst/vendor/threshing-atlas"
|
||||
cp data/public/policy.json core/solver.js core/graph.js "$dst/vendor/threshing-atlas/"
|
||||
id=$(python3 -c "import json; print(json.load(open('data/public/policy.json')).get('id', ''))")
|
||||
sha=$(git rev-parse --short HEAD)
|
||||
cd "$dst"
|
||||
if [ -z "$(git status --porcelain -- vendor/threshing-atlas)" ]; then
|
||||
echo "The bot already has route model $id."; exit 0
|
||||
fi
|
||||
git config user.name "Atlas data bot"
|
||||
git config user.email "atlas-bot@noreply.localhost"
|
||||
git checkout -q -b atlas/route-model
|
||||
git add vendor/threshing-atlas
|
||||
git commit -q -m "Threshing Atlas route model $id (atlas $sha)"
|
||||
git push -q --force origin atlas/route-model
|
||||
cd "$GITHUB_WORKSPACE"
|
||||
printf '%s\n' \
|
||||
"The Threshing Atlas route model changed (policy \`$id\`, atlas commit \`$sha\`)." \
|
||||
"" \
|
||||
"This replaces \`vendor/threshing-atlas/\` (policy.json, solver.js, graph.js) together, as \`npm run atlas:sync\` would." \
|
||||
"This pull request is updated in place each time the model changes. After merging, pull on the bot's machine:" \
|
||||
"the running bot reloads the files on its next decision." \
|
||||
"" \
|
||||
"Opened automatically by the Atlas's push-policy workflow." > "$RUNNER_TEMP/body.md"
|
||||
python -m collection forge open-pr --repo DragonKind/DragonKind --base main --head atlas/route-model \
|
||||
--title "Threshing Atlas route model update" --body-file "$RUNNER_TEMP/body.md"
|
||||
@@ -8,6 +8,9 @@
|
||||
# update branch and open the pull request; the built-in token cannot start the CI it needs.
|
||||
# RAW_ARCHIVE_KEY optional: encrypts the private raw store kept in the Actions cache between runs
|
||||
# REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET optional: read the megathreads through the Reddit API
|
||||
# DK_ACCOUNT_SALT the salt that turns bot account names into anonymous keys (keep it the same forever). With it,
|
||||
# the bot's private repo (DragonKind/DragonKind) is downloaded and imported; without it, skipped.
|
||||
# DRAGONKIND_TOKEN optional: a read-only token for the bot's repo (ATLAS_BOT_TOKEN is used otherwise)
|
||||
name: Check sources for new data
|
||||
on:
|
||||
schedule:
|
||||
@@ -41,9 +44,23 @@ jobs:
|
||||
if [ -n "$RAW_ARCHIVE_KEY" ] && [ -f "$f" ]; then
|
||||
openssl enc -d -aes-256-cbc -pbkdf2 -pass env:RAW_ARCHIVE_KEY -in "$f" | tar -xz -C data
|
||||
fi
|
||||
- name: Download the bot's data (DragonKind/DragonKind, private)
|
||||
env:
|
||||
DK_TOKEN: ${{ secrets.DRAGONKIND_TOKEN || secrets.ATLAS_BOT_TOKEN }}
|
||||
DK_ACCOUNT_SALT: ${{ secrets.DK_ACCOUNT_SALT }}
|
||||
run: |
|
||||
if [ -n "$DK_TOKEN" ] && [ -n "$DK_ACCOUNT_SALT" ]; then
|
||||
GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0="http.https://git.kenschiano.net/.extraheader" \
|
||||
GIT_CONFIG_VALUE_0="AUTHORIZATION: token $DK_TOKEN" \
|
||||
git clone --quiet --depth 1 https://git.kenschiano.net/DragonKind/DragonKind.git "$RUNNER_TEMP/dragonkind"
|
||||
echo "DRAGONKIND_BOT_DIR=$RUNNER_TEMP/dragonkind" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "DK_ACCOUNT_SALT is not set: the bot source is skipped"
|
||||
fi
|
||||
- id: check
|
||||
env:
|
||||
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||
DK_ACCOUNT_SALT: ${{ secrets.DK_ACCOUNT_SALT }}
|
||||
REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }}
|
||||
REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }}
|
||||
run: python -m collection update --report "$RUNNER_TEMP/report.md"
|
||||
|
||||
+16
-5
@@ -1,6 +1,6 @@
|
||||
"""The repository host (forge.yaml): links, and the few API calls the pipeline makes, for Gitea and GitHub.
|
||||
|
||||
python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge]
|
||||
python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge] [--repo OWNER/NAME --base B]
|
||||
python -m collection forge open-issue --title T --body-file F (skipped if an open issue has that title)
|
||||
python -m collection forge ensure-label --name N --color HEX
|
||||
|
||||
@@ -96,10 +96,19 @@ def list_issues(label, env=None, f=None):
|
||||
return sorted(out, key=lambda i: i["number"])
|
||||
|
||||
|
||||
def open_pr(head, title, body, automerge=False, env=None, f=None):
|
||||
f = f or config()
|
||||
def open_pr(head, title, body, automerge=False, env=None, f=None, repo=None, base=None):
|
||||
"""Open a pull request (on this repository, or `repo` on the same host). If one is open for the branch already,
|
||||
it is reused: pushing to its branch has updated it."""
|
||||
f = dict(f or config())
|
||||
if repo:
|
||||
f["repo"] = repo
|
||||
base = base or f["branch"]
|
||||
tok = token(env)
|
||||
pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": f["branch"], "title": title, "body": body}, tok, f)
|
||||
open_prs = _call("GET", f"{api_base(f)}/pulls?state=open&limit=50&per_page=100", tok=tok, f=f) or []
|
||||
same = [p for p in open_prs if (p.get("head") or {}).get("ref") == head]
|
||||
if same:
|
||||
return same[0]["number"]
|
||||
pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": base, "title": title, "body": body}, tok, f)
|
||||
n = pr["number"]
|
||||
if automerge:
|
||||
if f["kind"] == "github":
|
||||
@@ -141,6 +150,8 @@ def main(argv):
|
||||
p.add_argument("--title", required=True)
|
||||
p.add_argument("--body-file", required=True)
|
||||
p.add_argument("--automerge", action="store_true")
|
||||
p.add_argument("--repo", help="another repository on the same host (OWNER/NAME)")
|
||||
p.add_argument("--base", help="the branch to merge into (default: forge.yaml branch)")
|
||||
i = sub.add_parser("open-issue")
|
||||
i.add_argument("--title", required=True)
|
||||
i.add_argument("--body-file", required=True)
|
||||
@@ -150,7 +161,7 @@ def main(argv):
|
||||
a = ap.parse_args(argv)
|
||||
if a.cmd == "open-pr":
|
||||
body = open(a.body_file, encoding="utf-8").read()
|
||||
print(f"pull request #{open_pr(a.head, a.title, body, a.automerge)}")
|
||||
print(f"pull request #{open_pr(a.head, a.title, body, a.automerge, repo=a.repo, base=a.base)}")
|
||||
elif a.cmd == "open-issue":
|
||||
n = open_issue(a.title, open(a.body_file, encoding="utf-8").read())
|
||||
print(f"issue #{n}" if n else "an open issue with that title exists already")
|
||||
|
||||
@@ -95,5 +95,5 @@ sources:
|
||||
# data/raw/bot/account-salt.txt); never committed
|
||||
refuse_fields: [prose, proseNorm, proseExcerpt, email, password, accountId, accountName]
|
||||
max_string: 200 # anything longer looks like game prose
|
||||
load: data/bot # scenes.json (merged) + trajectories-<date>.jsonl
|
||||
load: data/bot # scenes.json and trajectories-bot.jsonl, both merged with what is published
|
||||
runs: bot
|
||||
|
||||
+25
-4
@@ -28,10 +28,33 @@ from datetime import datetime, timezone
|
||||
from .etl import extract as E
|
||||
from .etl import load as L
|
||||
from .etl import codegen, metadata, report_form, runs, validate
|
||||
from .etl.common import Report, path, pipeline
|
||||
from .etl.common import Report, path, pipeline, read_jsonl
|
||||
from .etl.transform import TRANSFORMS
|
||||
|
||||
|
||||
BOT_FILE = "trajectories-bot.jsonl"
|
||||
|
||||
|
||||
def merge_trajectories(target, new):
|
||||
"""Every bot attempt ever imported, in one file (data/bot/trajectories-bot.jsonl), sorted by id. The bot's repo
|
||||
holds each account's full history, so an import is a superset: a re-imported attempt replaces its old copy, and
|
||||
attempts no longer in the bot's repo (an archived account) stay. Older per-import files are folded in."""
|
||||
d = path(target)
|
||||
olds = sorted(d.glob("trajectories-*.jsonl"))
|
||||
by_id = {}
|
||||
for f in olds:
|
||||
for t in read_jsonl(f):
|
||||
by_id.setdefault(t["id"], t)
|
||||
for t in new:
|
||||
by_id[t["id"]] = t
|
||||
wrote = L.jsonl_file(f"{target}/{BOT_FILE}", [by_id[k] for k in sorted(by_id)])
|
||||
for f in olds:
|
||||
if f.name != BOT_FILE:
|
||||
f.unlink()
|
||||
wrote = True
|
||||
return wrote
|
||||
|
||||
|
||||
def update_source(name, spec, manifest, report, local_file=None, env=None):
|
||||
env = os.environ if env is None else env
|
||||
try:
|
||||
@@ -54,9 +77,7 @@ def update_source(name, spec, manifest, report, local_file=None, env=None):
|
||||
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
|
||||
lines = mod.describe(old, new)
|
||||
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
|
||||
# one file per import (date + raw checksum), so two imports on the same day (yours, Ken's) never overwrite
|
||||
tag = f"{ctx['changed']}-{manifest[name]['sha256'][:8]}"
|
||||
wrote |= L.jsonl_file(f"{target}/trajectories-{tag}.jsonl", new["trajectories"])
|
||||
wrote |= merge_trajectories(target, new["trajectories"])
|
||||
elif target:
|
||||
old = L.read_json(target)
|
||||
lines = mod.describe(old, new)
|
||||
|
||||
Reference in New Issue
Block a user