Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
30aabdf084 | ||
|
|
78e069f03f |
@@ -0,0 +1,60 @@
|
|||||||
|
# When the route model changes on main, send it to the bot: open (or update) a pull request on the bot's repo,
|
||||||
|
# DragonKind/DragonKind, that replaces vendor/threshing-atlas/{policy.json,solver.js,graph.js} with this repo's
|
||||||
|
# data/public/policy.json, core/solver.js and core/graph.js. The three always travel together, so the policy and the
|
||||||
|
# solver that reads it never get out of step. Ken reviews and merges it; the running bot reloads the files after a pull.
|
||||||
|
#
|
||||||
|
# Secrets: ATLAS_BOT_TOKEN (repository: read and write, with write access to DragonKind/DragonKind).
|
||||||
|
name: Send the route model to the bot
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
paths:
|
||||||
|
- data/public/policy.json
|
||||||
|
- core/solver.js
|
||||||
|
- core/graph.js
|
||||||
|
workflow_dispatch:
|
||||||
|
jobs:
|
||||||
|
send:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
token: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- run: pip install pyyaml
|
||||||
|
- name: Open or update the pull request on the bot's repo
|
||||||
|
env:
|
||||||
|
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -e
|
||||||
|
dst="$RUNNER_TEMP/dragonkind"
|
||||||
|
export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0="http.https://git.kenschiano.net/.extraheader" \
|
||||||
|
GIT_CONFIG_VALUE_0="AUTHORIZATION: token $ATLAS_BOT_TOKEN"
|
||||||
|
git clone --quiet --depth 1 https://git.kenschiano.net/DragonKind/DragonKind.git "$dst"
|
||||||
|
mkdir -p "$dst/vendor/threshing-atlas"
|
||||||
|
cp data/public/policy.json core/solver.js core/graph.js "$dst/vendor/threshing-atlas/"
|
||||||
|
id=$(python3 -c "import json; print(json.load(open('data/public/policy.json')).get('id', ''))")
|
||||||
|
sha=$(git rev-parse --short HEAD)
|
||||||
|
cd "$dst"
|
||||||
|
if [ -z "$(git status --porcelain -- vendor/threshing-atlas)" ]; then
|
||||||
|
echo "The bot already has route model $id."; exit 0
|
||||||
|
fi
|
||||||
|
git config user.name "Atlas data bot"
|
||||||
|
git config user.email "atlas-bot@noreply.localhost"
|
||||||
|
git checkout -q -b atlas/route-model
|
||||||
|
git add vendor/threshing-atlas
|
||||||
|
git commit -q -m "Threshing Atlas route model $id (atlas $sha)"
|
||||||
|
git push -q --force origin atlas/route-model
|
||||||
|
cd "$GITHUB_WORKSPACE"
|
||||||
|
printf '%s\n' \
|
||||||
|
"The Threshing Atlas route model changed (policy \`$id\`, atlas commit \`$sha\`)." \
|
||||||
|
"" \
|
||||||
|
"This replaces \`vendor/threshing-atlas/\` (policy.json, solver.js, graph.js) together, as \`npm run atlas:sync\` would." \
|
||||||
|
"This pull request is updated in place each time the model changes. After merging, pull on the bot's machine:" \
|
||||||
|
"the running bot reloads the files on its next decision." \
|
||||||
|
"" \
|
||||||
|
"Opened automatically by the Atlas's push-policy workflow." > "$RUNNER_TEMP/body.md"
|
||||||
|
python -m collection forge open-pr --repo DragonKind/DragonKind --base main --head atlas/route-model \
|
||||||
|
--title "Threshing Atlas route model update" --body-file "$RUNNER_TEMP/body.md"
|
||||||
@@ -8,6 +8,9 @@
|
|||||||
# update branch and open the pull request; the built-in token cannot start the CI it needs.
|
# update branch and open the pull request; the built-in token cannot start the CI it needs.
|
||||||
# RAW_ARCHIVE_KEY optional: encrypts the private raw store kept in the Actions cache between runs
|
# RAW_ARCHIVE_KEY optional: encrypts the private raw store kept in the Actions cache between runs
|
||||||
# REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET optional: read the megathreads through the Reddit API
|
# REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET optional: read the megathreads through the Reddit API
|
||||||
|
# DK_ACCOUNT_SALT the salt that turns bot account names into anonymous keys (keep it the same forever). With it,
|
||||||
|
# the bot's private repo (DragonKind/DragonKind) is downloaded and imported; without it, skipped.
|
||||||
|
# DRAGONKIND_TOKEN optional: a read-only token for the bot's repo (ATLAS_BOT_TOKEN is used otherwise)
|
||||||
name: Check sources for new data
|
name: Check sources for new data
|
||||||
on:
|
on:
|
||||||
schedule:
|
schedule:
|
||||||
@@ -41,9 +44,23 @@ jobs:
|
|||||||
if [ -n "$RAW_ARCHIVE_KEY" ] && [ -f "$f" ]; then
|
if [ -n "$RAW_ARCHIVE_KEY" ] && [ -f "$f" ]; then
|
||||||
openssl enc -d -aes-256-cbc -pbkdf2 -pass env:RAW_ARCHIVE_KEY -in "$f" | tar -xz -C data
|
openssl enc -d -aes-256-cbc -pbkdf2 -pass env:RAW_ARCHIVE_KEY -in "$f" | tar -xz -C data
|
||||||
fi
|
fi
|
||||||
|
- name: Download the bot's data (DragonKind/DragonKind, private)
|
||||||
|
env:
|
||||||
|
DK_TOKEN: ${{ secrets.DRAGONKIND_TOKEN || secrets.ATLAS_BOT_TOKEN }}
|
||||||
|
DK_ACCOUNT_SALT: ${{ secrets.DK_ACCOUNT_SALT }}
|
||||||
|
run: |
|
||||||
|
if [ -n "$DK_TOKEN" ] && [ -n "$DK_ACCOUNT_SALT" ]; then
|
||||||
|
GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0="http.https://git.kenschiano.net/.extraheader" \
|
||||||
|
GIT_CONFIG_VALUE_0="AUTHORIZATION: token $DK_TOKEN" \
|
||||||
|
git clone --quiet --depth 1 https://git.kenschiano.net/DragonKind/DragonKind.git "$RUNNER_TEMP/dragonkind"
|
||||||
|
echo "DRAGONKIND_BOT_DIR=$RUNNER_TEMP/dragonkind" >> "$GITHUB_ENV"
|
||||||
|
else
|
||||||
|
echo "DK_ACCOUNT_SALT is not set: the bot source is skipped"
|
||||||
|
fi
|
||||||
- id: check
|
- id: check
|
||||||
env:
|
env:
|
||||||
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||||
|
DK_ACCOUNT_SALT: ${{ secrets.DK_ACCOUNT_SALT }}
|
||||||
REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }}
|
REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }}
|
||||||
REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }}
|
REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }}
|
||||||
run: python -m collection update --report "$RUNNER_TEMP/report.md"
|
run: python -m collection update --report "$RUNNER_TEMP/report.md"
|
||||||
|
|||||||
+16
-5
@@ -1,6 +1,6 @@
|
|||||||
"""The repository host (forge.yaml): links, and the few API calls the pipeline makes, for Gitea and GitHub.
|
"""The repository host (forge.yaml): links, and the few API calls the pipeline makes, for Gitea and GitHub.
|
||||||
|
|
||||||
python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge]
|
python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge] [--repo OWNER/NAME --base B]
|
||||||
python -m collection forge open-issue --title T --body-file F (skipped if an open issue has that title)
|
python -m collection forge open-issue --title T --body-file F (skipped if an open issue has that title)
|
||||||
python -m collection forge ensure-label --name N --color HEX
|
python -m collection forge ensure-label --name N --color HEX
|
||||||
|
|
||||||
@@ -96,10 +96,19 @@ def list_issues(label, env=None, f=None):
|
|||||||
return sorted(out, key=lambda i: i["number"])
|
return sorted(out, key=lambda i: i["number"])
|
||||||
|
|
||||||
|
|
||||||
def open_pr(head, title, body, automerge=False, env=None, f=None):
|
def open_pr(head, title, body, automerge=False, env=None, f=None, repo=None, base=None):
|
||||||
f = f or config()
|
"""Open a pull request (on this repository, or `repo` on the same host). If one is open for the branch already,
|
||||||
|
it is reused: pushing to its branch has updated it."""
|
||||||
|
f = dict(f or config())
|
||||||
|
if repo:
|
||||||
|
f["repo"] = repo
|
||||||
|
base = base or f["branch"]
|
||||||
tok = token(env)
|
tok = token(env)
|
||||||
pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": f["branch"], "title": title, "body": body}, tok, f)
|
open_prs = _call("GET", f"{api_base(f)}/pulls?state=open&limit=50&per_page=100", tok=tok, f=f) or []
|
||||||
|
same = [p for p in open_prs if (p.get("head") or {}).get("ref") == head]
|
||||||
|
if same:
|
||||||
|
return same[0]["number"]
|
||||||
|
pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": base, "title": title, "body": body}, tok, f)
|
||||||
n = pr["number"]
|
n = pr["number"]
|
||||||
if automerge:
|
if automerge:
|
||||||
if f["kind"] == "github":
|
if f["kind"] == "github":
|
||||||
@@ -141,6 +150,8 @@ def main(argv):
|
|||||||
p.add_argument("--title", required=True)
|
p.add_argument("--title", required=True)
|
||||||
p.add_argument("--body-file", required=True)
|
p.add_argument("--body-file", required=True)
|
||||||
p.add_argument("--automerge", action="store_true")
|
p.add_argument("--automerge", action="store_true")
|
||||||
|
p.add_argument("--repo", help="another repository on the same host (OWNER/NAME)")
|
||||||
|
p.add_argument("--base", help="the branch to merge into (default: forge.yaml branch)")
|
||||||
i = sub.add_parser("open-issue")
|
i = sub.add_parser("open-issue")
|
||||||
i.add_argument("--title", required=True)
|
i.add_argument("--title", required=True)
|
||||||
i.add_argument("--body-file", required=True)
|
i.add_argument("--body-file", required=True)
|
||||||
@@ -150,7 +161,7 @@ def main(argv):
|
|||||||
a = ap.parse_args(argv)
|
a = ap.parse_args(argv)
|
||||||
if a.cmd == "open-pr":
|
if a.cmd == "open-pr":
|
||||||
body = open(a.body_file, encoding="utf-8").read()
|
body = open(a.body_file, encoding="utf-8").read()
|
||||||
print(f"pull request #{open_pr(a.head, a.title, body, a.automerge)}")
|
print(f"pull request #{open_pr(a.head, a.title, body, a.automerge, repo=a.repo, base=a.base)}")
|
||||||
elif a.cmd == "open-issue":
|
elif a.cmd == "open-issue":
|
||||||
n = open_issue(a.title, open(a.body_file, encoding="utf-8").read())
|
n = open_issue(a.title, open(a.body_file, encoding="utf-8").read())
|
||||||
print(f"issue #{n}" if n else "an open issue with that title exists already")
|
print(f"issue #{n}" if n else "an open issue with that title exists already")
|
||||||
|
|||||||
@@ -95,5 +95,5 @@ sources:
|
|||||||
# data/raw/bot/account-salt.txt); never committed
|
# data/raw/bot/account-salt.txt); never committed
|
||||||
refuse_fields: [prose, proseNorm, proseExcerpt, email, password, accountId, accountName]
|
refuse_fields: [prose, proseNorm, proseExcerpt, email, password, accountId, accountName]
|
||||||
max_string: 200 # anything longer looks like game prose
|
max_string: 200 # anything longer looks like game prose
|
||||||
load: data/bot # scenes.json (merged) + trajectories-<date>.jsonl
|
load: data/bot # scenes.json and trajectories-bot.jsonl, both merged with what is published
|
||||||
runs: bot
|
runs: bot
|
||||||
|
|||||||
+25
-4
@@ -28,10 +28,33 @@ from datetime import datetime, timezone
|
|||||||
from .etl import extract as E
|
from .etl import extract as E
|
||||||
from .etl import load as L
|
from .etl import load as L
|
||||||
from .etl import codegen, metadata, report_form, runs, validate
|
from .etl import codegen, metadata, report_form, runs, validate
|
||||||
from .etl.common import Report, path, pipeline
|
from .etl.common import Report, path, pipeline, read_jsonl
|
||||||
from .etl.transform import TRANSFORMS
|
from .etl.transform import TRANSFORMS
|
||||||
|
|
||||||
|
|
||||||
|
BOT_FILE = "trajectories-bot.jsonl"
|
||||||
|
|
||||||
|
|
||||||
|
def merge_trajectories(target, new):
|
||||||
|
"""Every bot attempt ever imported, in one file (data/bot/trajectories-bot.jsonl), sorted by id. The bot's repo
|
||||||
|
holds each account's full history, so an import is a superset: a re-imported attempt replaces its old copy, and
|
||||||
|
attempts no longer in the bot's repo (an archived account) stay. Older per-import files are folded in."""
|
||||||
|
d = path(target)
|
||||||
|
olds = sorted(d.glob("trajectories-*.jsonl"))
|
||||||
|
by_id = {}
|
||||||
|
for f in olds:
|
||||||
|
for t in read_jsonl(f):
|
||||||
|
by_id.setdefault(t["id"], t)
|
||||||
|
for t in new:
|
||||||
|
by_id[t["id"]] = t
|
||||||
|
wrote = L.jsonl_file(f"{target}/{BOT_FILE}", [by_id[k] for k in sorted(by_id)])
|
||||||
|
for f in olds:
|
||||||
|
if f.name != BOT_FILE:
|
||||||
|
f.unlink()
|
||||||
|
wrote = True
|
||||||
|
return wrote
|
||||||
|
|
||||||
|
|
||||||
def update_source(name, spec, manifest, report, local_file=None, env=None):
|
def update_source(name, spec, manifest, report, local_file=None, env=None):
|
||||||
env = os.environ if env is None else env
|
env = os.environ if env is None else env
|
||||||
try:
|
try:
|
||||||
@@ -54,9 +77,7 @@ def update_source(name, spec, manifest, report, local_file=None, env=None):
|
|||||||
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
|
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
|
||||||
lines = mod.describe(old, new)
|
lines = mod.describe(old, new)
|
||||||
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
|
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
|
||||||
# one file per import (date + raw checksum), so two imports on the same day (yours, Ken's) never overwrite
|
wrote |= merge_trajectories(target, new["trajectories"])
|
||||||
tag = f"{ctx['changed']}-{manifest[name]['sha256'][:8]}"
|
|
||||||
wrote |= L.jsonl_file(f"{target}/trajectories-{tag}.jsonl", new["trajectories"])
|
|
||||||
elif target:
|
elif target:
|
||||||
old = L.read_json(target)
|
old = L.read_json(target)
|
||||||
lines = mod.describe(old, new)
|
lines = mod.describe(old, new)
|
||||||
|
|||||||
Reference in New Issue
Block a user