Host on Gitea: forge.yaml, Gitea workflows, host-neutral links
- forge.yaml: the repository's host (Gitea at git.kenschiano.net, limited visibility); the "Report a run" collector, the page's links (data/public/links.json, filled in by site/build.js) and the dataset metadata read it - collection/etl/forge.py: issues, pull requests (merge when checks succeed) and labels for Gitea and GitHub; `python -m collection forge ...` - .gitea/workflows: CI and the daily update (pushes a branch, opens a pull request that merges itself, opens an issue when a source breaks); .github/workflows kept for GitHub - docs: Gitea setup (REPO_SETTINGS), the bot reading atlas files from the repository Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
8713ccff1a
commit
614bfcda7c
@@ -0,0 +1,24 @@
|
|||||||
|
# Tests on every push and pull request (Gitea Actions; GitHub's copy is .github/workflows/ci.yml).
|
||||||
|
# The branch rule for main (docs/REPO_SETTINGS.md) requires this check, named "CI / test".
|
||||||
|
name: CI
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
pull_request:
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
runs-on: ubuntu-latest # a runner label on the server (act_runner); change it if the server uses another
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: 20
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- run: pip install -r requirements.txt
|
||||||
|
- run: npm test
|
||||||
|
- name: graph.json is up to date
|
||||||
|
run: node analysis/network/build-graph.js && git diff --exit-code data/public/graph.json
|
||||||
|
- name: the page builds
|
||||||
|
run: node site/build.js
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
# Every day: check every source (collection/pipeline.yaml), and when something changed, rebuild the runs, the
|
||||||
|
# graph, the route model and the statistics, run the tests and open a pull request that merges itself once CI
|
||||||
|
# passes. A source that cannot be reached, or new answers the coding rules do not know, open an issue instead.
|
||||||
|
# Gitea Actions; GitHub's copy (with GitHub Pages) is .github/workflows/update-sources.yml.
|
||||||
|
#
|
||||||
|
# Secrets (repository Settings -> Actions -> Secrets; docs/REPO_SETTINGS.md):
|
||||||
|
# ATLAS_BOT_TOKEN a Gitea access token (repository: read and write; issue: read and write). Needed to push the
|
||||||
|
# update branch and open the pull request; the built-in token cannot start the CI it needs.
|
||||||
|
# RAW_ARCHIVE_KEY optional: encrypts the private raw store kept in the Actions cache between runs
|
||||||
|
# REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET optional: read the megathreads through the Reddit API
|
||||||
|
name: Check sources for new data
|
||||||
|
on:
|
||||||
|
schedule:
|
||||||
|
- cron: "23 10 * * *" # daily
|
||||||
|
workflow_dispatch:
|
||||||
|
jobs:
|
||||||
|
check:
|
||||||
|
runs-on: ubuntu-latest # a runner label on the server (act_runner)
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
token: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: 20
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- run: pip install -r requirements.txt
|
||||||
|
- name: Restore the private raw archive
|
||||||
|
uses: actions/cache/restore@v4
|
||||||
|
with:
|
||||||
|
path: ${{ runner.temp }}/raw-archive
|
||||||
|
key: raw-archive-none
|
||||||
|
restore-keys: raw-archive-
|
||||||
|
- name: Decrypt it
|
||||||
|
env:
|
||||||
|
RAW_ARCHIVE_KEY: ${{ secrets.RAW_ARCHIVE_KEY }}
|
||||||
|
run: |
|
||||||
|
f="$RUNNER_TEMP/raw-archive/raw.tgz.enc"
|
||||||
|
if [ -n "$RAW_ARCHIVE_KEY" ] && [ -f "$f" ]; then
|
||||||
|
openssl enc -d -aes-256-cbc -pbkdf2 -pass env:RAW_ARCHIVE_KEY -in "$f" | tar -xz -C data
|
||||||
|
fi
|
||||||
|
- id: check
|
||||||
|
env:
|
||||||
|
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||||
|
REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }}
|
||||||
|
REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }}
|
||||||
|
run: python -m collection update --report "$RUNNER_TEMP/report.md"
|
||||||
|
- name: Rebuild the analysis and run tests
|
||||||
|
if: steps.check.outputs.changed == 'true'
|
||||||
|
run: |
|
||||||
|
node analysis/network/build-graph.js
|
||||||
|
npm run policy
|
||||||
|
npm run stats
|
||||||
|
python -m collection build
|
||||||
|
npm test
|
||||||
|
- name: Encrypt the raw store
|
||||||
|
id: archive
|
||||||
|
env:
|
||||||
|
RAW_ARCHIVE_KEY: ${{ secrets.RAW_ARCHIVE_KEY }}
|
||||||
|
run: |
|
||||||
|
if [ -n "$RAW_ARCHIVE_KEY" ] && [ -d data/raw ]; then
|
||||||
|
mkdir -p "$RUNNER_TEMP/raw-archive"
|
||||||
|
tar -czf - -C data raw | openssl enc -aes-256-cbc -pbkdf2 -salt -pass env:RAW_ARCHIVE_KEY -out "$RUNNER_TEMP/raw-archive/raw.tgz.enc"
|
||||||
|
echo "save=true" >> "$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
|
- name: Save it (a new copy only when a raw file changed)
|
||||||
|
if: steps.archive.outputs.save == 'true'
|
||||||
|
uses: actions/cache/save@v4
|
||||||
|
with:
|
||||||
|
path: ${{ runner.temp }}/raw-archive
|
||||||
|
key: raw-archive-${{ hashFiles('data/manifest.json') }}
|
||||||
|
- name: Open a pull request with the new data (it merges itself once CI passes)
|
||||||
|
if: steps.check.outputs.changed == 'true'
|
||||||
|
env:
|
||||||
|
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||||
|
run: |
|
||||||
|
branch="data/source-update-$(date -u +%Y%m%d)"
|
||||||
|
git config user.name "Atlas data bot"
|
||||||
|
git config user.email "atlas-bot@noreply.localhost"
|
||||||
|
git checkout -b "$branch"
|
||||||
|
git add -A data analysis/reports core/graph.js .github/ISSUE_TEMPLATE
|
||||||
|
git commit -m "data: update from community sources"
|
||||||
|
git push origin "$branch"
|
||||||
|
python -m collection forge open-pr --head "$branch" --title "New data from the community sources" --body-file "$RUNNER_TEMP/report.md" --automerge
|
||||||
|
- name: Report a broken source or new answers
|
||||||
|
if: steps.check.outputs.failed != ''
|
||||||
|
env:
|
||||||
|
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||||
|
run: |
|
||||||
|
python -m collection forge open-issue --title "Source check failed: ${{ steps.check.outputs.failed }}" --body-file "$RUNNER_TEMP/report.md"
|
||||||
@@ -9,7 +9,8 @@ body:
|
|||||||
attributes:
|
attributes:
|
||||||
value: |
|
value: |
|
||||||
Thanks for reporting a run! Answer what you remember; leave anything you're unsure of on "Not sure".
|
Thanks for reporting a run! Answer what you remember; leave anything you're unsure of on "Not sure".
|
||||||
Every answer here is **public** (this is a GitHub issue) and goes into the Atlas's open data as source "reported".
|
Every answer here is visible to everyone who can see this repository (it is an issue) and goes into the Atlas's
|
||||||
|
open data as source "reported".
|
||||||
Please don't include your DragonKind account name or anything personal.
|
Please don't include your DragonKind account name or anything personal.
|
||||||
You can also use the community [Threshing Routes form](https://forms.gle/DRpj9tG8gb5E4Hcu7) linked from the r/fourthwing megathread; please report each run only once.
|
You can also use the community [Threshing Routes form](https://forms.gle/DRpj9tG8gb5E4Hcu7) linked from the r/fourthwing megathread; please report each run only once.
|
||||||
- type: dropdown
|
- type: dropdown
|
||||||
|
|||||||
@@ -42,7 +42,8 @@ npm test # pipeline, coding rules, statistics, route model, game spec, a
|
|||||||
|
|
||||||
## New data from the community sources
|
## New data from the community sources
|
||||||
|
|
||||||
A GitHub Action (`.github/workflows/update-sources.yml`) runs the collection pipeline every day
|
A scheduled workflow (`.gitea/workflows/update-sources.yml` on the Atlas's Gitea; `.github/workflows/` holds the
|
||||||
|
GitHub version) runs the collection pipeline every day
|
||||||
(`python -m collection update`, see `collection/README.md`). It checks every source in `collection/pipeline.yaml`,
|
(`python -m collection update`, see `collection/README.md`). It checks every source in `collection/pipeline.yaml`,
|
||||||
and if anything changed it rebuilds the runs and the analysis, runs the tests and opens a pull request that says
|
and if anything changed it rebuilds the runs and the analysis, runs the tests and opens a pull request that says
|
||||||
what changed: new responses, new or renamed questions, new run reports, new result links on Reddit, and for the PDF
|
what changed: new responses, new or renamed questions, new run reports, new result links on Reddit, and for the PDF
|
||||||
@@ -75,7 +76,7 @@ See `docs/BOT_DATA_PROTOCOL.md` for what the bot should log.
|
|||||||
|
|
||||||
## Running everything locally
|
## Running everything locally
|
||||||
|
|
||||||
`python -m collection refresh` does what the daily GitHub workflow does: checks every source, rebuilds the runs,
|
`python -m collection refresh` does what the daily workflow does: checks every source, rebuilds the runs,
|
||||||
the graph, the route model and the statistics. Add `--bot <folder or zip>` to import bot runs, and
|
the graph, the route model and the statistics. Add `--bot <folder or zip>` to import bot runs, and
|
||||||
`python -m collection export-bot-kit` zips the files the bot uses (see `docs/BOT_INTEGRATION.md`).
|
`python -m collection export-bot-kit` zips the files the bot uses (see `docs/BOT_INTEGRATION.md`).
|
||||||
|
|
||||||
|
|||||||
@@ -4,8 +4,24 @@
|
|||||||
"""
|
"""
|
||||||
import json
|
import json
|
||||||
|
|
||||||
|
from . import forge
|
||||||
from .common import path, read_yaml
|
from .common import path, read_yaml
|
||||||
|
|
||||||
|
LINKS = "data/public/links.json"
|
||||||
|
|
||||||
|
|
||||||
|
def links():
|
||||||
|
"""The links the page shows, from forge.yaml and pipeline.yaml (site/build.js fills them into the page)."""
|
||||||
|
f = forge.config()
|
||||||
|
src = read_yaml("collection/pipeline.yaml")["sources"]
|
||||||
|
host = f["url"].split("://", 1)[-1].rstrip("/")
|
||||||
|
return {
|
||||||
|
"repo": forge.repo_url(f), "repo_label": f"{host}/{f['repo']}", "host": host, "kind": f["kind"],
|
||||||
|
"visibility": f.get("visibility", "public"),
|
||||||
|
"report": forge.new_issue_url(".github/ISSUE_TEMPLATE/report-a-run.yml", f),
|
||||||
|
"form": src["reddit_spreadsheet"]["links"]["form"], "sheet": src["reddit_spreadsheet"]["links"]["sheet"],
|
||||||
|
}
|
||||||
|
|
||||||
GRAPH_JS = "core/graph.js"
|
GRAPH_JS = "core/graph.js"
|
||||||
BEGIN = " // BEGIN CROSSWALK"
|
BEGIN = " // BEGIN CROSSWALK"
|
||||||
END = " // END CROSSWALK\n"
|
END = " // END CROSSWALK\n"
|
||||||
|
|||||||
@@ -64,22 +64,10 @@ def reddit(spec, env):
|
|||||||
return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
def github_issues(spec, env):
|
def forge_issues(spec, env):
|
||||||
"""Every issue with the label (open and closed), through the GitHub REST API."""
|
"""Every issue with the label (open and closed), from the repository's host (forge.yaml: Gitea or GitHub)."""
|
||||||
headers = {"Accept": "application/vnd.github+json"}
|
from . import forge
|
||||||
token = env.get("GITHUB_TOKEN") or env.get("GH_TOKEN")
|
issues = forge.list_issues(spec["label"], env)
|
||||||
if token:
|
|
||||||
headers["Authorization"] = f"Bearer {token}"
|
|
||||||
label = urllib.parse.quote(spec["label"])
|
|
||||||
issues, page = [], 1
|
|
||||||
while True:
|
|
||||||
url = f"https://api.github.com/repos/{spec['repo']}/issues?labels={label}&state=all&per_page=100&page={page}"
|
|
||||||
batch = json.loads(_get(url, headers=headers))
|
|
||||||
issues += [i for i in batch if "pull_request" not in i]
|
|
||||||
if len(batch) < 100:
|
|
||||||
break
|
|
||||||
page += 1
|
|
||||||
issues.sort(key=lambda i: i["number"])
|
|
||||||
return json.dumps(issues, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
return json.dumps(issues, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
@@ -119,7 +107,7 @@ def pack_local(spec, src):
|
|||||||
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir}
|
EXTRACTORS = {"http": http, "reddit": reddit, "forge_issues": forge_issues, "local_dir": local_dir}
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- the raw store
|
# ---------------------------------------------------------------- the raw store
|
||||||
|
|||||||
@@ -0,0 +1,163 @@
|
|||||||
|
"""The repository host (forge.yaml): links, and the few API calls the pipeline makes, for Gitea and GitHub.
|
||||||
|
|
||||||
|
python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge]
|
||||||
|
python -m collection forge open-issue --title T --body-file F (skipped if an open issue has that title)
|
||||||
|
python -m collection forge ensure-label --name N --color HEX
|
||||||
|
|
||||||
|
The token comes from ATLAS_BOT_TOKEN, then GITEA_TOKEN / GITHUB_TOKEN / GH_TOKEN.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
from .common import read_yaml
|
||||||
|
|
||||||
|
USER_AGENT = "threshing-atlas-collector/1.0"
|
||||||
|
|
||||||
|
|
||||||
|
def config():
|
||||||
|
return read_yaml("forge.yaml")
|
||||||
|
|
||||||
|
|
||||||
|
def repo_url(f=None):
|
||||||
|
f = f or config()
|
||||||
|
return f"{f['url'].rstrip('/')}/{f['repo']}"
|
||||||
|
|
||||||
|
|
||||||
|
def file_url(path, f=None, raw=True):
|
||||||
|
"""A file in the repository on its main branch (raw content, or the page that shows it)."""
|
||||||
|
f = f or config()
|
||||||
|
if f["kind"] == "github":
|
||||||
|
if raw:
|
||||||
|
return f"https://raw.githubusercontent.com/{f['repo']}/{f['branch']}/{path}"
|
||||||
|
return f"{repo_url(f)}/blob/{f['branch']}/{path}"
|
||||||
|
return f"{repo_url(f)}/{'raw' if raw else 'src'}/branch/{f['branch']}/{path}"
|
||||||
|
|
||||||
|
|
||||||
|
def new_issue_url(template, f=None):
|
||||||
|
"""The "new issue" page for an issue form in .github/ISSUE_TEMPLATE/."""
|
||||||
|
f = f or config()
|
||||||
|
if f["kind"] == "github":
|
||||||
|
return f"{repo_url(f)}/issues/new?template={os.path.basename(template)}"
|
||||||
|
return f"{repo_url(f)}/issues/new?template={urllib.parse.quote(template)}"
|
||||||
|
|
||||||
|
|
||||||
|
def api_base(f=None):
|
||||||
|
f = f or config()
|
||||||
|
if f["kind"] == "github":
|
||||||
|
return f"https://api.github.com/repos/{f['repo']}"
|
||||||
|
return f"{f['url'].rstrip('/')}/api/v1/repos/{f['repo']}"
|
||||||
|
|
||||||
|
|
||||||
|
def token(env=None):
|
||||||
|
env = os.environ if env is None else env
|
||||||
|
for k in ("ATLAS_BOT_TOKEN", "GITEA_TOKEN", "GITHUB_TOKEN", "GH_TOKEN"):
|
||||||
|
if env.get(k):
|
||||||
|
return env[k]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _call(method, url, body=None, tok=None, f=None):
|
||||||
|
f = f or config()
|
||||||
|
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
||||||
|
if tok:
|
||||||
|
headers["Authorization"] = f"{'Bearer' if f['kind'] == 'github' else 'token'} {tok}"
|
||||||
|
data = None
|
||||||
|
if body is not None:
|
||||||
|
data = json.dumps(body).encode("utf-8")
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
req = urllib.request.Request(url, data=data, method=method, headers=headers)
|
||||||
|
with urllib.request.urlopen(req, timeout=60) as r:
|
||||||
|
text = r.read().decode("utf-8")
|
||||||
|
return json.loads(text) if text.strip() else None
|
||||||
|
|
||||||
|
|
||||||
|
def list_issues(label, env=None, f=None):
|
||||||
|
"""Every issue (open and closed, no pull requests) with the label, oldest first, in GitHub's field names
|
||||||
|
(number, body, labels, state, state_reason, created_at), which both hosts share."""
|
||||||
|
f = f or config()
|
||||||
|
tok = token(env)
|
||||||
|
lab = urllib.parse.quote(label)
|
||||||
|
out, page = [], 1
|
||||||
|
while True:
|
||||||
|
if f["kind"] == "github":
|
||||||
|
url = f"{api_base(f)}/issues?labels={lab}&state=all&per_page=100&page={page}"
|
||||||
|
else:
|
||||||
|
url = f"{api_base(f)}/issues?labels={lab}&state=all&type=issues&limit=50&page={page}"
|
||||||
|
batch = _call("GET", url, tok=tok, f=f) or []
|
||||||
|
out += [i for i in batch if "pull_request" not in i or not i["pull_request"]]
|
||||||
|
if len(batch) < (100 if f["kind"] == "github" else 50):
|
||||||
|
break
|
||||||
|
page += 1
|
||||||
|
for i in out:
|
||||||
|
i.setdefault("state_reason", None) # Gitea has no close reason: reports are rejected with labels
|
||||||
|
return sorted(out, key=lambda i: i["number"])
|
||||||
|
|
||||||
|
|
||||||
|
def open_pr(head, title, body, automerge=False, env=None, f=None):
|
||||||
|
f = f or config()
|
||||||
|
tok = token(env)
|
||||||
|
pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": f["branch"], "title": title, "body": body}, tok, f)
|
||||||
|
n = pr["number"]
|
||||||
|
if automerge:
|
||||||
|
if f["kind"] == "github":
|
||||||
|
# GitHub's auto-merge needs GraphQL; the repo's auto-merge setting merges it when checks pass
|
||||||
|
print(f"opened #{n}; turn on auto-merge in the pull request page if it is not on by default")
|
||||||
|
else:
|
||||||
|
_call("POST", f"{api_base(f)}/pulls/{n}/merge",
|
||||||
|
{"Do": "squash", "merge_when_checks_succeed": True, "delete_branch_after_merge": True}, tok, f)
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
|
def open_issue(title, body, env=None, f=None):
|
||||||
|
f = f or config()
|
||||||
|
tok = token(env)
|
||||||
|
q = urllib.parse.quote(title)
|
||||||
|
found = _call("GET", f"{api_base(f)}/issues?state=open&type=issues&q={q}" if f["kind"] == "gitea"
|
||||||
|
else f"{api_base(f)}/issues?state=open&per_page=100", tok=tok, f=f) or []
|
||||||
|
if any(i.get("title") == title for i in found):
|
||||||
|
return None
|
||||||
|
return _call("POST", f"{api_base(f)}/issues", {"title": title, "body": body}, tok, f)["number"]
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_label(name, color, env=None, f=None):
|
||||||
|
f = f or config()
|
||||||
|
tok = token(env)
|
||||||
|
labels = _call("GET", f"{api_base(f)}/labels?limit=50&per_page=100", tok=tok, f=f) or []
|
||||||
|
if any(lb["name"] == name for lb in labels):
|
||||||
|
return False
|
||||||
|
_call("POST", f"{api_base(f)}/labels", {"name": name, "color": ("#" if f["kind"] == "gitea" else "") + color.lstrip("#")}, tok, f)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv):
|
||||||
|
import argparse
|
||||||
|
ap = argparse.ArgumentParser(prog="python -m collection forge")
|
||||||
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||||
|
p = sub.add_parser("open-pr")
|
||||||
|
p.add_argument("--head", required=True)
|
||||||
|
p.add_argument("--title", required=True)
|
||||||
|
p.add_argument("--body-file", required=True)
|
||||||
|
p.add_argument("--automerge", action="store_true")
|
||||||
|
i = sub.add_parser("open-issue")
|
||||||
|
i.add_argument("--title", required=True)
|
||||||
|
i.add_argument("--body-file", required=True)
|
||||||
|
lb = sub.add_parser("ensure-label")
|
||||||
|
lb.add_argument("--name", required=True)
|
||||||
|
lb.add_argument("--color", default="e8b07e")
|
||||||
|
a = ap.parse_args(argv)
|
||||||
|
if a.cmd == "open-pr":
|
||||||
|
body = open(a.body_file, encoding="utf-8").read()
|
||||||
|
print(f"pull request #{open_pr(a.head, a.title, body, a.automerge)}")
|
||||||
|
elif a.cmd == "open-issue":
|
||||||
|
n = open_issue(a.title, open(a.body_file, encoding="utf-8").read())
|
||||||
|
print(f"issue #{n}" if n else "an open issue with that title exists already")
|
||||||
|
elif a.cmd == "ensure-label":
|
||||||
|
print("created" if ensure_label(a.name, a.color) else "exists")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main(sys.argv[1:]))
|
||||||
@@ -7,6 +7,7 @@ Written by `python -m collection build` (a test checks it is up to date).
|
|||||||
"""
|
"""
|
||||||
import json
|
import json
|
||||||
|
|
||||||
|
from . import forge
|
||||||
from .common import ROOT, pipeline, read_yaml, sha256_file
|
from .common import ROOT, pipeline, read_yaml, sha256_file
|
||||||
|
|
||||||
OUT = "data/croissant.json"
|
OUT = "data/croissant.json"
|
||||||
@@ -67,6 +68,7 @@ def field_description(spec):
|
|||||||
def build(dictionary=None):
|
def build(dictionary=None):
|
||||||
d = dictionary or load_dictionary()
|
d = dictionary or load_dictionary()
|
||||||
ds = d["dataset"]
|
ds = d["dataset"]
|
||||||
|
fg = forge.config()
|
||||||
sources = pipeline()["sources"]
|
sources = pipeline()["sources"]
|
||||||
|
|
||||||
distribution = []
|
distribution = []
|
||||||
@@ -77,7 +79,7 @@ def build(dictionary=None):
|
|||||||
"@id": key,
|
"@id": key,
|
||||||
"name": key,
|
"name": key,
|
||||||
"description": " ".join(spec["description"].split()),
|
"description": " ".join(spec["description"].split()),
|
||||||
"contentUrl": spec.get("url") or f"{ds['repository']}/blob/main/{spec['path']}",
|
"contentUrl": forge.file_url(spec["path"], fg),
|
||||||
"encodingFormat": spec["format"],
|
"encodingFormat": spec["format"],
|
||||||
"sha256": sha256(path),
|
"sha256": sha256(path),
|
||||||
"contentSize": f"{path.stat().st_size} B",
|
"contentSize": f"{path.stat().st_size} B",
|
||||||
@@ -102,12 +104,12 @@ def build(dictionary=None):
|
|||||||
"conformsTo": "http://mlcommons.org/croissant/1.0",
|
"conformsTo": "http://mlcommons.org/croissant/1.0",
|
||||||
"name": ds["name"],
|
"name": ds["name"],
|
||||||
"description": " ".join(ds["description"].split()),
|
"description": " ".join(ds["description"].split()),
|
||||||
"url": ds["url"],
|
"url": fg.get("site") or forge.repo_url(fg),
|
||||||
"sameAs": ds["repository"],
|
"sameAs": forge.repo_url(fg),
|
||||||
"license": ds["license"],
|
"license": ds["license"],
|
||||||
"creator": {"@type": "sc:Person", "name": ds["creator"], "url": f"https://github.com/{ds['creator']}"},
|
"creator": {"@type": "sc:Person", "name": ds["creator"]},
|
||||||
"keywords": ds["keywords"],
|
"keywords": ds["keywords"],
|
||||||
"citeAs": ds["citation"],
|
"citeAs": ds["citation"].format(repository=forge.repo_url(fg)),
|
||||||
"isLiveDataset": True,
|
"isLiveDataset": True,
|
||||||
"isBasedOn": [
|
"isBasedOn": [
|
||||||
{"@type": "sc:CreativeWork", "name": s["label"], "url": next(iter(s["links"].values()))}
|
{"@type": "sc:CreativeWork", "name": s["label"], "url": next(iter(s["links"].values()))}
|
||||||
|
|||||||
@@ -66,11 +66,11 @@ sources:
|
|||||||
runs: reddit_results
|
runs: reddit_results
|
||||||
|
|
||||||
reported:
|
reported:
|
||||||
label: Runs reported on the Atlas ("Report a run", a GitHub issue form)
|
label: Runs reported on the Atlas ("Report a run", an issue form on the Atlas's repository)
|
||||||
run_source: reported
|
run_source: reported
|
||||||
links: { report: "https://github.com/ekcSchiano/threshing-atlas/issues/new?template=report-a-run.yml" }
|
# the repository's host is forge.yaml (Gitea or GitHub); its "new issue" link is made from it
|
||||||
extract: { kind: github_issues, repo: ekcSchiano/threshing-atlas, label: run report, ext: json }
|
extract: { kind: forge_issues, label: run report, ext: json }
|
||||||
# GitHub usernames are dropped; issues closed as "not planned" or labelled "invalid" are left out
|
# usernames are dropped; issues closed as "not planned" (GitHub) or labelled invalid/duplicate are left out
|
||||||
transform: { name: reports, form: collection/specs/report_form.yaml, reject_labels: [invalid, duplicate] }
|
transform: { name: reports, form: collection/specs/report_form.yaml, reject_labels: [invalid, duplicate] }
|
||||||
load: data/public/reports.json
|
load: data/public/reports.json
|
||||||
runs: reports
|
runs: reports
|
||||||
|
|||||||
@@ -78,6 +78,8 @@ def build(check=False, report=None):
|
|||||||
out.append(report_form.TEMPLATE)
|
out.append(report_form.TEMPLATE)
|
||||||
if L.write_bytes(codegen.GRAPH_JS, codegen.graph_js().encode("utf-8"), check):
|
if L.write_bytes(codegen.GRAPH_JS, codegen.graph_js().encode("utf-8"), check):
|
||||||
out.append(codegen.GRAPH_JS)
|
out.append(codegen.GRAPH_JS)
|
||||||
|
if L.json_file(codegen.LINKS, codegen.links(), check, compact=False):
|
||||||
|
out.append(codegen.LINKS)
|
||||||
# the metadata hashes the files above, so it is built from what is (or would be) on disk
|
# the metadata hashes the files above, so it is built from what is (or would be) on disk
|
||||||
if not check or not out:
|
if not check or not out:
|
||||||
if L.write_bytes(metadata.OUT, metadata.render(metadata.build()).encode("utf-8"), check):
|
if L.write_bytes(metadata.OUT, metadata.render(metadata.build()).encode("utf-8"), check):
|
||||||
@@ -100,12 +102,17 @@ def main(argv=None):
|
|||||||
up.add_argument("--report")
|
up.add_argument("--report")
|
||||||
b = sub.add_parser("build")
|
b = sub.add_parser("build")
|
||||||
b.add_argument("--check", action="store_true")
|
b.add_argument("--check", action="store_true")
|
||||||
|
fg = sub.add_parser("forge", help="pull requests, issues and labels on the repository's host (forge.yaml)")
|
||||||
|
fg.add_argument("args", nargs=argparse.REMAINDER)
|
||||||
k = sub.add_parser("export-bot-kit", help="zip policy.json, solver.js and graph.js for the bot")
|
k = sub.add_parser("export-bot-kit", help="zip policy.json, solver.js and graph.js for the bot")
|
||||||
k.add_argument("--out", default="exports")
|
k.add_argument("--out", default="exports")
|
||||||
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
||||||
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
||||||
a = ap.parse_args(argv)
|
a = ap.parse_args(argv)
|
||||||
|
|
||||||
|
if a.cmd == "forge":
|
||||||
|
from .etl import forge
|
||||||
|
return forge.main(a.args)
|
||||||
if a.cmd == "export-bot-kit":
|
if a.cmd == "export-bot-kit":
|
||||||
from .etl.handoff import export_bot_kit
|
from .etl.handoff import export_bot_kit
|
||||||
print(f"wrote {export_bot_kit(a.out)}")
|
print(f"wrote {export_bot_kit(a.out)}")
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# "Report a run": the questions of the GitHub issue form, and how each answer becomes a column of runs_clean.csv.
|
# "Report a run": the questions of the issue form (on the Atlas's repository: forge.yaml), and how each answer becomes a column of runs_clean.csv.
|
||||||
#
|
#
|
||||||
# collection/etl/report_form.py writes .github/ISSUE_TEMPLATE/report-a-run.yml from this file (a test checks it is
|
# collection/etl/report_form.py writes .github/ISSUE_TEMPLATE/report-a-run.yml from this file (a test checks it is
|
||||||
# up to date), and collection/etl/runs.py codes every report with it. Each option is "label shown: column value";
|
# up to date), and collection/etl/runs.py codes every report with it. Each option is "label shown: column value";
|
||||||
@@ -11,7 +11,8 @@ title: "Run report"
|
|||||||
label: run report
|
label: run report
|
||||||
intro: |
|
intro: |
|
||||||
Thanks for reporting a run! Answer what you remember; leave anything you're unsure of on "Not sure".
|
Thanks for reporting a run! Answer what you remember; leave anything you're unsure of on "Not sure".
|
||||||
Every answer here is **public** (this is a GitHub issue) and goes into the Atlas's open data as source "reported".
|
Every answer here is visible to everyone who can see this repository (it is an issue) and goes into the Atlas's
|
||||||
|
open data as source "reported".
|
||||||
Please don't include your DragonKind account name or anything personal.
|
Please don't include your DragonKind account name or anything personal.
|
||||||
You can also use the community [Threshing Routes form](https://forms.gle/DRpj9tG8gb5E4Hcu7) linked from the r/fourthwing megathread; please report each run only once.
|
You can also use the community [Threshing Routes form](https://forms.gle/DRpj9tG8gb5E4Hcu7) linked from the r/fourthwing megathread; please report each run only once.
|
||||||
|
|
||||||
|
|||||||
@@ -14,19 +14,17 @@ dataset:
|
|||||||
Community-logged runs of the Dragonkind "threshing" (the dragon-bonding game based on Fourth Wing), coded into
|
Community-logged runs of the Dragonkind "threshing" (the dragon-bonding game based on Fourth Wing), coded into
|
||||||
one record per run: every choice, what the game rolled, and how the run ended. Also the route model built from
|
one record per run: every choice, what the game rolled, and how the run ended. Also the route model built from
|
||||||
them, the statistics shown on the Threshing Atlas page, and the route graph shared with a DragonKind bot.
|
them, the statistics shown on the Threshing Atlas page, and the route graph shared with a DragonKind bot.
|
||||||
url: https://ekcschiano.github.io/threshing-atlas/
|
# the repository, the page and every file's link come from forge.yaml
|
||||||
repository: https://github.com/ekcSchiano/threshing-atlas
|
|
||||||
license: https://creativecommons.org/licenses/by/4.0/
|
license: https://creativecommons.org/licenses/by/4.0/
|
||||||
code_license: MIT
|
code_license: MIT
|
||||||
creator: ekcSchiano
|
creator: ekcSchiano
|
||||||
keywords: [Dragonkind, Fourth Wing, threshing, decision tree, route planning, community data]
|
keywords: [Dragonkind, Fourth Wing, threshing, decision tree, route planning, community data]
|
||||||
citation: "ekcSchiano and contributors (2026). Threshing Atlas. https://github.com/ekcSchiano/threshing-atlas"
|
citation: "ekcSchiano and contributors (2026). Threshing Atlas. {repository}"
|
||||||
# Third-party sources keep their own terms (DATA_LICENSE.md); the route PDF is never redistributed.
|
# Third-party sources keep their own terms (DATA_LICENSE.md); the route PDF is never redistributed.
|
||||||
|
|
||||||
files:
|
files:
|
||||||
runs_clean:
|
runs_clean:
|
||||||
path: data/public/runs_clean.csv
|
path: data/public/runs_clean.csv
|
||||||
url: https://ekcschiano.github.io/threshing-atlas/data/runs_clean.csv
|
|
||||||
format: text/csv
|
format: text/csv
|
||||||
description: >-
|
description: >-
|
||||||
Every run from every source, one per row, coded into the same columns (collection/etl/runs.py; the form's
|
Every run from every source, one per row, coded into the same columns (collection/etl/runs.py; the form's
|
||||||
@@ -40,29 +38,25 @@ files:
|
|||||||
reports:
|
reports:
|
||||||
path: data/public/reports.json
|
path: data/public/reports.json
|
||||||
format: application/json
|
format: application/json
|
||||||
description: Runs reported through the Atlas's "Report a run" form (GitHub issues), answers only, no usernames.
|
description: Runs reported through the Atlas's "Report a run" form (issues on its repository), answers only, no usernames.
|
||||||
register:
|
register:
|
||||||
path: data/public/register.json
|
path: data/public/register.json
|
||||||
url: https://ekcschiano.github.io/threshing-atlas/data/register.json
|
|
||||||
format: application/json
|
format: application/json
|
||||||
description: The Unofficial Dragon Register (which dragon each rider bonded), without rider names.
|
description: The Unofficial Dragon Register (which dragon each rider bonded), without rider names.
|
||||||
stats:
|
stats:
|
||||||
path: data/public/stats.json
|
path: data/public/stats.json
|
||||||
url: https://ekcschiano.github.io/threshing-atlas/data/stats.json
|
|
||||||
format: application/json
|
format: application/json
|
||||||
description: >-
|
description: >-
|
||||||
Every number the Atlas page shows, for six filter presets (analysis/stats/site_stats.py; what is measured is
|
Every number the Atlas page shows, for six filter presets (analysis/stats/site_stats.py; what is measured is
|
||||||
analysis/stats/measures.yaml).
|
analysis/stats/measures.yaml).
|
||||||
policy:
|
policy:
|
||||||
path: data/public/policy.json
|
path: data/public/policy.json
|
||||||
url: https://ekcschiano.github.io/threshing-atlas/data/policy.json
|
|
||||||
format: application/json
|
format: application/json
|
||||||
description: >-
|
description: >-
|
||||||
The route model: the game (core/spec/game.yaml), the count tables behind every roll, and the best decision
|
The route model: the game (core/spec/game.yaml), the count tables behind every roll, and the best decision
|
||||||
tree for every dragon colour under two goals, with posterior intervals (planner/build_policy.py).
|
tree for every dragon colour under two goals, with posterior intervals (planner/build_policy.py).
|
||||||
graph:
|
graph:
|
||||||
path: data/public/graph.json
|
path: data/public/graph.json
|
||||||
url: https://ekcschiano.github.io/threshing-atlas/data/graph.json
|
|
||||||
format: application/json
|
format: application/json
|
||||||
description: The route graph over every run in runs_clean.csv, counted per source (analysis/network/build-graph.js).
|
description: The route graph over every run in runs_clean.csv, counted per source (analysis/network/build-graph.js).
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"repo": "https://git.kenschiano.net/YOUR-GITEA-USERNAME/threshing-atlas",
|
||||||
|
"repo_label": "git.kenschiano.net/YOUR-GITEA-USERNAME/threshing-atlas",
|
||||||
|
"host": "git.kenschiano.net",
|
||||||
|
"kind": "gitea",
|
||||||
|
"visibility": "limited",
|
||||||
|
"report": "https://git.kenschiano.net/YOUR-GITEA-USERNAME/threshing-atlas/issues/new?template=.github/ISSUE_TEMPLATE/report-a-run.yml",
|
||||||
|
"form": "https://forms.gle/DRpj9tG8gb5E4Hcu7",
|
||||||
|
"sheet": "https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0"
|
||||||
|
}
|
||||||
+14
-1
@@ -9,7 +9,20 @@ Data flows both ways:
|
|||||||
└──────────── collection --source bot + PR ◄┘
|
└──────────── collection --source bot + PR ◄┘
|
||||||
```
|
```
|
||||||
|
|
||||||
## Without GitHub: swapping zips
|
## From the Atlas's Gitea repository
|
||||||
|
|
||||||
|
Both repositories live on git.kenschiano.net. The bot can read the atlas files straight from the repository with a
|
||||||
|
Gitea token that can read it (the repository is "limited": signed-in users only), instead of a zip:
|
||||||
|
|
||||||
|
```
|
||||||
|
https://git.kenschiano.net/<owner>/threshing-atlas/raw/branch/main/data/public/policy.json
|
||||||
|
https://git.kenschiano.net/<owner>/threshing-atlas/raw/branch/main/core/solver.js
|
||||||
|
https://git.kenschiano.net/<owner>/threshing-atlas/raw/branch/main/core/graph.js
|
||||||
|
```
|
||||||
|
|
||||||
|
(header `Authorization: token <token>`). Keep the three files together: solver.js reads the game from policy.json.
|
||||||
|
|
||||||
|
## Without a shared repository: swapping zips
|
||||||
|
|
||||||
When GitHub Pages is not available, the same files travel as zips:
|
When GitHub Pages is not available, the same files travel as zips:
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -31,7 +31,7 @@ Rules every part of the project follows. If a change would break one, stop and a
|
|||||||
|
|
||||||
## Run reports
|
## Run reports
|
||||||
|
|
||||||
"Report a run" creates a public GitHub issue. The pipeline keeps only the issue number, the date and the answers
|
"Report a run" creates an issue on the Atlas's repository (forge.yaml), visible to everyone who can see it. The pipeline keeps only the issue number, the date and the answers
|
||||||
(`data/public/reports.json`); GitHub usernames are dropped. The form asks people not to include account names. A
|
(`data/public/reports.json`); GitHub usernames are dropped. The form asks people not to include account names. A
|
||||||
report closed as "not planned" or labelled `invalid` or `duplicate` is left out of the data.
|
report closed as "not planned" or labelled `invalid` or `duplicate` is left out of the data.
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,51 @@
|
|||||||
# Repository settings
|
# Repository settings
|
||||||
|
|
||||||
|
The Atlas is hosted on Gitea (`forge.yaml`: git.kenschiano.net). The GitHub repository and its GitHub Pages site are
|
||||||
|
paused while that account is suspended; their workflows stay in `.github/workflows/` for when it is back.
|
||||||
|
|
||||||
|
## Gitea: setting up the repository (once)
|
||||||
|
|
||||||
|
1. **Your account** on git.kenschiano.net (ask Ken), with 2FA on and your SSH key added (Settings → SSH / GPG Keys).
|
||||||
|
2. **Create the repository** `threshing-atlas` under your account, visibility **limited** (only people signed in to
|
||||||
|
the server), empty (no README or licence: they come from the push). Put your username in `forge.yaml` (`repo:`),
|
||||||
|
then `python -m collection build` and commit.
|
||||||
|
3. **Push** from your Mac copy:
|
||||||
|
|
||||||
|
```
|
||||||
|
cd ~/PycharmProjects/threshing-atlas
|
||||||
|
git remote rename origin github # keep the GitHub address for later
|
||||||
|
git remote add origin git@git.kenschiano.net:<you>/threshing-atlas.git
|
||||||
|
git push -u origin main
|
||||||
|
```
|
||||||
|
|
||||||
|
4. **Turn on Actions** (repository Settings → Units → Actions) and ask Ken which runner label the server has. The
|
||||||
|
workflows use `ubuntu-latest`; change `runs-on:` in `.gitea/workflows/*.yml` if it is different.
|
||||||
|
5. **Secrets** (Settings → Actions → Secrets):
|
||||||
|
|
||||||
|
| Secret | What it is | How to make it |
|
||||||
|
|---|---|---|
|
||||||
|
| `ATLAS_BOT_TOKEN` | lets the daily update push its branch, open the pull request and issues | your Settings → Applications → Generate token; permissions: repository read and write, issue read and write |
|
||||||
|
| `RAW_ARCHIVE_KEY` | encrypts the private raw store kept in the Actions cache between runs | any long random string (`openssl rand -base64 32`); keep a copy |
|
||||||
|
| `REDDIT_CLIENT_ID`, `REDDIT_CLIENT_SECRET` | optional: read the megathreads through the Reddit API | see "Data collection secrets" below |
|
||||||
|
|
||||||
|
6. **Protect main** (Settings → Branches → Add rule for `main`): require a pull request, enable status checks and
|
||||||
|
require the check from the CI workflow (it shows up as `CI / test` after its first run), and allow
|
||||||
|
"merge when checks succeed". Leave required approvals at 0 so the daily update can merge itself.
|
||||||
|
7. **The label** for reports: `python -m collection forge ensure-label --name "run report"` (with
|
||||||
|
`ATLAS_BOT_TOKEN` set), or create it under Issues → Labels.
|
||||||
|
|
||||||
|
The daily update then runs at 10:23 UTC; run it by hand from the Actions tab (workflow "Check sources for new data").
|
||||||
|
Locally, `python -m collection refresh` does the same.
|
||||||
|
|
||||||
|
## The page
|
||||||
|
|
||||||
|
The page is built by `node site/build.js` and viewed locally or published as a Claude artifact; Gitea has no Pages.
|
||||||
|
Its links (the repository, "Report a run") come from `forge.yaml` through `data/public/links.json`.
|
||||||
|
|
||||||
|
## GitHub (paused)
|
||||||
|
|
||||||
|
# Repository settings
|
||||||
|
|
||||||
Some settings live on GitHub, not in the code. Only the repository owner (ekcSchiano) can change them.
|
Some settings live on GitHub, not in the code. Only the repository owner (ekcSchiano) can change them.
|
||||||
They go together: the ruleset makes CI a gate on `main`, auto-merge lets a pull request merge itself once
|
They go together: the ruleset makes CI a gate on `main`, auto-merge lets a pull request merge itself once
|
||||||
it passes, and the token lets the daily data update do that on its own.
|
it passes, and the token lets the daily data update do that on its own.
|
||||||
|
|||||||
+22
@@ -0,0 +1,22 @@
|
|||||||
|
# Where the Atlas's repository is hosted. Everything that talks to the host reads this file: the daily workflow
|
||||||
|
# (opening its pull request and issues), the "Report a run" collector, the links on the page and in the dataset
|
||||||
|
# metadata (data/croissant.json). Moving hosts is an edit here (then `python -m collection build`).
|
||||||
|
#
|
||||||
|
# kind: gitea or github. The Gitea workflows live in .gitea/workflows/ (Gitea ignores .github/workflows/ when that
|
||||||
|
# folder exists); the GitHub ones, including GitHub Pages, stay in .github/workflows/ for when GitHub is used again.
|
||||||
|
version: 1
|
||||||
|
|
||||||
|
kind: gitea
|
||||||
|
url: https://git.kenschiano.net
|
||||||
|
repo: YOUR-GITEA-USERNAME/threshing-atlas # owner/name on the server (set your username here)
|
||||||
|
branch: main
|
||||||
|
# public, limited (only people signed in to this server) or private: who can open the links below
|
||||||
|
visibility: limited
|
||||||
|
|
||||||
|
# the published page, when it is hosted somewhere (GitHub Pages is paused while the GitHub account is suspended;
|
||||||
|
# the page is viewed locally or as a Claude artifact meanwhile)
|
||||||
|
site: null
|
||||||
|
|
||||||
|
# other copies of the repository (informational)
|
||||||
|
mirrors:
|
||||||
|
github: https://github.com/ekcSchiano/threshing-atlas # suspended; GitHub Pages workflow kept for later
|
||||||
@@ -608,7 +608,7 @@ a { color: var(--accent); text-underline-offset: 2px; }
|
|||||||
</section>
|
</section>
|
||||||
<section id="share" aria-labelledby="h-share">
|
<section id="share" aria-labelledby="h-share">
|
||||||
<div class="section-head"><span class="eyebrow">Open by design</span><h2 id="h-share">Share & reproduce</h2>
|
<div class="section-head"><span class="eyebrow">Open by design</span><h2 id="h-share">Share & reproduce</h2>
|
||||||
<p class="muted small">Everything on this page is computed from the community form export by the code in the <a href="https://github.com/ekcSchiano/threshing-atlas" target="_blank" rel="noopener">GitHub repository</a>, which updates the data every day. The rules for reading each answer are in <code>core/spec/form.yaml</code>, the game in <code>core/spec/game.yaml</code>, and what is measured in <code>analysis/stats/measures.yaml</code>.</p></div>
|
<p class="muted small">Everything on this page is computed from the community form export by the code in the <a href="{{repo}}" target="_blank" rel="noopener">repository</a>, which updates the data every day. The rules for reading each answer are in <code>core/spec/form.yaml</code>, the game in <code>core/spec/game.yaml</code>, and what is measured in <code>analysis/stats/measures.yaml</code>.</p></div>
|
||||||
<div class="grid2">
|
<div class="grid2">
|
||||||
<div class="panel"><h3>Copy the outputs</h3>
|
<div class="panel"><h3>Copy the outputs</h3>
|
||||||
<div class="btnrow"><button type="button" id="cp-reddit" class="primary">Copy Reddit post (Markdown)</button><a class="btnlike" href="data/runs_clean.csv" download>Download coded runs (CSV)</a><button type="button" id="cp-json">Copy all results (JSON)</button></div>
|
<div class="btnrow"><button type="button" id="cp-reddit" class="primary">Copy Reddit post (Markdown)</button><a class="btnlike" href="data/runs_clean.csv" download>Download coded runs (CSV)</a><button type="button" id="cp-json">Copy all results (JSON)</button></div>
|
||||||
@@ -692,8 +692,8 @@ adjusted OR = exp(βⱼ), 95% CI = exp(βⱼ ± 1.96·SEⱼ)</div>
|
|||||||
<p class="muted">Every run anyone logs ends up in the Atlas's open data, <a href="data/runs_clean.csv" download>runs_clean.csv</a>, which anyone can download and reuse. Deaths are as useful as bonds: the odds on this page are only as good as the runs behind them. Please report each run in one place only.</p></div>
|
<p class="muted">Every run anyone logs ends up in the Atlas's open data, <a href="data/runs_clean.csv" download>runs_clean.csv</a>, which anyone can download and reuse. Deaths are as useful as bonds: the odds on this page are only as good as the runs behind them. Please report each run in one place only.</p></div>
|
||||||
<div class="grid2">
|
<div class="grid2">
|
||||||
<div class="panel"><h3>Report it on the Atlas</h3>
|
<div class="panel"><h3>Report it on the Atlas</h3>
|
||||||
<p class="small">A short form with the game's own choices (a GitHub issue, so you need a free GitHub account). Reports show up in the data the next day with the source <b>reported</b>. Reports are public: please leave out your account name.</p>
|
<p class="small">A short form with the game's own choices: an issue on the Atlas's repository at {{host}}, so you need an account there. Reports show up in the data the next day with the source <b>reported</b>. Everyone who can see the repository can read reports: please leave out your account name.</p>
|
||||||
<p><a class="btnlike primary" href="https://github.com/ekcSchiano/threshing-atlas/issues/new?template=report-a-run.yml" target="_blank" rel="noopener">Report a run</a></p>
|
<p><a class="btnlike primary" href="{{report}}" target="_blank" rel="noopener">Report a run</a></p>
|
||||||
</div>
|
</div>
|
||||||
<div class="panel"><h3>Or use the community form</h3>
|
<div class="panel"><h3>Or use the community form</h3>
|
||||||
<p class="small">The r/fourthwing megathreads collect runs with u/ClumsyBadger's Google Form, and its answers go into a public spreadsheet. The Atlas reads that spreadsheet every day (source <b>reddit spreadsheet</b>), so a run logged there is counted here too.</p>
|
<p class="small">The r/fourthwing megathreads collect runs with u/ClumsyBadger's Google Form, and its answers go into a public spreadsheet. The Atlas reads that spreadsheet every day (source <b>reddit spreadsheet</b>), so a run logged there is counted here too.</p>
|
||||||
@@ -717,7 +717,7 @@ adjusted OR = exp(βⱼ), 95% CI = exp(βⱼ ± 1.96·SEⱼ)</div>
|
|||||||
</div>
|
</div>
|
||||||
</main>
|
</main>
|
||||||
</div>
|
</div>
|
||||||
<footer><div class="wrap"><p><b>Data:</b> the community <a href="https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0" target="_blank" rel="noopener">"Threshing Routes" responses</a> compiled by <a href="https://www.reddit.com/user/ClumsyBadger/" target="_blank" rel="noopener">u/ClumsyBadger</a>, with percentages in the <a href="https://drive.google.com/file/d/17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg/view" target="_blank" rel="noopener">route PDF</a> by <a href="https://www.reddit.com/user/ClumsyBadger/" target="_blank" rel="noopener">u/ClumsyBadger</a>, <a href="https://www.reddit.com/user/Old_Regular3018/" target="_blank" rel="noopener">u/Old_Regular3018</a> and <a href="https://www.reddit.com/user/Smart_Trifle_2099/" target="_blank" rel="noopener">u/Smart_Trifle_2099</a>; the <a href="https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ" target="_blank" rel="noopener">Unofficial Dragon Register</a>; and the r/fourthwing bonding megathreads (<a href="https://www.reddit.com/r/fourthwing/comments/1wvwfp9/" target="_blank" rel="noopener">Part 3</a>, <a href="https://www.reddit.com/r/fourthwing/comments/1wz5cnj/" target="_blank" rel="noopener">Part 4</a>). Code and data: <a href="https://github.com/ekcSchiano/threshing-atlas" target="_blank" rel="noopener">github.com/ekcSchiano/threshing-atlas</a> (MIT, data CC BY 4.0).</p>
|
<footer><div class="wrap"><p><b>Data:</b> the community <a href="https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0" target="_blank" rel="noopener">"Threshing Routes" responses</a> compiled by <a href="https://www.reddit.com/user/ClumsyBadger/" target="_blank" rel="noopener">u/ClumsyBadger</a>, with percentages in the <a href="https://drive.google.com/file/d/17o72ehHsSPW4UdsKF-sXTxMLEuinWeEg/view" target="_blank" rel="noopener">route PDF</a> by <a href="https://www.reddit.com/user/ClumsyBadger/" target="_blank" rel="noopener">u/ClumsyBadger</a>, <a href="https://www.reddit.com/user/Old_Regular3018/" target="_blank" rel="noopener">u/Old_Regular3018</a> and <a href="https://www.reddit.com/user/Smart_Trifle_2099/" target="_blank" rel="noopener">u/Smart_Trifle_2099</a>; the <a href="https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ" target="_blank" rel="noopener">Unofficial Dragon Register</a>; and the r/fourthwing bonding megathreads (<a href="https://www.reddit.com/r/fourthwing/comments/1wvwfp9/" target="_blank" rel="noopener">Part 3</a>, <a href="https://www.reddit.com/r/fourthwing/comments/1wz5cnj/" target="_blank" rel="noopener">Part 4</a>). Code and data: <a href="{{repo}}" target="_blank" rel="noopener">{{repo_label}}</a> (MIT, data CC BY 4.0).</p>
|
||||||
<p><b>Unofficial fan analysis,</b> not affiliated with <a href="https://dragonkind.com" target="_blank" rel="noopener">Dragonkind</a> or its authors. A 10% "chaos" chance of incineration on any choice has been described by the author, so <b>no route guarantees a bond</b>.</p></div></footer>
|
<p><b>Unofficial fan analysis,</b> not affiliated with <a href="https://dragonkind.com" target="_blank" rel="noopener">Dragonkind</a> or its authors. A 10% "chaos" chance of incineration on any choice has been described by the author, so <b>no route guarantees a bond</b>.</p></div></footer>
|
||||||
<div id="tip" hidden></div>
|
<div id="tip" hidden></div>
|
||||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js"></script>
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js"></script>
|
||||||
|
|||||||
+4
-1
@@ -16,7 +16,10 @@ for (const f of fs.readdirSync(path.join(root, "site/atlas"))) copy(`site/atlas/
|
|||||||
// local serving, give it its own doctype, charset and viewport.
|
// local serving, give it its own doctype, charset and viewport.
|
||||||
{
|
{
|
||||||
const p = path.join(out, "index.html");
|
const p = path.join(out, "index.html");
|
||||||
const html = fs.readFileSync(p, "utf8");
|
// links from forge.yaml (data/public/links.json, written by `python -m collection build`)
|
||||||
|
const links = JSON.parse(fs.readFileSync(path.join(root, "data/public/links.json"), "utf8"));
|
||||||
|
const html = fs.readFileSync(p, "utf8").replace(/\{\{(\w+)\}\}/g, (m, k) => (k in links ? links[k] : m));
|
||||||
|
fs.writeFileSync(p, html);
|
||||||
if (!/^\s*<!doctype/i.test(html))
|
if (!/^\s*<!doctype/i.test(html))
|
||||||
fs.writeFileSync(p, `<!doctype html>\n<meta charset="utf-8">\n<meta name="viewport" content="width=device-width, initial-scale=1">\n${html}`);
|
fs.writeFileSync(p, `<!doctype html>\n<meta charset="utf-8">\n<meta name="viewport" content="width=device-width, initial-scale=1">\n${html}`);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
"""The collection pipeline: extract's raw store, every transform, the coders, validation and the generated files."""
|
"""The collection pipeline: extract's raw store, every transform, the coders, validation and the generated files."""
|
||||||
import csv
|
import csv
|
||||||
import json
|
import json
|
||||||
|
import re
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
@@ -167,6 +168,54 @@ class Handoff(unittest.TestCase):
|
|||||||
json.loads((ROOT / "data/public/policy.json").read_text(encoding="utf-8"))["id"])
|
json.loads((ROOT / "data/public/policy.json").read_text(encoding="utf-8"))["id"])
|
||||||
|
|
||||||
|
|
||||||
|
class Forge(unittest.TestCase):
|
||||||
|
GITEA = {"kind": "gitea", "url": "https://git.example.net/", "repo": "me/threshing-atlas", "branch": "main"}
|
||||||
|
GITHUB = {"kind": "github", "url": "https://github.com", "repo": "me/threshing-atlas", "branch": "main"}
|
||||||
|
|
||||||
|
def test_links(self):
|
||||||
|
from collection.etl import forge
|
||||||
|
self.assertEqual(forge.file_url("data/x.csv", self.GITEA), "https://git.example.net/me/threshing-atlas/raw/branch/main/data/x.csv")
|
||||||
|
self.assertEqual(forge.file_url("data/x.csv", self.GITHUB), "https://raw.githubusercontent.com/me/threshing-atlas/main/data/x.csv")
|
||||||
|
self.assertIn("template=.github/ISSUE_TEMPLATE/report-a-run.yml", forge.new_issue_url(".github/ISSUE_TEMPLATE/report-a-run.yml", self.GITEA))
|
||||||
|
self.assertTrue(forge.new_issue_url(".github/ISSUE_TEMPLATE/report-a-run.yml", self.GITHUB).endswith("template=report-a-run.yml"))
|
||||||
|
self.assertEqual(forge.api_base(self.GITEA), "https://git.example.net/api/v1/repos/me/threshing-atlas")
|
||||||
|
|
||||||
|
def test_gitea_calls(self):
|
||||||
|
from collection.etl import forge
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def fake(method, url, body=None, tok=None, f=None):
|
||||||
|
calls.append((method, url, body, tok))
|
||||||
|
if url.endswith("/pulls"):
|
||||||
|
return {"number": 7}
|
||||||
|
if "/issues?" in url:
|
||||||
|
page = int(url.rsplit("page=", 1)[1])
|
||||||
|
return [{"number": 50 * (page - 1) + i, "body": "", "labels": [], "state": "open"} for i in range(50 if page == 1 else 3)]
|
||||||
|
return None
|
||||||
|
old = forge._call
|
||||||
|
forge._call = fake
|
||||||
|
try:
|
||||||
|
issues = forge.list_issues("run report", {"ATLAS_BOT_TOKEN": "t"}, self.GITEA)
|
||||||
|
self.assertEqual(len(issues), 53)
|
||||||
|
self.assertIsNone(issues[0]["state_reason"])
|
||||||
|
self.assertIn("type=issues", calls[0][1])
|
||||||
|
self.assertEqual(forge.open_pr("data/x", "T", "B", True, {"ATLAS_BOT_TOKEN": "t"}, self.GITEA), 7)
|
||||||
|
merge = calls[-1]
|
||||||
|
self.assertTrue(merge[1].endswith("/pulls/7/merge") and merge[2]["merge_when_checks_succeed"])
|
||||||
|
self.assertEqual(merge[3], "t")
|
||||||
|
finally:
|
||||||
|
forge._call = old
|
||||||
|
|
||||||
|
def test_page_links_follow_forge_yaml(self):
|
||||||
|
from collection.etl import codegen, forge
|
||||||
|
links = codegen.links()
|
||||||
|
self.assertEqual(links["repo"], forge.repo_url())
|
||||||
|
self.assertTrue(links["report"].startswith(forge.repo_url()))
|
||||||
|
page = (ROOT / "site/atlas/index.html").read_text(encoding="utf-8")
|
||||||
|
for key in set(re.findall(r"\{\{(\w+)\}\}", page)):
|
||||||
|
self.assertIn(key, links)
|
||||||
|
|
||||||
|
|
||||||
class Sheets(unittest.TestCase):
|
class Sheets(unittest.TestCase):
|
||||||
def test_privacy_rules(self):
|
def test_privacy_rules(self):
|
||||||
p = TMP / "sheet.csv"
|
p = TMP / "sheet.csv"
|
||||||
|
|||||||
Reference in New Issue
Block a user