Files
threshing-atlas/.gitea/workflows/update-sources.yml
T
kathleenandClaude Opus 5.5 bd89a8941b
CI / test (pull_request) Successful in 1m24s
Daily update: one branch per run, so a leftover branch never blocks the push
Run #24 (2026-11-17) failed at the push: data/source-update-20261117 already
existed from an earlier run that day.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-17 23:37:14 +00:00

91 lines
4.3 KiB
YAML

# Every day: check every source (collection/pipeline.yaml), and when something changed, rebuild the runs, the
# graph, the route model and the statistics, run the tests and open a pull request that merges itself once CI
# passes. A source that cannot be reached, or new answers the coding rules do not know, open an issue instead.
# Gitea Actions; GitHub's copy (with GitHub Pages) is .github/workflows/update-sources.yml.
#
# Secrets (repository Settings -> Actions -> Secrets; docs/REPO_SETTINGS.md):
# ATLAS_BOT_TOKEN a Gitea access token (repository: read and write; issue: read and write). Needed to push the
# update branch and open the pull request; the built-in token cannot start the CI it needs.
# RAW_ARCHIVE_KEY optional: encrypts the private raw store kept in the Actions cache between runs
# REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET optional: read the megathreads through the Reddit API
name: Check sources for new data
on:
schedule:
- cron: "23 10 * * *" # daily
workflow_dispatch:
jobs:
check:
runs-on: ubuntu-latest # a runner label on the server (act_runner)
steps:
- uses: actions/checkout@v4
with:
token: ${{ secrets.ATLAS_BOT_TOKEN }}
- uses: actions/setup-node@v4
with:
node-version: 20
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- run: pip install -r requirements.txt
- name: Restore the private raw archive
uses: actions/cache/restore@v4
with:
path: ${{ runner.temp }}/raw-archive
key: raw-archive-none
restore-keys: raw-archive-
- name: Decrypt it
env:
RAW_ARCHIVE_KEY: ${{ secrets.RAW_ARCHIVE_KEY }}
run: |
f="$RUNNER_TEMP/raw-archive/raw.tgz.enc"
if [ -n "$RAW_ARCHIVE_KEY" ] && [ -f "$f" ]; then
openssl enc -d -aes-256-cbc -pbkdf2 -pass env:RAW_ARCHIVE_KEY -in "$f" | tar -xz -C data
fi
- id: check
env:
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }}
REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }}
run: python -m collection update --report "$RUNNER_TEMP/report.md"
- name: Rebuild the analysis and run tests
if: steps.check.outputs.changed == 'true'
run: |
python -m collection rebuild
npm test
- name: Encrypt the raw store
id: archive
env:
RAW_ARCHIVE_KEY: ${{ secrets.RAW_ARCHIVE_KEY }}
run: |
if [ -n "$RAW_ARCHIVE_KEY" ] && [ -d data/raw ]; then
mkdir -p "$RUNNER_TEMP/raw-archive"
tar -czf - -C data raw | openssl enc -aes-256-cbc -pbkdf2 -salt -pass env:RAW_ARCHIVE_KEY -out "$RUNNER_TEMP/raw-archive/raw.tgz.enc"
echo "save=true" >> "$GITHUB_OUTPUT"
fi
- name: Save it (a new copy only when a raw file changed)
if: steps.archive.outputs.save == 'true'
uses: actions/cache/save@v4
with:
path: ${{ runner.temp }}/raw-archive
key: raw-archive-${{ hashFiles('data/manifest.json') }}
- name: Open a pull request with the new data (it merges itself once CI passes)
if: steps.check.outputs.changed == 'true'
env:
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
run: |
# one branch per run: a second run on the same day (or a leftover branch) never blocks the push
branch="data/source-update-$(date -u +%Y%m%d)-run${{ github.run_number }}"
git config user.name "Atlas data bot"
git config user.email "atlas-bot@noreply.localhost"
git checkout -b "$branch"
git add -A data analysis/reports core/graph.js .github/ISSUE_TEMPLATE
git commit -m "data: update from community sources"
git push origin "$branch"
python -m collection forge open-pr --head "$branch" --title "New data from the community sources" --body-file "$RUNNER_TEMP/report.md" --automerge
- name: Report a broken source or new answers
if: steps.check.outputs.failed != ''
env:
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
run: |
python -m collection forge open-issue --title "Source check failed: ${{ steps.check.outputs.failed }}" --body-file "$RUNNER_TEMP/report.md"