# Every day: check every source (collection/pipeline.yaml), and when something changed, rebuild the runs, the # graph, the route model and the statistics, run the tests and open a pull request that merges itself once CI # passes. A source that cannot be reached, or new answers the coding rules do not know, open an issue instead. # Gitea Actions; GitHub's copy (with GitHub Pages) is .github/workflows/update-sources.yml. # # Secrets (repository Settings -> Actions -> Secrets; docs/REPO_SETTINGS.md): # ATLAS_BOT_TOKEN a Gitea access token (repository: read and write; issue: read and write). Needed to push the # update branch and open the pull request; the built-in token cannot start the CI it needs. # RAW_ARCHIVE_KEY optional: encrypts the private raw store kept in the Actions cache between runs # REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET optional: read the megathreads through the Reddit API # DK_ACCOUNT_SALT the salt that turns bot account names into anonymous keys (keep it the same forever). With it, # the bot's private repo (DragonKind/DragonKind) is downloaded and imported; without it, skipped. # DRAGONKIND_TOKEN optional: a read-only token for the bot's repo (ATLAS_BOT_TOKEN is used otherwise) name: Check sources for new data on: schedule: - cron: "23 10 * * *" # daily workflow_dispatch: # one check at a time: a second dispatch waits instead of building and pushing alongside the first concurrency: group: update-sources cancel-in-progress: false jobs: check: runs-on: ubuntu-latest # a runner label on the server (act_runner) steps: - uses: actions/checkout@v4 with: token: ${{ secrets.ATLAS_BOT_TOKEN }} - uses: actions/setup-node@v4 with: node-version: 20 - uses: actions/setup-python@v5 with: python-version: "3.12" - run: pip install -r requirements.txt - name: Restore the private raw archive uses: actions/cache/restore@v4 with: path: ${{ runner.temp }}/raw-archive key: raw-archive-none restore-keys: raw-archive- - name: Decrypt it env: RAW_ARCHIVE_KEY: ${{ secrets.RAW_ARCHIVE_KEY }} run: | f="$RUNNER_TEMP/raw-archive/raw.tgz.enc" if [ -n "$RAW_ARCHIVE_KEY" ] && [ -f "$f" ]; then openssl enc -d -aes-256-cbc -pbkdf2 -pass env:RAW_ARCHIVE_KEY -in "$f" | tar -xz -C data fi - name: Download the bot's data (DragonKind/DragonKind, private) env: DK_TOKEN: ${{ secrets.DRAGONKIND_TOKEN || secrets.ATLAS_BOT_TOKEN }} DK_ACCOUNT_SALT: ${{ secrets.DK_ACCOUNT_SALT }} run: | if [ -n "$DK_TOKEN" ] && [ -n "$DK_ACCOUNT_SALT" ]; then GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0="http.https://git.kenschiano.net/.extraheader" \ GIT_CONFIG_VALUE_0="AUTHORIZATION: token $DK_TOKEN" \ git clone --quiet --depth 1 https://git.kenschiano.net/DragonKind/DragonKind.git "$RUNNER_TEMP/dragonkind" echo "DRAGONKIND_BOT_DIR=$RUNNER_TEMP/dragonkind" >> "$GITHUB_ENV" else echo "DK_ACCOUNT_SALT is not set: the bot source is skipped" fi - id: check env: ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }} DK_ACCOUNT_SALT: ${{ secrets.DK_ACCOUNT_SALT }} REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }} REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }} run: python -m collection update --report "$RUNNER_TEMP/report.md" - name: Rebuild the analysis and run tests # not when the combined runs failed validation: the issue below reports the new answers instead if: steps.check.outputs.changed == 'true' && !contains(steps.check.outputs.failed, 'build') run: | python -m collection rebuild npm test - name: Encrypt the raw store id: archive env: RAW_ARCHIVE_KEY: ${{ secrets.RAW_ARCHIVE_KEY }} run: | if [ -n "$RAW_ARCHIVE_KEY" ] && [ -d data/raw ]; then mkdir -p "$RUNNER_TEMP/raw-archive" tar -czf - -C data raw | openssl enc -aes-256-cbc -pbkdf2 -salt -pass env:RAW_ARCHIVE_KEY -out "$RUNNER_TEMP/raw-archive/raw.tgz.enc" echo "save=true" >> "$GITHUB_OUTPUT" fi - name: Save it (a new copy only when a raw file changed) if: steps.archive.outputs.save == 'true' uses: actions/cache/save@v4 with: path: ${{ runner.temp }}/raw-archive key: raw-archive-${{ hashFiles('data/manifest.json') }} - name: Send a changed route PDF to the bot (its private repo; the PDF is never published here) if: always() && steps.check.outcome == 'success' continue-on-error: true env: ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }} run: python -m collection send-pdf - name: Open a pull request with the new data (it merges itself once CI passes) if: steps.check.outputs.changed == 'true' && !contains(steps.check.outputs.failed, 'build') env: ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }} run: | # one branch per run: a second run on the same day (or a leftover branch) never blocks the push branch="data/source-update-$(date -u +%Y%m%d)-run${{ github.run_number }}" git config user.name "Atlas data bot" git config user.email "atlas-bot@noreply.localhost" git checkout -b "$branch" git add -A data analysis/reports core/graph.js .github/ISSUE_TEMPLATE git commit -m "data: update from community sources" git push origin "$branch" python -m collection forge open-pr --head "$branch" --title "New data from the community sources" --body-file "$RUNNER_TEMP/report.md" --automerge - name: Report a broken source or new answers if: always() && steps.check.outputs.failed != '' env: ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }} run: | python -m collection forge open-issue --title "Source check failed: ${{ steps.check.outputs.failed }}" --body-file "$RUNNER_TEMP/report.md"