name: Check sources for new data on: schedule: - cron: "23 10 * * *" # daily workflow_dispatch: permissions: contents: write pull-requests: write issues: write jobs: check: runs-on: ubuntu-latest env: # secrets cannot be tested in `if:` directly HAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN != '' }} steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 20 - uses: actions/setup-python@v5 with: python-version: "3.12" - run: pip install -r requirements.txt # The raw store (data/raw: the current and previous raw file of each source) is never committed. Between runs # it lives in the Actions cache, encrypted with the RAW_ARCHIVE_KEY secret, so only the previous version is # kept and nobody without the key can read it (the register has rider names). Without the secret there is no # archive: each run starts from the manifest alone. - name: Restore the private raw archive uses: actions/cache/restore@v4 with: path: ${{ runner.temp }}/raw-archive key: raw-archive-none restore-keys: raw-archive- - name: Decrypt it env: RAW_ARCHIVE_KEY: ${{ secrets.RAW_ARCHIVE_KEY }} run: | f="$RUNNER_TEMP/raw-archive/raw.tgz.enc" if [ -n "$RAW_ARCHIVE_KEY" ] && [ -f "$f" ]; then openssl enc -d -aes-256-cbc -pbkdf2 -pass env:RAW_ARCHIVE_KEY -in "$f" | tar -xz -C data fi # extract -> transform -> load for every source, then the combined runs (collection/pipeline.yaml) - id: check env: GITHUB_TOKEN: ${{ github.token }} REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }} REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }} run: python -m collection update --report "$RUNNER_TEMP/report.md" - name: Rebuild the analysis and run tests if: steps.check.outputs.changed == 'true' run: | node analysis/network/build-graph.js npm run policy npm run stats python -m collection build npm test - name: Encrypt the raw store id: archive env: RAW_ARCHIVE_KEY: ${{ secrets.RAW_ARCHIVE_KEY }} run: | if [ -n "$RAW_ARCHIVE_KEY" ] && [ -d data/raw ]; then mkdir -p "$RUNNER_TEMP/raw-archive" tar -czf - -C data raw | openssl enc -aes-256-cbc -pbkdf2 -salt -pass env:RAW_ARCHIVE_KEY -out "$RUNNER_TEMP/raw-archive/raw.tgz.enc" echo "save=true" >> "$GITHUB_OUTPUT" fi - name: Save it (a new copy only when a raw file changed) if: steps.archive.outputs.save == 'true' uses: actions/cache/save@v4 with: path: ${{ runner.temp }}/raw-archive key: raw-archive-${{ hashFiles('data/manifest.json') }} # The PR is opened with ATLAS_BOT_TOKEN (a fine-grained token, see docs/REPO_SETTINGS.md) when it is set: # changes made with the built-in GITHUB_TOKEN do not start other workflows, so its merge would not # redeploy the site. Without the secret it falls back to GITHUB_TOKEN and the PR waits for a manual merge. - name: Open a pull request with the new data id: pr if: steps.check.outputs.changed == 'true' uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.ATLAS_BOT_TOKEN || github.token }} branch: data/source-update delete-branch: true commit-message: "data: update from community sources" title: "New data from the community sources" body-path: ${{ runner.temp }}/report.md labels: data - name: Merge it automatically once CI passes if: steps.pr.outputs.pull-request-number != '' && env.HAS_BOT_TOKEN == 'true' env: GH_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }} run: gh pr merge "${{ steps.pr.outputs.pull-request-number }}" --auto --squash - name: Report a broken source link if: steps.check.outputs.failed != '' env: GH_TOKEN: ${{ github.token }} run: | title="Source check failed: ${{ steps.check.outputs.failed }}" if ! gh issue list --state open --search "$title in:title" --json number --jq '.[0].number' | grep -q .; then gh issue create --title "$title" --body-file "$RUNNER_TEMP/report.md" fi