- collection/pipeline.yaml declares every source (extract kind, transform, privacy rules, load target, run source); `python -m collection update|build` - extract keeps data/raw/<source>/current + previous only (git-ignored); data/manifest.json records the current raw checksum and change date - transforms are pure; validate checks runs against data/dictionary.yaml before load; load writes byte-stable files only when they change - data/public/runs_clean.csv: every run from every source with a new `source` column (reddit spreadsheet, reddit thread, reported, bot, reddit pdf); the sheet's own row label moves to `entry`; run_id is text - bot importer ported to Python (same output as bot.js on the fixture); labels and crosswalk in collection/specs/bot.yaml, written into graph.js - Reddit through the official API (result links to add by hand); the 12 result pages move to collection/manual/reddit_results.yaml - "Report a run": a GitHub issue form generated from collection/specs/report_form.yaml, collected as source "reported", plus a site tab pointing to it, the community spreadsheet and the megathread - analysis and the route model read runs_clean.csv (sources in measures.yaml); stats.json and policy.json unchanged apart from the factor's new name - daily workflow runs the pipeline; the raw store is kept between runs in the Actions cache, encrypted with RAW_ARCHIVE_KEY Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
133 lines
5.6 KiB
Python
133 lines
5.6 KiB
Python
"""Extract: fetch each source as it is now into the raw store. No transformation happens here.
|
|
|
|
The raw store is data/raw/<source>/ (git-ignored, never published):
|
|
current.<ext> the latest version
|
|
previous.<ext> the version before it (kept only in case it is needed; the daily workflow keeps it in an
|
|
encrypted, private cache). Nothing older is kept and old versions are not hashed.
|
|
data/manifest.json records, for each source, the sha256 of the current raw file and the date it last changed.
|
|
|
|
Each extractor returns the raw bytes, or raises Skip when the source cannot be reached from here (a missing
|
|
secret, the bot's private repo not on this machine).
|
|
"""
|
|
import base64
|
|
import json
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
import urllib.parse
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from .common import path, sha256_bytes
|
|
|
|
USER_AGENT = "threshing-atlas-collector/1.0 (+https://github.com/ekcSchiano/threshing-atlas)"
|
|
RAW_DIR = path("data/raw")
|
|
|
|
|
|
class Skip(Exception):
|
|
"""The source cannot be extracted here; not an error."""
|
|
|
|
|
|
def _get(url, headers=None, data=None, timeout=60):
|
|
req = urllib.request.Request(url, data=data, headers={"User-Agent": USER_AGENT, **(headers or {})})
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return r.read()
|
|
|
|
|
|
# ---------------------------------------------------------------- extractors, by `kind`
|
|
|
|
def http(spec, env):
|
|
data = _get(spec["url"])
|
|
head = data[:15].lower()
|
|
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
|
|
raise RuntimeError(f"{spec['url']} returned a web page, not a file (is the link still shared publicly?)")
|
|
return data
|
|
|
|
|
|
def reddit(spec, env):
|
|
"""The threads' comments through the official Reddit API (app-only OAuth: a "script" app's id and secret)."""
|
|
cid, secret = (env.get(k) for k in spec["secrets"])
|
|
if not (cid and secret):
|
|
raise Skip(f"set the {' and '.join(spec['secrets'])} secrets to read the threads through the Reddit API")
|
|
auth = base64.b64encode(f"{cid}:{secret}".encode()).decode()
|
|
token = json.loads(_get("https://www.reddit.com/api/v1/access_token",
|
|
headers={"Authorization": f"Basic {auth}"},
|
|
data=b"grant_type=client_credentials"))["access_token"]
|
|
threads = {}
|
|
for tid in spec["threads"]:
|
|
threads[tid] = json.loads(_get(f"https://oauth.reddit.com/comments/{tid}?limit=500&depth=10&raw_json=1",
|
|
headers={"Authorization": f"Bearer {token}"}))
|
|
return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
|
|
|
|
|
def github_issues(spec, env):
|
|
"""Every issue with the label (open and closed), through the GitHub REST API."""
|
|
headers = {"Accept": "application/vnd.github+json"}
|
|
token = env.get("GITHUB_TOKEN") or env.get("GH_TOKEN")
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
label = urllib.parse.quote(spec["label"])
|
|
issues, page = [], 1
|
|
while True:
|
|
url = f"https://api.github.com/repos/{spec['repo']}/issues?labels={label}&state=all&per_page=100&page={page}"
|
|
batch = json.loads(_get(url, headers=headers))
|
|
issues += [i for i in batch if "pull_request" not in i]
|
|
if len(batch) < 100:
|
|
break
|
|
page += 1
|
|
issues.sort(key=lambda i: i["number"])
|
|
return json.dumps(issues, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
|
|
|
|
|
def local_dir(spec, env):
|
|
"""The bot's files, from a local copy of its private repo (DRAGONKIND_BOT_DIR), packed as one JSON."""
|
|
root = env.get(spec["env"])
|
|
if not root:
|
|
raise Skip(f"the bot's repo is private: set {spec['env']} to a local copy to import it")
|
|
files = {}
|
|
for rel in spec["files"]:
|
|
p = os.path.join(root, rel)
|
|
files[rel] = Path(p).read_text(encoding="utf-8") if os.path.exists(p) else None
|
|
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
|
|
|
|
|
EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir}
|
|
|
|
|
|
# ---------------------------------------------------------------- the raw store
|
|
|
|
def raw_paths(name, ext):
|
|
d = RAW_DIR / name
|
|
return d / f"current.{ext}", d / f"previous.{ext}"
|
|
|
|
|
|
def store(name, ext, data, manifest, today=None):
|
|
"""Keep `data` as the current raw file. Returns True when it differs from the last one seen."""
|
|
current, previous = raw_paths(name, ext)
|
|
digest = sha256_bytes(data)
|
|
entry = manifest.get(name) or {}
|
|
changed = entry.get("sha256") != digest
|
|
current.parent.mkdir(parents=True, exist_ok=True)
|
|
on_disk = sha256_bytes(current.read_bytes()) if current.exists() else None
|
|
if on_disk and on_disk != digest:
|
|
shutil.copyfile(current, previous) # keep one previous version; anything older is gone
|
|
if on_disk != digest:
|
|
fd, tmp = tempfile.mkstemp(dir=current.parent)
|
|
with os.fdopen(fd, "wb") as f:
|
|
f.write(data)
|
|
os.replace(tmp, current)
|
|
if changed:
|
|
manifest[name] = {"sha256": digest, "bytes": len(data),
|
|
"changed": today or datetime.now(timezone.utc).strftime("%Y-%m-%d")}
|
|
return changed
|
|
|
|
|
|
def extract(name, spec, manifest, env=None, local_file=None):
|
|
"""Fetch one source (or take a local file) into the raw store. Returns (raw path, changed)."""
|
|
env = os.environ if env is None else env
|
|
ext = spec["extract"].get("ext") or "json"
|
|
data = Path(local_file).read_bytes() if local_file else EXTRACTORS[spec["extract"]["kind"]](spec["extract"], env)
|
|
changed = store(name, ext, data, manifest)
|
|
return raw_paths(name, ext)[0], changed
|