- collection/pipeline.yaml declares every source (extract kind, transform, privacy rules, load target, run source); `python -m collection update|build` - extract keeps data/raw/<source>/current + previous only (git-ignored); data/manifest.json records the current raw checksum and change date - transforms are pure; validate checks runs against data/dictionary.yaml before load; load writes byte-stable files only when they change - data/public/runs_clean.csv: every run from every source with a new `source` column (reddit spreadsheet, reddit thread, reported, bot, reddit pdf); the sheet's own row label moves to `entry`; run_id is text - bot importer ported to Python (same output as bot.js on the fixture); labels and crosswalk in collection/specs/bot.yaml, written into graph.js - Reddit through the official API (result links to add by hand); the 12 result pages move to collection/manual/reddit_results.yaml - "Report a run": a GitHub issue form generated from collection/specs/report_form.yaml, collected as source "reported", plus a site tab pointing to it, the community spreadsheet and the megathread - analysis and the route model read runs_clean.csv (sources in measures.yaml); stats.json and policy.json unchanged apart from the factor's new name - daily workflow runs the pipeline; the raw store is kept between runs in the Actions cache, encrypted with RAW_ARCHIVE_KEY Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
52 lines
2.2 KiB
Python
52 lines
2.2 KiB
Python
"""The route PDF -> its fingerprint only. The PDF is never redistributed: the public file holds hashes of the file,
|
|
of each page's text and of each line, so a change can be detected and located without keeping the text."""
|
|
import hashlib
|
|
|
|
|
|
def _sha(text, n):
|
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:n]
|
|
|
|
|
|
def pdf_lines(path):
|
|
from pypdf import PdfReader
|
|
pages = []
|
|
for page in PdfReader(str(path)).pages:
|
|
text = page.extract_text() or ""
|
|
pages.append([" ".join(line.split()) for line in text.splitlines() if line.strip()])
|
|
return pages
|
|
|
|
|
|
def transform(raw_path, spec, ctx):
|
|
with open(raw_path, "rb") as f:
|
|
data = f.read()
|
|
pages = pdf_lines(raw_path)
|
|
return {
|
|
"sha256": hashlib.sha256(data).hexdigest(),
|
|
"bytes": len(data),
|
|
"pages": len(pages),
|
|
"page_hashes": [_sha("\n".join(p), 16) for p in pages],
|
|
"line_hashes": sorted({_sha(line.lower(), 12) for p in pages for line in p}),
|
|
"checked": ctx.get("changed"),
|
|
# transient: the new lines' text, for the report only (load drops every key starting with "_")
|
|
"_lines": [line for p in pages for line in p],
|
|
}
|
|
|
|
|
|
def describe(old, new):
|
|
if old is None:
|
|
return [f"fingerprint recorded for the first time ({new['pages']} pages)"]
|
|
if old.get("page_hashes") == new["page_hashes"]:
|
|
return ["the file changed but its text did not (layout or images only). Worth a look."]
|
|
changed = [i + 1 for i, h in enumerate(new["page_hashes"])
|
|
if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
|
|
old_lines = set(old["line_hashes"])
|
|
new_lines = [line for line in new.get("_lines", []) if _sha(line.lower(), 12) not in old_lines]
|
|
lines = [f"changed. {old['pages']} → {new['pages']} pages; pages with new text: {', '.join(map(str, changed)) or 'none'}; "
|
|
f"{len(new_lines)} new lines, {len(old_lines - set(new['line_hashes']))} lines gone."]
|
|
if new_lines:
|
|
lines.append(" - New lines (update core/spec/game.yaml or the crosswalks if they add options):")
|
|
lines += [f" - {line}" for line in new_lines[:60]]
|
|
if len(new_lines) > 60:
|
|
lines.append(f" - … and {len(new_lines) - 60} more")
|
|
return lines
|