"""Paths, the pipeline spec, and byte-stable writers shared by every stage.""" import hashlib import json import sys from pathlib import Path import yaml ROOT = Path(__file__).resolve().parents[2] if str(ROOT / "core") not in sys.path: sys.path.insert(0, str(ROOT / "core")) def path(p): return ROOT / p def read_yaml(p): with open(path(p), encoding="utf-8") as f: return yaml.safe_load(f) def pipeline(): return read_yaml("collection/pipeline.yaml") def sha256_bytes(data): return hashlib.sha256(data).hexdigest() def sha256_file(p): h = hashlib.sha256() with open(p, "rb") as f: for block in iter(lambda: f.read(1 << 20), b""): h.update(block) return h.hexdigest() def json_bytes(obj, compact=True): if compact: return (json.dumps(obj, ensure_ascii=False, separators=(",", ":"))).encode("utf-8") return (json.dumps(obj, ensure_ascii=False, indent=1) + "\n").encode("utf-8") def jsonl_bytes(rows): return "".join(json.dumps(r, ensure_ascii=False, separators=(",", ":")) + "\n" for r in rows).encode("utf-8") def read_jsonl(p): p = Path(p) if not p.exists(): return [] out = [] for line in p.read_text(encoding="utf-8").splitlines(): if line.strip(): try: out.append(json.loads(line)) except json.JSONDecodeError: pass return out class Report: """What a pipeline run did, as Markdown lines (the daily pull request's body).""" def __init__(self): self.lines = [] self.changed = False self.failed = [] def add(self, line): self.lines.append(line) print(line) def text(self, title): return "\n".join([f"## {title}", ""] + self.lines) + "\n"