"""Reddit megathreads (raw: the threads' comment trees from the official API) -> dragonkind.com result links that are not yet in collection/manual/reddit_results.yaml. Usernames and comment text are never kept: only the result ids and the comment permalinks, for the report, so a person can read the result page and add it by hand.""" import json import re from pathlib import Path from ..common import read_yaml RESULT_RE = re.compile(r"dragonkind\.com/results/(THRESH-[A-Z0-9]+)", re.I) def _comments(node): """Every comment in a listing tree.""" if isinstance(node, list): for x in node: yield from _comments(x) elif isinstance(node, dict): data = node.get("data") or {} if node.get("kind") in ("t1", "t3"): yield data for key in ("children", "replies"): if isinstance(data.get(key), (dict, list)): yield from _comments(data[key]) def transform(raw_path, spec, ctx): threads = json.loads(Path(raw_path).read_text(encoding="utf-8")) known = {r["id"].upper() for r in read_yaml(spec["transform"]["manual"])["results"]} found = {} for tid, tree in sorted(threads.items()): for c in _comments(tree): for m in RESULT_RE.finditer((c.get("body") or "") + " " + (c.get("selftext") or "") + " " + (c.get("url") or "")): rid = m.group(1).upper() if rid not in known and rid not in found: found[rid] = "https://www.reddit.com" + (c.get("permalink") or f"/comments/{tid}/") return {"new_results": [{"id": k, "comment": v} for k, v in sorted(found.items())]} def describe(old, new): rs = new["new_results"] if not rs: return ["no new result links in the threads"] return [f"{len(rs)} result link(s) not yet in collection/manual/reddit_results.yaml (read each page, then add it):"] + \ [f" - https://dragonkind.com/results/{r['id']} (posted at {r['comment']})" for r in rs[:40]]