pyrtan / sourcev0.14.0 · main

pyrtan / site / snapshot.py

10,692 bytes · last changed 2026-10-27 in No seeds: pyrtan.com is GitHub Pages and a hub list, nothing more

1"""Bake a snapshot of the directory into the site.
2
3GitHub Actions runs this on every build and on a schedule: it asks each
4hub in the hub list (``rendezvous/hubs.json``, which operators add to by
5pull request) for its ``/directory``, then every other hub it knows of
6(from the last snapshot and from what the hubs it asked said), so the
7directory stays fresh while any hub it knows of is up. pyrtan.com runs no
8hub of its own; this is the whole of its part. It
9merges what they say with the snapshot already published (so a build when
10nobody answers keeps the last good one rather than publishing an empty
11list), drops entries nobody has heard of for a week, and writes
12``rendezvous/directory.json``. It keeps only the default network's part
13of each directory (``--network``, the ``network.json`` the site
14publishes) and what was heard outside any network; a private network's
15hubs never reach the snapshot. The landing page
16shows the live directory when a listed hub answers the visitor's browser
17and this snapshot, marked with its time, when none does; ``pyrtan join``
18with no invite tries the hubs in it after the listed ones.
19
20Standard library only; no pyrtan import, so it runs before anything is
21installed.
22
23 python3 site/snapshot.py HUBS OUT [--previous URL_OR_FILE ...]
24 [--network URL_OR_FILE] [--max N]
25 [--reports OUT --previous-reports URL_OR_FILE ...]
26
27With ``--reports`` it also asks each hub that answered for ``/reports``, the gap
28reports hubs that opted in (``--report-gaps``) gossip for catalog trees,
29merges them the same way, and writes ``rendezvous/reports.json``; the
30catalog page shows them beside each tree.
31"""
32
33from __future__ import annotations
34
35import argparse
36import datetime as _dt
37import hashlib
38import json
39import sys
40import time
41from concurrent.futures import ThreadPoolExecutor
42import urllib.request
43from pathlib import Path
44from typing import Any
45
46TTL = 7 * 86400
47REPORT_TTL = 30 * 86400
48TIMEOUT = 8.0
49MAX_ASK = 200 # hubs asked per run, listed ones included
50PARALLEL = 16
51
52
53def fetch(source: str) -> Any:
54 if source.startswith(("http://", "https://")):
55 req = urllib.request.Request(source, headers={"Accept": "application/json", "User-Agent": "pyrtan.com snapshot"})
56 with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
57 return json.loads(resp.read().decode("utf-8"))
58 return json.loads(Path(source).read_text(encoding="utf-8"))
59
60
61def network_id(name: str, secret: str) -> str:
62 """As pyrtan.federation.network_id; repeated so this runs before pyrtan is installed."""
63 return hashlib.sha256(f"pyrtan-network\n{name}\n{secret}".encode("utf-8")).hexdigest()[:16]
64
65
66def merge(into: dict[str, dict[str, Any]], entries: Any, via: str, now: float, network: str | None = None) -> int:
67 """Take a directory's rows; with ``network`` (an id), only those heard
68 in it or outside any network."""
69 taken = 0
70 for e in entries if isinstance(entries, list) else []:
71 if not isinstance(e, dict) or not isinstance(e.get("id"), str) or not e["id"].startswith(("http://", "https://")):
72 continue
73 nets = e.get("networks")
74 if network is not None and isinstance(nets, list) and nets and network not in nets:
75 continue
76 try:
77 seen = min(float(e.get("seen", 0)), now)
78 except (TypeError, ValueError):
79 continue
80 if now - seen >= TTL:
81 continue
82 hub_id = e["id"].rstrip("/")
83 current = into.get(hub_id)
84 if current is not None and current["seen"] >= seen:
85 continue
86 into[hub_id] = {"id": hub_id, "name": str(e.get("name") or "")[:80], "trees": [str(t)[:80] for t in e.get("trees", []) if isinstance(t, (str, int))][:200], "seen": seen, "via": via}
87 taken += 1
88 return taken
89
90
91def merge_reports(into: dict[str, dict[str, Any]], rows: Any, now: float) -> int:
92 """One row per gap: the hub's view already sums reporters, so a newer
93 or larger row for the same gap replaces an older one."""
94 taken = 0
95 for r in rows if isinstance(rows, list) else []:
96 if not isinstance(r, dict) or not isinstance(r.get("id"), str) or "@" not in r["id"] or not isinstance(r.get("node"), str):
97 continue
98 try:
99 last = min(float(r.get("last", 0)), now)
100 count = max(1, int(r.get("count", 1)))
101 reporters = max(1, int(r.get("reporters", 1)))
102 except (TypeError, ValueError):
103 continue
104 if now - last >= REPORT_TTL:
105 continue
106 key = json.dumps([r["id"], r["node"], r.get("on"), r.get("observed")], sort_keys=True, ensure_ascii=False, default=str)
107 current = into.get(key)
108 if current is not None and (current["last"], current["count"]) >= (last, count):
109 continue
110 observed = r.get("observed")
111 into[key] = {"id": r["id"][:120], "node": r["node"][:80], "on": str(r.get("on", ""))[:120], "observed": observed if isinstance(observed, (str, int, float, bool)) or observed is None else str(observed)[:80], "count": count, "reporters": reporters, "last": last}
112 taken += 1
113 return taken
114
115
116def main(argv: list[str] | None = None) -> int:
117 ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
118 ap.add_argument("hub_list", help="hubs.json, a path or URL")
119 ap.add_argument("out", help="where to write directory.json")
120 ap.add_argument("--previous", action="append", default=[], help="an earlier directory.json to keep entries from (URL or path; repeatable, first that answers wins)")
121 ap.add_argument("--network", default=None, help="the default network's network.json (URL or path): keep only its part of each directory")
122 ap.add_argument("--max", type=int, default=MAX_ASK, help=f"hubs to ask at most, listed ones included (default {MAX_ASK})")
123 ap.add_argument("--reports", default=None, help="also write the gap reports the hubs carry to this reports.json")
124 ap.add_argument("--previous-reports", action="append", default=[], help="an earlier reports.json to keep rows from (URL or path; repeatable, first that answers wins)")
125 args = ap.parse_args(argv)
126 now = time.time()
127 hubs: dict[str, dict[str, Any]] = {}
128 report: dict[str, Any] = {"answered": [], "silent": [], "previous": None}
129 reports: dict[str, dict[str, Any]] = {}
130 if args.reports:
131 for source in args.previous_reports:
132 try:
133 prev = fetch(source)
134 except Exception:
135 continue
136 if isinstance(prev, dict) and isinstance(prev.get("reports"), list):
137 merge_reports(reports, prev["reports"], now)
138 break
139
140 for source in args.previous:
141 try:
142 prev = fetch(source)
143 except Exception:
144 continue
145 if isinstance(prev, dict) and isinstance(prev.get("hubs"), list):
146 merge(hubs, prev["hubs"], "snapshot", now)
147 report["previous"] = source
148 break
149
150 network: str | None = None
151 if args.network:
152 try:
153 n = fetch(args.network)
154 network = network_id(n["name"], n["secret"])
155 except Exception as e:
156 print(f"snapshot: cannot read the default network from {args.network}: {e}; keeping every network's part", file=sys.stderr)
157 try:
158 listed = [h.rstrip("/") for h in fetch(args.hub_list).get("hubs", []) if isinstance(h, str)]
159 except Exception as e:
160 print(f"snapshot: cannot read the hub list from {args.hub_list}: {e}", file=sys.stderr)
161 listed = []
162 report["crawled"] = {"answered": [], "silent": []}
163
164 def ask(hub: str) -> tuple[str, Any, Any, str | None]:
165 try:
166 d = fetch(f"{hub}/directory")
167 except Exception as e:
168 return hub, None, None, str(e)[:120]
169 r = None
170 if args.reports and isinstance(d, dict):
171 try:
172 r = fetch(f"{hub}/reports")
173 except Exception:
174 pass
175 return hub, d, r, None
176
177 # The listed hubs first, then every hub the last snapshot heard of, most
178 # recent first, then whoever the hubs asked name: rounds of parallel
179 # asks until nobody new is named or --max is reached.
180 asked: set[str] = set()
181 pending = list(dict.fromkeys(listed + [h["id"] for h in sorted(hubs.values(), key=lambda h: -h["seen"])]))
182 while pending and len(asked) < args.max:
183 batch = [h for h in pending if h not in asked][: args.max - len(asked)]
184 pending = []
185 asked.update(batch)
186 with ThreadPoolExecutor(max_workers=PARALLEL) as pool:
187 results = list(pool.map(ask, batch))
188 for hub, d, r, error in results:
189 book = report if hub in listed else report["crawled"]
190 if error is None and isinstance(d, dict) and isinstance(d.get("hubs"), list):
191 merge(hubs, d["hubs"], hub, now, network)
192 book["answered"].append(hub)
193 pending.extend(e["id"].rstrip("/") for e in d["hubs"] if isinstance(e, dict) and isinstance(e.get("id"), str) and e["id"].startswith(("http://", "https://")) and e["id"].rstrip("/") not in asked)
194 if isinstance(r, dict):
195 merge_reports(reports, r.get("reports"), now)
196 else:
197 book["silent"].append({"hub": hub, "error": error or "not a directory"})
198 pending = list(dict.fromkeys(pending))
199
200 out = {
201 "pyrtan-directory": 1,
202 "as_of": _dt.datetime.fromtimestamp(now, _dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
203 "listed": report,
204 "hubs": sorted(hubs.values(), key=lambda h: (-h["seen"], h["id"])),
205 }
206 path = Path(args.out)
207 path.parent.mkdir(parents=True, exist_ok=True)
208 path.write_text(json.dumps(out, indent=2) + "\n", encoding="utf-8")
209 print(f"snapshot: {len(out['hubs'])} hub(s); {len(report['answered'])} listed hub(s) answered, {len(report['silent'])} silent; {len(report['crawled']['answered'])} other hub(s) answered, {len(report['crawled']['silent'])} silent" + (f"; kept entries from {report['previous']}" if report["previous"] else ""))
210 if args.reports:
211 rows = sorted(reports.values(), key=lambda r: (-r["count"], -r["last"], r["id"]))
212 rpath = Path(args.reports)
213 rpath.parent.mkdir(parents=True, exist_ok=True)
214 rpath.write_text(json.dumps({"pyrtan-reports": 1, "as_of": out["as_of"], "count": len(rows), "reports": rows}, indent=2) + "\n", encoding="utf-8")
215 print(f"snapshot: {len(rows)} gap report(s) -> {rpath}")
216 return 0
217
218
219if __name__ == "__main__":
220 sys.exit(main())