pyrtan / sourcev0.19.0 · main

pyrtan / src / pyrtan / catalog.py

22,113 bytes · last changed 2026-10-27 in Make the console a hub of its own until it joins one

1"""The catalog: a directory of trees people can pull by name.
2
3A catalog is static files behind any web server (or a directory on disk):
4an ``index.json`` and the tree files it points to. There is no account, no
5API and no server-side code; a mirror is a copy of the files, and a
6private catalog is the same files on a private host. The one at
7``https://pyrtan.com/catalog/`` is built from a public repository whose CI
8runs ``pyrtan check`` on every tree and this module's :func:`build_index`
9on every merge.
10
11``index.json``::
12
13 {"pyrtan-catalog": 1, "generated": "2026-10-25T12:00:00Z",
14 "trees": [{"id": "pyrtan/weather", "namespace": "pyrtan", "name": "weather",
15 "version": 3, "description": "...", "tags": [...], "hosts": [...],
16 "sources": ["http"], "triggers": [...], "license": "MIT",
17 "path": "pyrtan/weather.toml", "sha256": "...", "size": 1234}]}
18
19``path`` is relative to the index. Beside it, ``trees.json`` carries every
20tree the index lists as JSON (the TOML's own table), for a runtime that
21reads JSON and not TOML: the console in a browser, which runs the
22catalog's trees itself until it joins a hub::
23
24 {"pyrtan-trees": 1, "generated": "...", "count": 1,
25 "trees": [{"id": "pyrtan/weather", "sha256": "...", "tree": {...}}]}
26
27``sha256`` is the TOML file's, as in the index. A tree's ``id`` is ``namespace/name``:
28the namespace is who published it (the tree's own ``namespace`` line), so
29two people can each have a ``weather``; ``pyrtan pull weather`` takes the
30one there is and asks when there are two, ``pyrtan pull barry/weather``
31says which. A pull fetches the file, checks its
32``sha256`` against the index, loads it as a tree, checks that the name and
33version inside match the entry, and only then writes it. A catalog cannot
34hand out a file other than the one its index describes, and a tree a
35service loads is still checked like any other: the catalog is a place to
36find trees, not a channel that bypasses the checker.
37
38Not confused with :class:`pyrtan.registry.Registry`, which is the set of
39trees one running service holds. A catalog is where a registry's files
40may have come from.
41"""
42
43from __future__ import annotations
44
45import datetime as _dt
46import hashlib
47import json
48import os
49import re
50import urllib.error
51import urllib.parse
52import urllib.request
53from dataclasses import dataclass
54from pathlib import Path
55from typing import Any
56
57from .tree import Tree, TreeError, check, load_tree, load_tree_text
58
59DEFAULT_CATALOG = "https://pyrtan.com/catalog/"
60INDEX = "index.json"
61FORMAT = 1
62SPEC = re.compile(r"^(?:([a-z0-9][a-z0-9-]*)/)?([a-z0-9][a-z0-9-]*)(?:@(\d+))?$")
63
64
65class CatalogError(RuntimeError):
66 pass
67
68
69def catalog_url() -> str:
70 """The catalog to use when none is given: ``$PYRTAN_CATALOG`` or the public one."""
71 return os.environ.get("PYRTAN_CATALOG") or DEFAULT_CATALOG
72
73
74# ---------------------------------------------------------------- building
75
76
77def sha256(data: bytes) -> str:
78 return hashlib.sha256(data).hexdigest()
79
80
81def entry_for(tree: Tree, data: bytes, path: str) -> dict[str, Any]:
82 """One index entry: the tree's capability plus where its file is and what it hashes to."""
83 cap = tree.capability()
84 return {
85 "id": tree.id,
86 "namespace": tree.namespace,
87 "name": tree.name,
88 "version": tree.version,
89 "description": tree.description,
90 "tags": tree.tags,
91 "hosts": tree.hosts,
92 **{k: cap[k] for k in ("paths", "commands", "devices", "topics") if k in cap},
93 "sources": cap["sources"],
94 "triggers": cap["triggers"],
95 "inputs": cap["inputs"],
96 "license": tree.license,
97 "path": path,
98 "sha256": sha256(data),
99 "size": len(data),
100 }
101
102
103@dataclass
104class Problem:
105 path: str
106 message: str
107
108 def __str__(self) -> str:
109 return f"{self.path}: {self.message}"
110
111
112def build_index(root: str | Path | list[str | Path], *, require_license: bool = True, copy_to: str | Path | None = None, require_namespace: bool = False) -> tuple[dict[str, Any], list[Problem]]:
113 """Walk ``root`` (one directory, or several) for tree files and describe
114 each in an index.
115
116 With one root and no ``copy_to``, paths are relative to that root and
117 the index belongs beside it. With ``copy_to``, every tree file is copied
118 there as ``<namespace>/<name>.toml`` (``_/<name>.toml`` without a
119 namespace) and the index describes that layout: the shape the public
120 catalog is served in, from any number of source directories.
121
122 Returns the index and the problems that kept a file out of it: a file
123 that does not load, two files with the same id, (when required) a tree
124 with no ``license`` or no ``namespace``. The caller decides whether
125 problems fail the build; the catalog's CI does.
126 """
127 roots = [Path(r) for r in (root if isinstance(root, list) else [root])]
128 if len(roots) > 1 and copy_to is None:
129 raise ValueError("an index over several directories needs copy_to: its paths must be relative to one place")
130 trees: list[dict[str, Any]] = []
131 problems: list[Problem] = []
132 seen: dict[str, str] = {}
133 for base in roots:
134 for path in sorted(base.rglob("*.toml")):
135 if path.name == "pyproject.toml" or any(part.startswith(".") for part in path.relative_to(base).parts):
136 continue
137 rel = path.relative_to(base).as_posix()
138 where = rel if len(roots) == 1 else f"{base}/{rel}"
139 try:
140 tree = load_tree(path)
141 except TreeError as e:
142 problems.append(Problem(where, str(e)))
143 continue
144 errors = [p for p in check(tree) if p.level == "error"]
145 if errors:
146 problems.append(Problem(where, "; ".join(f"{p.where}: {p.message}" for p in errors)))
147 continue
148 if tree.id in seen:
149 problems.append(Problem(where, f"a tree with id {tree.id!r} is already at {seen[tree.id]}"))
150 continue
151 if require_license and not tree.license:
152 problems.append(Problem(where, "no 'license': a catalog tree must say what others may do with it"))
153 continue
154 if require_namespace and not tree.namespace:
155 problems.append(Problem(where, "no 'namespace': a catalog tree says who publishes it"))
156 continue
157 seen[tree.id] = where
158 data = path.read_bytes()
159 if copy_to is not None:
160 rel = f"{tree.namespace or '_'}/{tree.name}.toml"
161 target = Path(copy_to) / rel
162 target.parent.mkdir(parents=True, exist_ok=True)
163 target.write_bytes(data)
164 trees.append(entry_for(tree, data, rel))
165 index = {
166 "pyrtan-catalog": FORMAT,
167 "generated": _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
168 "count": len(trees),
169 "trees": trees,
170 }
171 return index, problems
172
173
174def build_bundle(index: dict[str, Any], base: str | Path) -> dict[str, Any]:
175 """``trees.json`` for an index whose files are under ``base``: each tree's
176 table as JSON, in the index's order, with the file's ``sha256``."""
177 trees = []
178 for entry in index.get("trees", []):
179 path = Path(base) / entry["path"]
180 trees.append({"id": entry["id"], "sha256": entry["sha256"], "tree": load_tree(path).raw})
181 return {"pyrtan-trees": FORMAT, "generated": index.get("generated"), "count": len(trees), "trees": trees}
182
183
184# ---------------------------------------------------------------- reading
185
186
187def _fetch(url: str, timeout: float = 20.0) -> bytes:
188 parts = urllib.parse.urlsplit(url)
189 if parts.scheme == "file" or not parts.scheme:
190 path = Path(urllib.request.url2pathname(parts.path) if parts.scheme else url)
191 try:
192 return path.read_bytes()
193 except OSError as e:
194 raise CatalogError(f"{url}: {e}") from e
195 if parts.scheme not in ("http", "https"):
196 raise CatalogError(f"{url}: a catalog is reached over https (or a file: path)")
197 req = urllib.request.Request(url, headers={"User-Agent": "pyrtan-catalog/0.9", "Accept": "application/json, application/toml, text/plain, */*"})
198 try:
199 with urllib.request.urlopen(req, timeout=timeout) as resp:
200 return resp.read()
201 except urllib.error.HTTPError as e:
202 raise CatalogError(f"{url}: {e.code}") from e
203 except (urllib.error.URLError, OSError) as e:
204 raise CatalogError(f"{url}: {e}") from e
205
206
207def _join(base: str, rel: str) -> str:
208 base = base if base.endswith("/") else base + "/"
209 if urllib.parse.urlsplit(base).scheme in ("http", "https", "file"):
210 if "://" in rel or rel.startswith("/") or ".." in rel.split("/"):
211 raise CatalogError(f"index points outside the catalog: {rel!r}")
212 return urllib.parse.urljoin(base, rel)
213 return str(Path(base) / rel)
214
215
216class Catalog:
217 """A catalog at a URL (or a directory), read lazily."""
218
219 def __init__(self, base: str | None = None) -> None:
220 self.base = (base or catalog_url()).rstrip("/") + "/"
221 self._index: dict[str, Any] | None = None
222
223 @property
224 def index(self) -> dict[str, Any]:
225 if self._index is None:
226 raw = _fetch(_join(self.base, INDEX))
227 try:
228 index = json.loads(raw.decode("utf-8"))
229 except (json.JSONDecodeError, UnicodeDecodeError) as e:
230 raise CatalogError(f"{self.base}{INDEX}: not JSON: {e}") from e
231 if not isinstance(index, dict) or index.get("pyrtan-catalog") != FORMAT or not isinstance(index.get("trees"), list):
232 raise CatalogError(f"{self.base}{INDEX}: not a pyrtan catalog index (format {FORMAT})")
233 self._index = index
234 return self._index
235
236 def entries(self) -> list[dict[str, Any]]:
237 out = []
238 for e in self.index["trees"]:
239 if isinstance(e, dict) and isinstance(e.get("name"), str):
240 e.setdefault("namespace", "")
241 e.setdefault("id", f"{e['namespace']}/{e['name']}" if e["namespace"] else e["name"])
242 out.append(e)
243 return out
244
245 def search(self, query: str = "") -> list[dict[str, Any]]:
246 """Entries whose id, description, tags, hosts, sources or triggers
247 mention every word of ``query``. An empty query lists everything."""
248 words = [w.lower() for w in query.split()]
249 out: list[dict[str, Any]] = []
250 for e in self.entries():
251 hay = " ".join([e["id"], str(e.get("description", "")), " ".join(e.get("tags", [])), " ".join(e.get("hosts", [])), " ".join(e.get("sources", [])), " ".join(e.get("triggers", []))]).lower()
252 if all(w in hay for w in words):
253 out.append(e)
254 return out
255
256 def find(self, spec: str) -> dict[str, Any]:
257 """The entry for ``[namespace/]name[@version]``. A bare name is enough
258 when one namespace has it; with two, the caller is told which."""
259 m = SPEC.match(spec)
260 if not m:
261 raise CatalogError(f"{spec!r}: expected [NAMESPACE/]NAME[@VERSION]")
262 namespace, name, version = m.group(1), m.group(2), m.group(3)
263 have = [e for e in self.entries() if e["name"] == name and (namespace is None or e["namespace"] == namespace)]
264 if not have:
265 raise CatalogError(f"{spec!r} is not in the catalog at {self.base}")
266 if len({e["id"] for e in have}) > 1:
267 raise CatalogError(f"{name!r} is published by more than one namespace: {', '.join(sorted({e['id'] for e in have}))}; say which")
268 for e in have:
269 if version is None or e.get("version") == int(version):
270 return e
271 raise CatalogError(f"{have[0]['id']}: the catalog has version {have[0].get('version')}, not {version}")
272
273 def fetch(self, entry: dict[str, Any]) -> tuple[Tree, bytes]:
274 """The tree file an entry describes, verified against the entry."""
275 rel = entry.get("path")
276 if not isinstance(rel, str) or not rel:
277 raise CatalogError(f"{entry['name']}: the index entry has no path")
278 data = _fetch(_join(self.base, rel))
279 digest = sha256(data)
280 if entry.get("sha256") and digest != entry["sha256"]:
281 raise CatalogError(f"{entry['name']}: the file at {rel} does not match the index (sha256 {digest[:12]}… vs {str(entry['sha256'])[:12]}…); refusing it")
282 try:
283 tree = load_tree_text(data.decode("utf-8"), source=f"{self.base}{rel}")
284 except (TreeError, UnicodeDecodeError) as e:
285 raise CatalogError(f"{entry['name']}: the file does not load as a tree: {e}") from e
286 if tree.name != entry["name"]:
287 raise CatalogError(f"{entry['name']}: the file says it is {tree.name!r}; refusing it")
288 if entry.get("namespace") and tree.namespace != entry["namespace"]:
289 raise CatalogError(f"{entry.get('id', entry['name'])}: the file says its namespace is {tree.namespace!r}; refusing it")
290 if entry.get("version") is not None and tree.version != entry["version"]:
291 raise CatalogError(f"{entry['name']}: the file is version {tree.version}, the index says {entry['version']}; refusing it")
292 return tree, data
293
294 def pull(self, spec: str, into: str | Path, *, force: bool = False) -> Path:
295 """Fetch ``spec`` and write it as ``into/NAME.toml``. Refuses to
296 overwrite a file unless ``force``; a tree someone edited locally is
297 theirs."""
298 entry = self.find(spec)
299 tree, data = self.fetch(entry)
300 into = Path(into)
301 target = into / f"{tree.name}.toml"
302 if target.exists() and not force:
303 if target.read_bytes() == data:
304 return target
305 raise CatalogError(f"{target} exists and differs from the catalog's; pass --force to replace it")
306 into.mkdir(parents=True, exist_ok=True)
307 target.write_bytes(data)
308 return target
309
310
311# ---------------------------------------------------------------- publishing
312
313OWNERS = "OWNERS"
314EXAMPLES_NAMESPACE = "pyrtan"
315
316
317def owners(namespace_dir: str | Path) -> list[str]:
318 """The GitHub logins in ``catalog/<namespace>/OWNERS``: one per line,
319 ``#`` starts a comment."""
320 path = Path(namespace_dir) / OWNERS
321 if not path.is_file():
322 return []
323 out = []
324 for line in path.read_text(encoding="utf-8").splitlines():
325 line = line.split("#", 1)[0].strip()
326 if line:
327 out.append(line.lstrip("@").lower())
328 return out
329
330
331def gate(root: str | Path, *, changed: list[str] | None = None, author: str | None = None, base: dict[str, tuple[int, bytes]] | None = None) -> list[Problem]:
332 """The checks a pull request to the catalog repository must pass. The
333 repository is ``root``: its ``examples/`` are the ``pyrtan`` namespace,
334 its ``catalog/<namespace>/<name>.toml`` files everyone else's, each
335 namespace with an ``OWNERS`` file of GitHub logins.
336
337 Every tree must load clean, carry ``license`` and ``namespace``, sit in
338 the directory its namespace names under the file its name names, and
339 ids must be unique across both places. Given ``changed`` (paths the
340 pull request touches) and ``author`` (its GitHub login), a change under
341 ``catalog/<namespace>/`` must come from one of that namespace's owners.
342 Given ``base`` (``path -> (version, bytes)`` on the base branch), a tree
343 whose file changed must have bumped its version. Returns the problems;
344 none means the gate is open.
345 """
346 import tempfile
347
348 root = Path(root)
349 problems: list[Problem] = []
350 examples = root / "examples"
351 catalog = root / "catalog"
352 roots = [d for d in (examples, catalog) if d.is_dir()]
353 if not roots:
354 return [Problem(str(root), "no examples/ or catalog/ directory: is this a checkout of the pyrtan repository?")]
355 with tempfile.TemporaryDirectory() as tmp:
356 _, problems = build_index(roots, require_license=True, require_namespace=True, copy_to=tmp)
357 if examples.is_dir():
358 for path in sorted(examples.rglob("*.toml")):
359 try:
360 tree = load_tree(path)
361 except TreeError:
362 continue # already reported
363 if tree.namespace != EXAMPLES_NAMESPACE:
364 problems.append(Problem(path.relative_to(root).as_posix(), f"a tree under examples/ is published by {EXAMPLES_NAMESPACE!r}, not {tree.namespace!r}"))
365 if catalog.is_dir():
366 for ns_dir in sorted(catalog.iterdir()):
367 rel = ns_dir.relative_to(root).as_posix()
368 if ns_dir.name.startswith(".") or ns_dir.name == "README.md":
369 continue
370 if not ns_dir.is_dir():
371 problems.append(Problem(rel, "only namespace directories belong directly under catalog/"))
372 continue
373 if not re.match(r"^[a-z0-9][a-z0-9-]*$", ns_dir.name):
374 problems.append(Problem(rel, "a namespace is lowercase letters, digits and dashes"))
375 if not owners(ns_dir):
376 problems.append(Problem(f"{rel}/{OWNERS}", "a namespace needs an OWNERS file with at least one GitHub login"))
377 for path in sorted(ns_dir.rglob("*")):
378 if not path.is_file() or path.name.startswith("."):
379 continue
380 prel = path.relative_to(root).as_posix()
381 if path.name == OWNERS and path.parent == ns_dir:
382 continue
383 if path.suffix != ".toml" or path.parent != ns_dir:
384 problems.append(Problem(prel, "only <name>.toml files and OWNERS belong in a namespace directory"))
385 continue
386 try:
387 tree = load_tree(path)
388 except TreeError:
389 continue
390 if tree.namespace != ns_dir.name:
391 problems.append(Problem(prel, f"the tree says namespace {tree.namespace!r}; it sits under catalog/{ns_dir.name}/"))
392 if path.stem != tree.name:
393 problems.append(Problem(prel, f"the tree is named {tree.name!r}; its file must be {tree.name}.toml"))
394 if changed and author:
395 login = author.lstrip("@").lower()
396 for c in changed:
397 parts = Path(c).as_posix().split("/")
398 if len(parts) >= 3 and parts[0] == "catalog":
399 own = owners(catalog / parts[1])
400 if own and login not in own:
401 problems.append(Problem(c, f"{author} is not in catalog/{parts[1]}/{OWNERS}; only a namespace's owners change it"))
402 for c, (base_version, base_bytes) in (base or {}).items():
403 path = root / c
404 if not path.is_file():
405 continue # a removal
406 if path.read_bytes() == base_bytes:
407 continue
408 try:
409 tree = load_tree(path)
410 except TreeError:
411 continue
412 if tree.version <= base_version:
413 problems.append(Problem(c, f"the file changed but version is still {tree.version}; bump it so pulls notice"))
414 return problems
415
416
417def publish(file: str | Path, checkout: str | Path, *, owners: list[str] | None = None) -> tuple[Path, list[str]]:
418 """Put a tree into ``checkout/catalog/<namespace>/<name>.toml``, write
419 OWNERS for a new namespace, and run the gate. Returns the placed path
420 and notes for the person. The pull request they open is the publishing."""
421 logins = [o.lstrip("@").lower() for o in (owners or [])]
422 src = Path(file)
423 try:
424 tree = load_tree(src)
425 except TreeError as e:
426 raise CatalogError(str(e)) from e
427 errors = [p for p in check(tree) if p.level == "error"]
428 if errors:
429 raise CatalogError(f"{src}: " + "; ".join(f"{p.where}: {p.message}" for p in errors))
430 if not tree.namespace:
431 raise CatalogError(f"{src}: add a line namespace = \"you\" (lowercase; it is who publishes the tree, and the catalog id becomes you/{tree.name})")
432 if not tree.license:
433 raise CatalogError(f"{src}: add a line license = \"MIT\" (an SPDX identifier: what others may do with it)")
434 catalog = Path(checkout) / "catalog"
435 if not catalog.is_dir():
436 raise CatalogError(f"{checkout}: no catalog/ directory; is this a checkout of github.com/b-i-t-8/pyrtan?")
437 ns_dir = catalog / tree.namespace
438 notes: list[str] = []
439 if not ns_dir.is_dir():
440 if not logins:
441 raise CatalogError(f"catalog/{tree.namespace}/ is new: say who owns it with --owner YOUR-GITHUB-LOGIN")
442 ns_dir.mkdir(parents=True)
443 (ns_dir / OWNERS).write_text("# GitHub logins allowed to change this namespace, one per line.\n" + "".join(f"{o}\n" for o in logins), encoding="utf-8")
444 notes.append(f"new namespace catalog/{tree.namespace}/ owned by {', '.join(logins)}")
445 elif logins:
446 have = globals()["owners"](ns_dir)
447 missing = [o for o in logins if o not in have]
448 if missing:
449 with (ns_dir / OWNERS).open("a", encoding="utf-8") as f:
450 f.write("".join(f"{o}\n" for o in missing))
451 notes.append(f"added {', '.join(missing)} to catalog/{tree.namespace}/{OWNERS}")
452 target = ns_dir / f"{tree.name}.toml"
453 data = src.read_bytes()
454 if target.exists():
455 if target.read_bytes() == data:
456 notes.append(f"{target.relative_to(checkout)} already holds this exact file")
457 else:
458 old = load_tree(target)
459 if tree.version <= old.version:
460 raise CatalogError(f"catalog/{tree.namespace}/{tree.name}.toml is at version {old.version}; bump this file's version past it")
461 notes.append(f"replaces version {old.version} with {tree.version}")
462 target.write_bytes(data)
463 problems = gate(checkout)
464 if problems:
465 raise CatalogError("the gate refuses the checkout as it now stands:\n " + "\n ".join(str(p) for p in problems))
466 return target.resolve(), notes