pyrtan / sourcev0.14.0 · main

pyrtan / tests / test_catalog.py

15,867 bytes · last changed 2026-10-27 in Merge main (chat, hub list, releases 0.11-0.13) into the console as a hub of its own

1"""The catalog: building an index from a directory, and pulling from one
2over HTTP with every check that keeps a bad file out."""
3
4import hashlib
5import json
6import shutil
7import threading
8from functools import partial
9from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
10
11import pytest
12
13from pyrtan.catalog import Catalog, CatalogError, build_index
14from pyrtan.cli import main
15from pyrtan.tree import load_tree
16from conftest import EXAMPLES
17
18
19@pytest.fixture
20def catalog_dir(tmp_path):
21 root = tmp_path / "catalog"
22 shutil.copytree(EXAMPLES, root)
23 index, problems = build_index(root)
24 assert not problems, problems
25 (root / "index.json").write_text(json.dumps(index))
26 return root
27
28
29@pytest.fixture
30def served(catalog_dir):
31 class Quiet(SimpleHTTPRequestHandler):
32 def log_message(self, *a):
33 pass
34
35 httpd = ThreadingHTTPServer(("127.0.0.1", 0), partial(Quiet, directory=str(catalog_dir)))
36 threading.Thread(target=httpd.serve_forever, daemon=True).start()
37 yield f"http://127.0.0.1:{httpd.server_address[1]}/"
38 httpd.shutdown()
39
40
41def test_build_index_describes_every_tree(catalog_dir):
42 index = json.loads((catalog_dir / "index.json").read_text())
43 assert index["pyrtan-catalog"] == 1 and index["count"] == 11
44 weather = next(e for e in index["trees"] if e["name"] == "weather")
45 assert weather["path"] == "weather/weather.toml" and weather["license"] == "MIT"
46 assert weather["sha256"] == hashlib.sha256((catalog_dir / "weather/weather.toml").read_bytes()).hexdigest()
47 assert "[what's the] weather in {place}" in weather["triggers"]
48 assert weather["inputs"]["place"]["required"] is True
49
50
51def test_build_index_leaves_out_what_does_not_belong(tmp_path):
52 (tmp_path / "ok.toml").write_text('name = "ok"\nlicense = "MIT"\nstart = "s"\n[[triggers]]\npattern = "ok now"\n[nodes.s]\nkind = "done"\nsay = "ok"\n')
53 (tmp_path / "zz-dup.toml").write_text('name = "ok"\nlicense = "MIT"\nstart = "s"\n[[triggers]]\npattern = "ok again"\n[nodes.s]\nkind = "done"\nsay = "ok"\n')
54 (tmp_path / "unlicensed.toml").write_text('name = "free"\nstart = "s"\n[[triggers]]\npattern = "free now"\n[nodes.s]\nkind = "done"\nsay = "ok"\n')
55 (tmp_path / "broken.toml").write_text("name = \n")
56 (tmp_path / ".hidden").mkdir()
57 (tmp_path / ".hidden" / "x.toml").write_text("nope")
58 index, problems = build_index(tmp_path)
59 assert [e["name"] for e in index["trees"]] == ["ok"]
60 messages = {p.path: p.message for p in problems}
61 assert "already at ok.toml" in messages["zz-dup.toml"]
62 assert "license" in messages["unlicensed.toml"]
63 assert "broken.toml" in messages
64 assert "x.toml" not in " ".join(messages)
65 index, problems = build_index(tmp_path, require_license=False)
66 assert {e["name"] for e in index["trees"]} == {"ok", "free"}
67
68
69def test_search_and_find(served):
70 cat = Catalog(served)
71 assert {e["name"] for e in cat.search("")} == {"weather", "hacker-news", "hacker-news-item", "site-status", "github-release", "disk-space", "log-errors", "room-temperature", "rendered-headline", "modem-check", "small-talk"}
72 assert [e["name"] for e in cat.search("open-meteo")] == ["weather"]
73 assert [e["name"] for e in cat.search("news stories")] == ["hacker-news"]
74 assert cat.search("nothing like this") == []
75 assert cat.find("weather")["version"] == 2
76 assert cat.find("weather@2")["name"] == "weather"
77 with pytest.raises(CatalogError, match="has version 2, not 4"):
78 cat.find("weather@4")
79 with pytest.raises(CatalogError, match="not in the catalog"):
80 cat.find("nope")
81 with pytest.raises(CatalogError, match="NAME"):
82 cat.find("Not Valid")
83
84
85def test_pull_writes_verified_file_and_refuses_to_clobber(served, tmp_path):
86 cat = Catalog(served)
87 target = cat.pull("weather", tmp_path / "trees")
88 assert target == tmp_path / "trees" / "weather.toml"
89 assert target.read_bytes() == (EXAMPLES / "weather" / "weather.toml").read_bytes()
90 assert cat.pull("weather", tmp_path / "trees") == target # same bytes: fine
91 target.write_text(target.read_text() + "\n# mine\n")
92 with pytest.raises(CatalogError, match="--force"):
93 cat.pull("weather", tmp_path / "trees")
94 cat.pull("weather", tmp_path / "trees", force=True)
95 assert "# mine" not in target.read_text()
96
97
98def test_pull_refuses_a_file_that_does_not_match_the_index(catalog_dir, served, tmp_path):
99 index = json.loads((catalog_dir / "index.json").read_text())
100 # Tamper with the served file after the index was built.
101 path = catalog_dir / "weather" / "weather.toml"
102 path.write_text(path.read_text().replace("api.open-meteo.com", "evil.example"))
103 with pytest.raises(CatalogError, match="does not match the index"):
104 Catalog(served).pull("weather", tmp_path)
105 # An index that points outside the catalog is refused too.
106 for e in index["trees"]:
107 if e["name"] == "weather":
108 e["path"] = "../../etc/passwd"
109 (catalog_dir / "index.json").write_text(json.dumps(index))
110 with pytest.raises(CatalogError, match="outside the catalog"):
111 Catalog(served).pull("weather", tmp_path)
112 # An index whose entry says one name while the file says another.
113 index = json.loads((catalog_dir / "index.json").read_text())
114 hn = next(e for e in index["trees"] if e["name"] == "hacker-news")
115 hn["path"] = "github/github-release.toml"
116 hn["sha256"] = hashlib.sha256((catalog_dir / "github/github-release.toml").read_bytes()).hexdigest()
117 (catalog_dir / "index.json").write_text(json.dumps(index))
118 with pytest.raises(CatalogError, match="says it is 'github-release'"):
119 Catalog(served).pull("hacker-news", tmp_path)
120
121
122def test_not_a_catalog(served, tmp_path):
123 (tmp_path / "index.json").write_text("{}")
124 with pytest.raises(CatalogError, match="not a pyrtan catalog"):
125 Catalog(str(tmp_path)).entries()
126 with pytest.raises(CatalogError):
127 Catalog(served + "missing/").entries()
128
129
130def test_cli_index_search_pull(catalog_dir, served, tmp_path, capsys):
131 out = tmp_path / "idx.json"
132 assert main(["index", str(catalog_dir), "--out", str(out)]) == 0
133 assert json.loads(out.read_text())["count"] == 11
134 assert main(["search", "weather", "--catalog", served]) == 0
135 assert "weather@2" in capsys.readouterr().out
136 assert main(["search", "zzz", "--catalog", served]) == 1
137 assert main(["pull", "site-status", "weather@2", "--catalog", served, "--dir", str(tmp_path / "t")]) == 0
138 assert (tmp_path / "t" / "site-status.toml").exists() and (tmp_path / "t" / "weather.toml").exists()
139 assert main(["pull", "weather@9", "--catalog", served, "--dir", str(tmp_path / "t")]) == 1
140 assert "not 9" in capsys.readouterr().err
141 assert main(["console", "--out", str(tmp_path / "site" / "console.html")]) == 0
142 assert "<title>Pyrtan</title>" in (tmp_path / "site" / "console.html").read_text()
143
144
145def test_index_writes_the_trees_as_json_for_the_browser(catalog_dir, tmp_path):
146 # The console in a browser holds the catalog's trees: the same trees, as JSON, beside the index.
147 site = tmp_path / "site"
148 assert main(["index", str(catalog_dir), "--copy", str(site)]) == 0
149 index = json.loads((site / "index.json").read_text())
150 bundle = json.loads((site / "trees.json").read_text())
151 assert bundle["pyrtan-trees"] == 1 and bundle["count"] == index["count"] == len(index["trees"]) > 0
152 assert [(t["id"], t["sha256"]) for t in bundle["trees"]] == [(e["id"], e["sha256"]) for e in index["trees"]]
153 weather = next(t["tree"] for t in bundle["trees"] if t["id"].endswith("weather"))
154 assert weather == load_tree(site / next(e["path"] for e in index["trees"] if e["id"].endswith("weather"))).raw
155
156
157# ---------------------------------------------------------------- namespaces, the gate, publishing
158
159
160OTHER = '''
161name = "weather"
162version = 2
163namespace = "barry"
164description = "someone else's weather"
165license = "MIT"
166hosts = ["example.com"]
167start = "get"
168
169[nodes.get]
170kind = "observe"
171http = { url = "https://example.com/" }
172next = "say"
173
174[nodes.say]
175kind = "done"
176say = "ok"
177'''
178
179
180def test_namespaces_in_the_index_and_in_pulls(tmp_path):
181 src = tmp_path / "src"
182 (src / "examples").mkdir(parents=True)
183 shutil.copytree(EXAMPLES, src / "examples", dirs_exist_ok=True)
184 (src / "catalog" / "barry").mkdir(parents=True)
185 (src / "catalog" / "barry" / "weather.toml").write_text(OTHER)
186 out = tmp_path / "out"
187 index, problems = build_index([src / "examples", src / "catalog"], copy_to=out, require_namespace=True)
188 assert not problems, problems
189 ids = {e["id"]: e for e in index["trees"]}
190 assert ids["pyrtan/weather"]["path"] == "pyrtan/weather.toml" and ids["barry/weather"]["path"] == "barry/weather.toml"
191 assert (out / "pyrtan" / "weather.toml").exists() and (out / "barry" / "weather.toml").read_text() == OTHER
192 assert ids["pyrtan/disk-space"]["sources"] == ["shell"] and ids["pyrtan/disk-space"]["commands"] == ["df"]
193 (out / "index.json").write_text(json.dumps(index))
194 cat = Catalog(str(out))
195 # A bare name is enough when one namespace has it; with two, say which.
196 assert cat.find("github-release")["id"] == "pyrtan/github-release"
197 with pytest.raises(CatalogError, match="more than one namespace: barry/weather, pyrtan/weather"):
198 cat.find("weather")
199 assert cat.find("barry/weather")["version"] == 2 and cat.find("pyrtan/weather@2")["namespace"] == "pyrtan"
200 with pytest.raises(CatalogError, match="has version 2, not 9"):
201 cat.find("barry/weather@9")
202 with pytest.raises(CatalogError, match="not in the catalog"):
203 cat.find("nobody/weather")
204 assert [e["id"] for e in cat.search("barry")] == ["barry/weather"]
205 assert {e["id"] for e in cat.search("shell")} == {"pyrtan/disk-space"}
206 target = cat.pull("barry/weather", tmp_path / "trees")
207 assert target.name == "weather.toml" and target.read_text() == OTHER
208 # A file whose namespace is not the index's is refused.
209 (out / "barry" / "weather.toml").write_text(OTHER.replace('namespace = "barry"', 'namespace = "mallory"'))
210 index["trees"] = [dict(e, sha256=hashlib.sha256((out / e["path"]).read_bytes()).hexdigest()) if e["id"] == "barry/weather" else e for e in index["trees"]]
211 (out / "index.json").write_text(json.dumps(index))
212 with pytest.raises(CatalogError, match="namespace is 'mallory'"):
213 Catalog(str(out)).pull("barry/weather", tmp_path / "trees2")
214 # Two directories need a place to copy to; one does not.
215 with pytest.raises(ValueError):
216 build_index([src / "examples", src / "catalog"])
217 one, problems = build_index(src / "examples")
218 assert not problems and {e["path"] for e in one["trees"]} >= {"weather/weather.toml"}
219
220
221def test_gate_and_publish(tmp_path, capsys):
222 from pyrtan.catalog import gate, publish
223
224 repo = tmp_path / "repo"
225 shutil.copytree(EXAMPLES, repo / "examples")
226 (repo / "catalog").mkdir()
227 assert gate(repo) == []
228 # Publishing needs a namespace and a licence in the file, and an owner for a new namespace.
229 tree = tmp_path / "lan-printer.toml"
230 tree.write_text(OTHER.replace('name = "weather"', 'name = "lan-printer"').replace('namespace = "barry"\n', ""))
231 with pytest.raises(CatalogError, match='namespace = "you"'):
232 publish(tree, repo, owners=["barry"])
233 tree.write_text(OTHER.replace('name = "weather"', 'name = "lan-printer"'))
234 with pytest.raises(CatalogError, match="say who owns it"):
235 publish(tree, repo)
236 placed, notes = publish(tree, repo, owners=["@Barry"])
237 assert placed == (repo / "catalog" / "barry" / "lan-printer.toml").resolve() and notes == ["new namespace catalog/barry/ owned by barry"]
238 assert (repo / "catalog" / "barry" / "OWNERS").read_text().endswith("barry\n")
239 assert gate(repo) == []
240 # The same file again is a no-op; a changed file must bump its version.
241 assert publish(tree, repo)[1] == ["catalog/barry/lan-printer.toml already holds this exact file"]
242 tree.write_text(tree.read_text().replace('say = "ok"', 'say = "changed"'))
243 with pytest.raises(CatalogError, match="bump this file's version past it"):
244 publish(tree, repo)
245 tree.write_text(tree.read_text().replace("version = 2", "version = 3"))
246 assert publish(tree, repo, owners=["alice"])[1] == ["added alice to catalog/barry/OWNERS", "replaces version 2 with 3"]
247 # What the gate refuses.
248 bad = repo / "catalog" / "barry" / "weather.toml"
249 bad.write_text(OTHER.replace('namespace = "barry"', 'namespace = "pyrtan"'))
250 problems = [str(p) for p in gate(repo)]
251 assert any("catalog/barry/weather.toml: a tree with id 'pyrtan/weather' is already at" in p for p in problems)
252 assert any("sits under catalog/barry/" in p for p in problems)
253 bad.write_text(OTHER.replace('name = "weather"', 'name = "other"'))
254 assert any("its file must be other.toml" in p for p in [str(p) for p in gate(repo)])
255 bad.unlink()
256 (repo / "catalog" / "barry" / "notes.txt").write_text("x")
257 assert any("only <name>.toml files and OWNERS belong" in p for p in [str(p) for p in gate(repo)])
258 (repo / "catalog" / "barry" / "notes.txt").unlink()
259 (repo / "catalog" / "nobody").mkdir()
260 (repo / "catalog" / "nobody" / "t.toml").write_text(OTHER.replace('namespace = "barry"', 'namespace = "nobody"').replace('name = "weather"', 'name = "t"'))
261 assert any("needs an OWNERS file" in p for p in [str(p) for p in gate(repo)])
262 shutil.rmtree(repo / "catalog" / "nobody")
263 (repo / "examples" / "weather" / "weather.toml").write_text((repo / "examples" / "weather" / "weather.toml").read_text().replace('namespace = "pyrtan"', 'namespace = "barry"'))
264 assert any("published by 'pyrtan', not 'barry'" in p for p in [str(p) for p in gate(repo)])
265 shutil.copy(EXAMPLES / "weather" / "weather.toml", repo / "examples" / "weather" / "weather.toml")
266 assert gate(repo) == []
267 # Only owners change a namespace; a changed file bumps its version.
268 assert [str(p) for p in gate(repo, changed=["catalog/barry/lan-printer.toml"], author="mallory")] == ["catalog/barry/lan-printer.toml: mallory is not in catalog/barry/OWNERS; only a namespace's owners change it"]
269 assert gate(repo, changed=["catalog/barry/lan-printer.toml"], author="Alice") == []
270 assert gate(repo, changed=["catalog/barry/lan-printer.toml"], author="alice", base={"catalog/barry/lan-printer.toml": (3, b"older bytes")}) != []
271 assert gate(repo, changed=["catalog/barry/lan-printer.toml"], author="alice", base={"catalog/barry/lan-printer.toml": (2, b"older bytes")}) == []
272 assert gate(repo, base={"catalog/barry/gone.toml": (1, b"x")}) == [] # a removal is fine
273 # The command, and the index the site serves.
274 assert main(["publish", str(tree), "--into", str(repo)]) == 0
275 out = capsys.readouterr().out
276 assert "placed" in out and "git checkout -b catalog/barry-lan-printer" in out
277 assert main(["index", str(repo / "examples"), str(repo / "catalog"), "--copy", str(tmp_path / "site"), "--require-namespace"]) == 0
278 served = json.loads((tmp_path / "site" / "index.json").read_text())
279 assert {e["id"] for e in served["trees"]} == {f"pyrtan/{n}" for n in ("weather", "hacker-news", "hacker-news-item", "site-status", "github-release", "disk-space", "log-errors", "room-temperature", "rendered-headline", "modem-check", "small-talk")} | {"barry/lan-printer"}
280 assert main(["index", str(repo / "examples"), str(repo / "catalog")]) == 2 # two roots need --copy
281 assert main(["search", "lan", "--catalog", str(tmp_path / "site")]) == 0 and "barry/lan-printer@3" in capsys.readouterr().out
282 assert main(["pull", "lan-printer", "--catalog", str(tmp_path / "site"), "--dir", str(tmp_path / "pulled")]) == 0
283 assert "barry/lan-printer@3 ->" in capsys.readouterr().out