pyrtan / sourcev0.19.0 · main

pyrtan / tests / conformance / make_vectors.py

20,377 bytes · last changed 2026-10-28 in Merge remote-tracking branch 'origin/main' into claude/great-heisenberg-o6ocah

1"""Write tests/conformance/vectors.json: what both runtimes must say.
2
3Every example tree as JSON (the TOML's own table), and for a fixed set of
4runs (a sentence, or a tree and inputs) the outcome the Python runtime
5produced against the recorded responses in tests/fixtures/examples.json,
6with a fixed clock. The JavaScript runtime (src/pyrtan/js/pyrtan.js) must
7produce the same status, say, output, reason, path and gaps.
8
9Since the browser reads sentences too (it is a hub of its own until it
10joins one), the vectors also hold how Python reads: every sentence's
11reading (the rewrites, each trigger's verdict, the intent step), the
12language rules and value parsers one by one, the tree proposed for a
13sentence nobody answers, and a sequence of requests to a service and
14what it answered, which pyrtan.js's LocalHub must answer the same.
15tests/conformance/run.cjs checks it all, and tests/test_conformance.py
16keeps this file from drifting.
17
18 uv run python tests/conformance/make_vectors.py
19"""
20
21from __future__ import annotations
22
23import datetime as _dt
24import json
25import sys
26from pathlib import Path
27
28ROOT = Path(__file__).resolve().parents[2]
29sys.path.insert(0, str(ROOT / "src"))
30
31from pyrtan import Registry, RecordedTransport, Runner # noqa: E402
32from pyrtan import nlp # noqa: E402
33from pyrtan.grow import draft_branch, propose # noqa: E402
34from pyrtan.service import Service # noqa: E402
35from pyrtan.tree import build as build_tree # noqa: E402
36
37CLOCK = _dt.datetime(2026, 10, 25, 12, 0, tzinfo=_dt.timezone.utc)
38CASES: list[dict] = [
39 {"intent": "What's the top story on Hacker News?"},
40 {"intent": "top 3 stories on hacker news"},
41 {"intent": "top 99 stories on hacker news"},
42 {"intent": "hacker news item 999"},
43 {"intent": "weather in Oslo"},
44 {"tree": "weather", "inputs": {"place": "Oslo", "units": "fahrenheit"}},
45 {"tree": "weather", "inputs": {"place": "Oslo", "units": "kelvin"}},
46 {"intent": "how cold is it in Nowhereville"},
47 {"intent": "is Example.com up"},
48 {"intent": "is broken.example up"},
49 {"intent": "check status of teapot.example"},
50 {"tree": "site-status", "inputs": {"host": "http://x"}},
51 {"intent": "latest release of b-i-t-8/pyrtan"},
52 {"intent": "latest b-i-t-8/nothing release"},
53 {"intent": "latest release of b-i-t-8/busy"},
54 {"intent": "weather in Paris"},
55 {"tree": "hacker-news-item", "inputs": {"id": 101}},
56 {"tree": "hacker-news", "inputs": {"count": 2}},
57 # The other sources, answered by the same fixture.
58 {"intent": "how full is the disk"},
59 {"intent": "disk space on /data"},
60 {"intent": "disk space on /nowhere"},
61 {"intent": "errors in the log"},
62 {"intent": "errors in /var/log/quiet.log"},
63 {"intent": "errors in /var/log/nothing.log"},
64 {"intent": "errors in /etc/passwd"},
65 {"intent": "temperature in the kitchen"},
66 {"intent": "how warm is the attic"},
67 {"intent": "temperature in the garage"},
68 {"intent": "temperature in the hall"},
69 {"tree": "room-temperature", "inputs": {"room": "a-b"}},
70 {"intent": "headline of https://app.example/dashboard"},
71 {"intent": "what does https://app.example/empty say"},
72 {"intent": "headline of https://app.example/gone"},
73 {"intent": "is the modem up"},
74 {"intent": "is the modem on /dev/ttyUSB1 up"},
75 {"intent": "is the modem on /dev/ttyACM0 up"},
76 {"intent": "is the modem on /dev/ttyS0 up"},
77 {"intent": "how are you today?"},
78 {"intent": "are you a bot"},
79 {"intent": "goodbye"},
80 # Trees that run in a browser too.
81 {"intent": "tell me about Ursula K. Le Guin"},
82 {"intent": "wiki Mercury"},
83 {"intent": "who is Xyzzyplugh Notaperson"},
84 {"intent": "who was Alan Turing"},
85 {"intent": "what does serendipity mean"},
86 {"intent": "define zzzqx"},
87 {"intent": "when is sunset in Oslo"},
88 {"intent": "sunrise in Longyearbyen"},
89 {"intent": "air quality in Oslo"},
90 {"intent": "how is the air in Delhi"},
91 {"intent": "latest version of @types/node on npm"},
92 {"intent": "npm package nope-not-a-pkg-zz"},
93 {"intent": "latest requests on pypi"},
94 {"intent": "pip version nope-not-a-pkg-zz"},
95 {"intent": "any earthquakes today"},
96 {"intent": "where is the iss"},
97 {"intent": "search hacker news for sqlite"},
98 {"intent": "search hacker news for qqqzzzxxyyy"},
99 {"intent": "who wrote The Left Hand of Darkness"},
100 {"intent": "find the book qqqzzzxxyyy"},
101 {"intent": "next holiday in no"},
102 {"intent": "next holiday in XX"},
103 {"intent": "tell me a joke"},
104 {"intent": "convert 100 usd to eur"},
105 {"intent": "convert 1,000 EUR into JPY"},
106 {"intent": "usd to xyz rate"},
107 {"intent": "exchange rate from usd to usd"},
108 {"intent": "synonyms for happy"},
109 {"intent": "another word for qqqzzz"},
110 {"intent": "weather in Oslo tomorrow"},
111 {"intent": "uv index in Sydney"},
112 {"intent": "what time is it in La Paz"},
113 {"intent": "how high is La Paz"},
114 {"intent": "how high is Oslo"},
115 {"intent": "where is zip 90210"},
116 {"intent": "postcode SW1A in GB"},
117 {"intent": "where is zip 00000"},
118 {"intent": "is example.com archived"},
119 {"intent": "is nothing-here-qqzz.example archived"},
120 {"intent": "what rhymes with cat"},
121 {"intent": "what rhymes with qqzzx"},
122 {"intent": "how many stars does astral-sh/uv have"},
123 {"intent": "github repo b-i-t-8/nothing-here"},
124 {"intent": "latest version of github.com/gorilla/mux on go"},
125 {"intent": "go module github.com/nope/nope-qq"},
126 {"intent": "brew formula wget"},
127 {"intent": "brew formula nope-qqzz"},
128 {"intent": "pokemon pikachu"},
129 {"intent": "what type is charizard"},
130 {"intent": "pokemon notapokemonqq"},
131 {"intent": "what is my ip"},
132 {"intent": "give me some advice"},
133 {"intent": "tell me a cat fact"},
134 {"intent": "when is the next launch"},
135 {"intent": "space news"},
136 {"intent": "astronomy picture of the day"},
137]
138
139
140# Trees that exercise what the examples do not: every slot kind, synonyms
141# (a word and a phrase), typos, fixed inputs, a tie, and an ask.
142def _echo(name: str, triggers: list, inputs: dict, **extra) -> dict:
143 return {"name": name, "version": 1, "description": f"echo for {name}", "hosts": [], "start": "reply", "triggers": triggers,
144 "inputs": inputs, "nodes": {"reply": {"kind": "done", "say": "ok", "output": {k: f"{{{{ inputs.{k} }}}}" for k in inputs}}}, **extra}
145
146
147LANGUAGE_TREES: list[dict] = [
148 _echo("remind", ["remind me {when:date} at {at:time}", "remind me in {wait:duration}"], {"when": {"type": "str", "default": ""}, "at": {"type": "str", "default": ""}, "wait": {"type": "int", "default": 0}}),
149 _echo("count", ["count to {n:number}", "add {x:float} and {y:int}"], {"n": {"type": "float", "default": 0}, "x": {"type": "float", "default": 0.0}, "y": {"type": "int", "default": 0}}),
150 _echo("ping", ["ping {host:host}", "fetch {url:url}", "mail {to:email}"], {"host": {"type": "str", "default": ""}, "url": {"type": "str", "default": ""}, "to": {"type": "str", "default": ""}}),
151 _echo("library", ["(find|search) books by {author}", "top story on book club"], {"author": {"type": "str", "default": ""}, "count": {"type": "int", "default": 3}},
152 language={"synonyms": {"books": ["novels", "paperback books"], "Find": "look up"}, "typos": True}),
153 _echo("tie-a", ["shout {x:word}"], {"x": {"type": "str", "default": ""}}),
154 _echo("tie-b", ["shout {y:word}"], {"y": {"type": "str", "default": ""}}),
155 _echo("fixed", [{"pattern": "the usual", "inputs": {"what": "coffee"}}], {"what": {"type": "str", "default": "tea"}}),
156 {"name": "order", "version": 1, "description": "asks, then answers", "hosts": [], "start": "size",
157 "triggers": ["order a {drink:word}"], "inputs": {"drink": {"type": "str", "required": True}, "shots": {"type": "int", "default": 1}},
158 "nodes": {"size": {"kind": "ask", "prompt": "What size of {{ inputs.drink }}?", "into": "state.size", "next": "shots"},
159 "shots": {"kind": "ask", "prompt": "How many shots?", "into": "inputs.shots", "next": "reply"},
160 "reply": {"kind": "done", "say": "A {{ state.size }} {{ inputs.drink }}, {{ inputs.shots }} shot(s).", "output": {"size": "{{ state.size }}", "shots": "{{ inputs.shots }}"}}}},
161]
162
163READINGS = [
164 # the examples, read
165 "What's the top story on Hacker News?", "top five stories on hacker news", "hey pyrtan, top 3 stories on hacker news please",
166 "weather in Oslo", "what's the forecast for New York", "how cold is it in Oslo", "is Example.com up", "is http://Example.com:8080/x up",
167 "is not a host up", "latest release of b-i-t-8/pyrtan", "how full is the disk", "disk space on /data", "the temperature in the attic",
168 "headline of https://app.example/dashboard", "is the modem on /dev/ttyUSB1 up", "make me a sandwich", "", " ?! ",
169 "Could you please tell me the weather in Zürich, thanks!", "TOP 3 STORIES ON HACKER NEWS", "top twenty-one stories on hacker news",
170 "weather in Oslo for me", "hello", "pyrtan",
171 # the language trees, read
172 "remind me tomorrow at 3pm", "remind me next friday at noon", "remind me 25 dec at 9:30 pm", "remind me 3/4 at 10am",
173 "remind me 31/12 at 25:00", "remind me in 2 hours", "remind me in an hour and a half", "remind me in 1h30m", "remind me in half an hour",
174 "remind me in 2 parsecs", "count to two hundred and five", "count to 1,000.5", "count to 2.0", "count to a dozen", "add 1.5 and 3",
175 "ping localhost", "ping 10.0.0.1:22", "ping not_a_host", "fetch https://example.com/a?b=c", "fetch ftp://x", "mail Barry@Example.COM",
176 "mail nobody", "find books by Ursula Le Guin", "search novels by Le Guin", "look up paperback books by Le Guin", "finnd books by Le Guin",
177 "serch books by Le Guin", "top stories on book club", "shout hello", "the usual", "order a latte", "it's the usual",
178]
179
180NLP = [
181 ("clean", "Hey, can you please tell me what's up?"), ("clean", "“Curly” quotes — and dashes"), ("clean", "ＦＵＬＬ width"),
182 ("clean", "I'm sure it's a dozen"), ("clean", "one hundred and twenty three things"), ("clean", "a hundred and one"),
183 ("clean", "Thanks"), ("clean", "well, um, okay pyrtan: weather."), ("clean", "St. Ives. Then Mr. Smith"),
184 ("clean", "convert 1,000.5 EUR, then 2, 3"), ("clean", "a,b and 1 ,2 and 3, 4"),
185 ("singular", "stories"), ("singular", "boxes"), ("singular", "news"), ("singular", "glass"), ("singular", "cats"), ("singular", "bus"),
186 ("plural", "story"), ("plural", "box"), ("plural", "day"), ("plural", "status"),
187 ("parse_number", "twenty-one"), ("parse_number", "1,234"), ("parse_number", "-3.25"), ("parse_number", "lots"),
188 ("parse_duration", "90 seconds"), ("parse_duration", "2 days and 3 hours"), ("parse_duration", "a week"), ("parse_duration", "1.5 hours and a half"),
189 ("parse_duration", "soon"), ("parse_duration", "3 fortnights"),
190 ("parse_time", "15:30"), ("parse_time", "3:05 pm"), ("parse_time", "9 o'clock"), ("parse_time", "12am"), ("parse_time", "13pm"),
191 ("parse_time", "7"), ("parse_time", "midnight"), ("parse_time", "nine thirty"),
192 ("parse_date", "2026-02-29"), ("parse_date", "2028-02-29"), ("parse_date", "25 oct"), ("parse_date", "oct 25, 2027"),
193 ("parse_date", "the 1st of march"), ("parse_date", "sept 3"), ("parse_date", "25/10/2026"), ("parse_date", "10/10"),
194 ("parse_date", "3/4"), ("parse_date", "today"), ("parse_date", "yesterday"), ("parse_date", "day after tomorrow"),
195 ("parse_date", "sunday"), ("parse_date", "this sunday"), ("parse_date", "next sunday"), ("parse_date", "last sunday"),
196 ("parse_date", "last mon"), ("parse_date", "in 3 weeks"), ("parse_date", "in 4 months"), ("parse_date", "2 years ago"),
197 ("parse_date", "next month"), ("parse_date", "end of month"), ("parse_date", "end of the year"), ("parse_date", "start of month"),
198 ("parse_date", "someday"), ("parse_date", "30 feb"),
199 ("parse_host", "Example.COM"), ("parse_host", "https://a.b.example:8443/x"), ("parse_host", "not a host"),
200 ("parse_url", "http://x.example/y"), ("parse_url", "x.example"), ("parse_email", "A.B@Example.org"), ("parse_email", "a@b"),
201]
202
203PROPOSALS = [
204 "make me a sandwich", "is example.com up", "top 3 stories on lobsters", "what is 2.5 times 4", "latest release of foo/bar",
205 "", "the the the", "{weird} [pattern] (here)", "what's the weather on mars?", "ping 10.0.0.1 and 10.0.0.2", "café opening hours",
206 'say "hello" to $USER', "it's raining",
207]
208
209# Gaps on the examples, and the branch grow.py's rules draft for each.
210DRAFTS = [
211 ("weather", {"node": "sky", "on": "code", "observed": 4}), # the hole between 3 and 45
212 ("weather", {"node": "sky", "on": "code", "observed": 100}), # past the last range
213 ("weather", {"node": "sky", "on": "code", "observed": 61}), # covered already
214 ("weather", {"node": "sky", "on": "code", "observed": None}), # missing
215 ("weather", {"node": "sky", "on": "code", "observed": "<dict>"}), # a report's type
216 ("weather", {"node": "sky", "on": "code", "observed": "Sunny Spells"}),
217 ("weather", {"node": "geocode", "on": "x", "observed": 1}), # not a decide
218 ("site-status", {"node": "status", "observed": 700}),
219 ("site-status", {"node": "status", "observed": True}),
220]
221
222HUB = [
223 ("GET", "/capabilities", None),
224 ("POST", "/intent", {"text": "top 3 stories on hacker news"}),
225 ("POST", "/intent", {"text": "weather in Oslo", "trace": False}),
226 ("POST", "/intent", {"text": "how cold is it in Nowhereville"}),
227 ("POST", "/intent", {"text": "is broken.example up"}),
228 ("POST", "/intent", {"text": "is broken.example up"}),
229 ("POST", "/intent", {"text": "make me a sandwich"}),
230 ("POST", "/intent", {"text": "Make me a sandwich!"}),
231 ("POST", "/intent", {"text": "shout hello"}),
232 ("POST", "/intent", {"text": " "}),
233 ("POST", "/intent", {}),
234 ("POST", "/run", {"tree": "hacker-news", "inputs": {"count": 2}}),
235 ("POST", "/run", {"tree": "nope"}),
236 ("POST", "/run", {"tree": "weather", "inputs": {"units": "kelvin", "place": "Oslo"}}),
237 ("POST", "/intent", {"text": "order a latte"}),
238 ("POST", "/resume", {"snapshot": "@last", "answer": "large"}),
239 ("POST", "/resume", {"snapshot": "@last", "answer": "2"}),
240 ("POST", "/resume", {"snapshot": {"nope": 1}}),
241 ("POST", "/intent", {"text": "order a mocha"}),
242 ("POST", "/resume", {"snapshot": "@last", "answer": "small"}),
243 ("POST", "/resume", {"snapshot": "@last", "answer": "many"}),
244 ("POST", "/intent", {"text": "temperature in the hall"}),
245 ("POST", "/intent", {"text": "what's the temperature in the hall"}),
246 ("GET", "/gaps", None),
247 ("GET", "/unanswered", None),
248 ("POST", "/suggestions", {"suggestions": [
249 {"kind": "branch", "tree": "weather", "node": "sky", "on": "code", "observed": 100, "meaning": {"kind": "fail", "say": "{{ state.name }}: code 100 is not a WMO code.", "reason": "not-wmo"}, "why": "WMO codes stop at 99.", "by": {"agent": "scout", "model": "test"}, "extra": "dropped"},
250 {"kind": "trigger", "tree": "weather", "pattern": "is it raining in {place}", "covers": ["is it raining in Oslo"]},
251 {"kind": "tree", "text": "top 3 stories on lobsters", "description": "Top stories from lobste.rs.", "hosts_hint": ["lobste.rs"]},
252 {"kind": "branch", "tree": "weather", "node": "sky", "observed": 1, "meaning": {"kind": "done", "say": "{{ env.SECRET }}"}},
253 {"kind": "trigger", "tree": "weather", "pattern": "{place}", "covers": ["Oslo"]},
254 {"kind": "act", "tree": "weather"},
255 "not a table",
256 ]}),
257 ("POST", "/suggestions", {"kind": "trigger", "tree": "weather", "pattern": "is it raining in {place}", "covers": ["is it raining in Oslo", "is it raining in Bergen"]}),
258 ("POST", "/suggestions", {"nothing": True}),
259 ("GET", "/suggestions", None),
260 ("GET", "/health", None),
261 ("GET", "/watches", None),
262 ("GET", "/nowhere", None),
263]
264
265
266def _nlp_case(fn: str, text: str) -> dict:
267 try:
268 if fn == "parse_date":
269 value = nlp.parse_date(text, CLOCK)
270 else:
271 value = getattr(nlp, fn)(text)
272 except nlp.ParseError as e:
273 return {"fn": fn, "text": text, "error": str(e)}
274 if fn == "clean":
275 value = list(value)
276 return {"fn": fn, "text": text, "value": value}
277
278
279def _hub_exchanges(reg: Registry, fixture: Path) -> list[dict]:
280 """What a service with these trees answers, request by request. A
281 snapshot of "@last" is the one the previous answer carried."""
282 service = Service(reg, name="this browser", transport=RecordedTransport.from_file(fixture))
283 service.runner.clock = lambda: CLOCK
284 out, last = [], None
285 for method, path, body in HUB:
286 if body is not None and body.get("snapshot") == "@last":
287 body = {**body, "snapshot": last}
288 status, payload = service.route(method, path, json.loads(json.dumps(body)) if body is not None else None)
289 if isinstance(payload, dict) and isinstance(payload.get("snapshot"), dict):
290 last = payload["snapshot"]
291 out.append({"method": method, "path": path, "body": body, "status": status, "expect": _stable(_untimed(json.loads(json.dumps(payload))))})
292 return out
293
294
295def _untimed(d):
296 """Drop when a gap or a sentence was seen: the counts and order stay."""
297 if isinstance(d, dict) and "uptime_s" in d:
298 d.pop("uptime_s")
299 d.pop("egress", None) # a page has no egress policy to report
300 for key in ("gaps", "unanswered", "suggestions"):
301 for e in d.get(key, []) if isinstance(d, dict) and isinstance(d.get(key), list) else []:
302 for k in ("first_seen", "last_seen", "first", "last"):
303 e.pop(k, None)
304 return d
305
306
307def _stable(d: dict) -> dict:
308 """Drop the wall-clock fields (when the trace started, how long each step took)."""
309 trace = d.get("trace")
310 if isinstance(trace, dict):
311 trace.pop("started", None)
312 for step in trace.get("steps", []):
313 step.pop("elapsed_ms", None)
314 return d
315
316
317def build() -> dict:
318 reg = Registry.from_paths([ROOT / "examples"])
319 assert not reg.errors, reg.errors
320 fixture = ROOT / "tests" / "fixtures" / "examples.json"
321 runs = []
322 for case in CASES:
323 runner = Runner(reg, RecordedTransport.from_file(fixture), clock=lambda: CLOCK)
324 if "intent" in case:
325 reading = reg.read(case["intent"], CLOCK)
326 match = reading.winner
327 assert match is not None, case
328 outcome = runner.run_intent(case["intent"])
329 runs.append({"intent": case["intent"], "tree": match.tree, "inputs": match.inputs, "expect": _stable(outcome.to_dict())})
330 else:
331 outcome = runner.run_tree(case["tree"], case["inputs"])
332 runs.append({"tree": case["tree"], "inputs": case["inputs"], "expect": _stable(outcome.to_dict())})
333 both = Registry.from_paths([ROOT / "examples"])
334 for raw in LANGUAGE_TREES:
335 both.add(build_tree(raw))
336 readings = [{"text": text, "expect": {**(r := both.read(text, CLOCK)).to_dict(), "step": r.step()}} for text in READINGS]
337 return {
338 "pyrtan-conformance": 1,
339 "clock": CLOCK.isoformat(),
340 "fixture": "tests/fixtures/examples.json",
341 "trees": {t.name: t.raw for t in reg.trees()},
342 "runs": runs,
343 "language_trees": LANGUAGE_TREES,
344 "readings": readings,
345 "nlp": [_nlp_case(fn, text) for fn, text in NLP],
346 "proposals": [{"text": text, "expect": propose(text).to_dict()} for text in PROPOSALS],
347 "hub": _hub_exchanges(both, fixture),
348 "drafts": [{"tree": t, "gap": g, "expect": _draft(reg, t, g)} for t, g in DRAFTS],
349 }
350
351
352def _draft(reg: Registry, tree: str, gap: dict) -> dict | None:
353 d = draft_branch(reg.get(tree), gap)
354 return None if d is None else {k: v for k, v in d.to_dict().items() if k != "node_text"}
355
356
357def main() -> int:
358 out = ROOT / "tests" / "conformance" / "vectors.json"
359 out.write_text(json.dumps(build(), indent=1, ensure_ascii=False, sort_keys=True) + "\n", encoding="utf-8")
360 print(f"wrote {out}: {len(CASES)} runs, {len(READINGS)} readings, {len(NLP)} rules, {len(PROPOSALS)} proposals, {len(HUB)} requests, {len(DRAFTS)} drafts")
361 return 0
362
363
364if __name__ == "__main__":
365 sys.exit(main())