Merge pull request #2 from b-i-t-8/claude/ecstatic-euler-rki9t4
b-i-t-8 · 2026-10-25 · c1bfd35529bfb27137c4e6675256637d2a7e0510 · parents 3ec395d ad079b7
Tokens, CORS, persistent gaps, egress policy, the catalog and the console (v0.2.0)
Changes
CONTRIBUTING.md | 10 +- README.md | 81 +++++++++++--- _meta/design.md | 44 +++++--- _meta/platform.md | 162 +++++++++++++++++++++++++++ _meta/tree-format.md | 1 + deploy/README.md | 12 +- deploy/openmediavault/docker-compose.yml | 16 ++- deploy/pyrtan.com/README.md | 162 +++++++++++++++++++++++++++ deploy/pyrtan.com/catalog.yml | 65 +++++++++++ deploy/pyrtan.com/index.html | 75 +++++++++++++ deploy/pyrtan.com/site.yml | 66 +++++++++++ deploy/pyrtan.com/trees-README.md | 46 ++++++++ examples/github/github-release.toml | 1 + examples/hacker-news/hacker-news-item.toml | 1 + examples/hacker-news/hacker-news.toml | 1 + examples/site-status/site-status.toml | 1 + examples/weather/weather.toml | 1 + pyproject.toml | 2 +- src/pyrtan/__init__.py | 2 +- src/pyrtan/catalog.py | 251 +++++++++++++++++++++++++++++++++++++++++ src/pyrtan/cli.py | 170 +++++++++++++++++++++++++--- src/pyrtan/gaps.py | 95 ++++++++++++++++ src/pyrtan/grow.py | 1 + src/pyrtan/hub.py | 27 +++-- src/pyrtan/page.py | 294 +++++++++++++++++++++++++++++++++++++++++-------- src/pyrtan/runtime.py | 7 +- src/pyrtan/service.py | 201 +++++++++++++++++++++++++++------ src/pyrtan/transport.py | 79 ++++++++++++- src/pyrtan/tree.py | 7 +- tests/test_access.py | 246 +++++++++++++++++++++++++++++++++++++++++ tests/test_catalog.py | 141 ++++++++++++++++++++++++ tests/test_service_hub.py | 5 +- uv.lock | 2 +- 33 files changed, 2117 insertions(+), 158 deletions(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc547c9..39ca8d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md@@ -24,6 +24,12 @@ does not exist, write it; if one gives a wrong answer, the trace shows where.A tree that reads a page should prefer an API or a feed when the site offers one, and should decide on the status code before it decides on the body.+A tree meant for the catalog at pyrtan.com carries a `license` line (an+SPDX identifier; `MIT`, `Apache-2.0` or `CC0-1.0` are the usual choices)+and is contributed to the catalog repository, not this one; see+[`deploy/pyrtan.com/README.md`](deploy/pyrtan.com/README.md). The examples+here are MIT so they can be pulled from there too.+## Changing the runtime The runtime is small on purpose, and every node kind is paid for by every@@ -33,7 +39,9 @@ in the pull request. The same goes for a new filter or condition.Nothing in `src/pyrtan` may call a model, execute code from a tree, or reach a host outside a tree's `hosts`. Those are the three rules the design rests-on; see `_meta/design.md`.+on; see `_meta/design.md`. A fourth, from `_meta/platform.md`: nothing in+`src/pyrtan` knows about tenants, plans or billing. The hosted product runs+this code unchanged.## License of contributions diff --git a/README.md b/README.md index 583963d..110c9f1 100644 --- a/README.md +++ b/README.md@@ -122,12 +122,21 @@ whole format is in [`_meta/tree-format.md`](_meta/tree-format.md).| `pyrtan list DIR` | What is loaded and the sentences it answers to. | | `pyrtan run "…" --trees DIR` | Answers a sentence in this process, printing the trace. `--replay FIXTURE` answers HTTP from a recording; `--record FIXTURE` makes one. | | `pyrtan run --tree NAME --input k=v` | Runs a named tree. |-| `pyrtan serve DIR --port 8701` | Hosts the trees as a service: `/health`, `/capabilities`, `/intent`, `/run`, `/resume`, `/gaps`, `/unanswered`. Reloads trees when their files change. |+| `pyrtan serve DIR --port 8701` | Hosts the trees as a service: `/health`, `/capabilities`, `/intent`, `/run`, `/resume`, `/gaps`, `/unanswered`. Reloads trees when their files change. `--gaps FILE` keeps gaps across restarts; `--egress public` refuses private addresses. || `pyrtan hub --port 8700` | Routes between services. Services register with `--hub URL`; peers carry a `link` of `up`, `stale` or `down`. | | `pyrtan ask "…" --at URL` | Answers a sentence through a running service or hub. | | `pyrtan gaps --at URL` | Every decision that met a value with no branch: the branches to write next. | | `pyrtan unanswered --at URL` | Every sentence no tree answered, with the tree that would: the trees to write next. `--scaffold DIR` writes them. | | `pyrtan new --from "…"` | Scaffolds a tree file from a sentence: numbers and hosts in it become slots, and the file passes `pyrtan check`. `pyrtan new NAME` scaffolds from a name. |+| `pyrtan search WORDS` | Finds trees in a catalog (default: the one at pyrtan.com). |+| `pyrtan pull NAME[@VERSION]` | Fetches a tree from a catalog into your trees folder, verified against the catalog's index. |+| `pyrtan index DIR` | Builds a catalog `index.json` from a directory of trees; what a catalog's CI runs. |+| `pyrtan console --out FILE` | Writes the console as one static HTML file, to host anywhere. |++`serve` and `hub` take `--token` (or `$PYRTAN_TOKEN`) to require+`Authorization: Bearer` on everything but `/health` and the console, and+`--cors ORIGIN` to let a browser at another origin call. `ask`, `gaps` and+`unanswered` take `--token` too.## A swarm@@ -138,9 +147,48 @@ pyrtan serve examples/hacker-news --port 8702 --hub http://127.0.0.1:8700pyrtan ask "top 3 stories on hacker news" ```-Open <http://127.0.0.1:8700/> for the demo page: a sentence box, the-answer, the output and the trace, and everything the swarm can answer. To-run the demo on a NAS or any Docker host, see [`deploy/README.md`](deploy/README.md).+Open <http://127.0.0.1:8700/> for the console: a sentence box, the answer,+the output and the trace; everything the swarm can answer; each peer and+its `link`; and the gaps. The same console lives at+<https://pyrtan.com/console/> and can be pointed at any hub started with+`--cors https://pyrtan.com`. To run the swarm on a NAS or any Docker host,+see [`deploy/README.md`](deploy/README.md).++## The catalog++Trees other people wrote are at <https://pyrtan.com/catalog/>, a directory+of static files anyone can mirror.++```+pyrtan search weather+pyrtan pull weather --dir trees+pyrtan serve trees+```++A pull checks the file's hash against the catalog's index, loads it as a+tree, and checks the name and version inside before writing it; a file you+have edited is not overwritten without `--force`. Point `--catalog` (or+`$PYRTAN_CATALOG`) at a directory or URL of your own to use a private+catalog; `pyrtan index DIR` builds one from any folder of trees. A tree in+a catalog says what others may do with it in a `license` line (an SPDX+identifier such as `MIT`).++## Leaving the LAN++Nothing authenticates by default. When a hub is reachable from outside:++```+pyrtan hub --host 0.0.0.0 --token "$(openssl rand -hex 24)" --cors https://pyrtan.com+pyrtan serve trees --hub http://hub:8700 --token … --gaps /var/lib/pyrtan/gaps.jsonl --egress public+```++A swarm shares one token: a service presents it when it registers, the hub+presents it when it forwards, and a stranger can neither join nor ask.+`--egress public` makes a service refuse any host that resolves to a+loopback, private, link-local or metadata address, on the first URL and on+every redirect, so a tree whose host comes from its input cannot be+pointed at your own network. Put TLS in front with whatever proxy you+already run; Pyrtan does not do it itself.A service and a hub answer the same requests, so a caller never needs to know which it is talking to. A swarm starts as one process holding every@@ -193,24 +241,27 @@ shows the same proposal and the same command whenever nothing answers.``` src/pyrtan/ the package: tree, intent, template, conditions, extract,- html, transport, runtime, trace, registry, grow, service,- hub, cli+ html, transport, runtime, trace, registry, gaps, grow,+ service, hub, page (the console), catalog, cliexamples/ trees that run against real public APIs, and offline against tests/fixtures/examples.json tests/ pytest; a swarm of two services behind a hub runs on localhost-_meta/ the design record and the tree format reference-deploy/ Dockerfile at the root; a compose stack for OpenMediaVault- and a systemd unit, with the steps for each+_meta/ the design record, the tree format reference, and the+ design for pyrtan.com+deploy/ Dockerfile at the root; a compose stack for OpenMediaVault,+ a systemd unit, and what to put in the pyrtan.com repositories``` ## Status-0.1.0. The core, the service, the hub and the command are built and tested,-and HTTP is the one form of I/O the core speaks. Not yet built: further-adapters (a browser for pages that need a script to render; files, shells,-serial lines, message queues), authentication between services, persistent-storage for gaps and unanswered sentences, and a tree marketplace. See-[`_meta/design.md`](_meta/design.md) for the sequencing.+0.2.0. The core, the service, the hub, the command, the console and the+catalog client are built and tested; HTTP is the one form of I/O the core+speaks; a swarm can require a token, keep its gaps, and refuse private+addresses. Not yet built: further adapters (a browser for pages that need+a script to render; files, shells, serial lines, message queues),+persistent storage for unanswered sentences, `pyrtan watch`, and hosting.+See [`_meta/design.md`](_meta/design.md) for the runtime's design and+[`_meta/platform.md`](_meta/platform.md) for pyrtan.com.Pyrtan is a product of [Hillvon.io](https://hillvon.io). diff --git a/_meta/design.md b/_meta/design.md index 1a142bb..6e97cad 100644 --- a/_meta/design.md +++ b/_meta/design.md@@ -2,7 +2,7 @@| | | |---|---|-| **Status** | **Built, v0.1.** The core, the service, the hub and the command exist and are tested (133 tests, including a two-service swarm behind a hub on localhost). No deployment, no users; HTTP is the only form of I/O the core speaks yet. |+| **Status** | **Built, v0.1; extended in v0.2.** The core, the service, the hub and the command exist and are tested (a two-service swarm behind a hub runs on localhost in the tests). HTTP is the only form of I/O the core speaks yet. v0.2 adds tokens, CORS, persistent gaps, an egress policy, the catalog client and the console; see [`platform.md`](platform.md). || **Date** | 2026-10-24 | | **Decision** | Pyrtan is an open-source automation tool: a core set of expandable services, each holding declarative decision trees that read and act on all forms of I/O, routed by a hub, with no model in the loop and a complete trace of every run. | | **Where** | This repository: `src/pyrtan`, `examples/`, `tests/`. AGPL-3.0-or-later. |@@ -215,10 +215,13 @@ service; nothing consumes it yet. Request/response over HTTP covers everycase the examples needed, and a bus would be a dependency and an operational component before there is a swarm large enough to need one.-**Persisting gaps and unanswered sentences.** Both live in a service's or-hub's memory and are lost on restart. Enough to prove the two loops (run,-see gap, add branch, reload; ask, see unanswered, scaffold, reload), not-enough for a fleet. Deferred to Section 7.+**Persisting gaps and unanswered sentences.** In v0.1 both lived in a+service's or hub's memory and were lost on restart: enough to prove the+two loops (run, see gap, add branch, reload; ask, see unanswered,+scaffold, reload), not enough for a fleet. v0.2 keeps gaps in a JSON-lines+file per service (`--gaps`), deduplicated with a count; the hub still+collects from peers rather than keeping its own. Unanswered sentences are+still in memory; the same store is the obvious next step.## 5. What the examples show@@ -236,11 +239,15 @@ Five trees, all against real public APIs, all tested offline:1. **v0.1, built.** Trees, checker, runtime, trace, service, hub, command, examples, tests. This document.-2. **v0.2: first real swarm.** Three or more services on separate boxes- behind one hub, with a status page off `/peers`. Gaps persisted to a- file per service and collected by the hub. A `pyrtan watch` that- re-runs a set of sentences on a schedule and reports when a path changes- (the way to learn a site has changed before a person does).+2. **v0.2, built: a swarm that can leave the LAN, and trees as a shared+ thing.** A shared token between hub and services, CORS for the console,+ gaps persisted to a file per service, an opt-in egress policy, the+ console with a peers view off `/peers`, and the catalog (`pyrtan+ search`, `pull`, `index`) with `version` pinning a pull. Design in+ [`platform.md`](platform.md). Still to come from the original v0.2: a+ `pyrtan watch` that re-runs a set of sentences on a schedule and+ reports when a path changes (the way to learn a site has changed before+ a person does).3. **v0.3: more forms of I/O.** First the browser: `observe` with `browser = { url, wait_for, actions }` on Playwright as an optional extra, for pages that need a script to render, with the same host rule@@ -248,19 +255,20 @@ Five trees, all against real public APIs, all tested offline:each a node source with its own allow-list (paths, commands, devices, topics), so a tree reads a log, runs a named command or polls a device by the same rules it reads a page, and the trace reads the same.-4. **v0.4: trees as a shared thing.** A directory of trees people can pull- by name, with the checker run on contribution, and `version` meaning- something.+4. **v0.4: the catalog grows up.** Namespaces, publishing through pull+ requests with the checker as the gate, and opt-in gap reporting from+ hubs back to a tree's author.5. **Beside the swarm, never inside it:** an authoring aid that reads gaps and drafts the branch for a person to accept. ## 7. Open questions-- **Authentication between services.** None in v0.1; the swarm is assumed- to sit on a trusted network behind an operator's ingress. The first- swarm that crosses a network boundary needs a shared secret or mTLS at- the hub, and `env` already carries per-service secrets for upstream APIs.-- **What a tree's `version` governs.** A hub could prefer the highest+- **Authentication between services.** v0.2 answers the first half: a+ shared bearer token, presented by services on registration and by the+ hub on every forward, with `/register` behind it. mTLS, and a token per+ service rather than per swarm, wait for a swarm that needs them.+- **What a tree's `version` governs.** In the catalog it is what a pull+ pins. On the hub it is still undecided: a hub could prefer the highestversion of a name across peers, or refuse to route to a peer whose version is below the caller's. Undecided until two services hold the same tree at different versions. diff --git a/_meta/platform.md b/_meta/platform.md new file mode 100644 index 0000000..3db1807 --- /dev/null +++ b/_meta/platform.md@@ -0,0 +1,162 @@+# Design — pyrtan.com: the console, the catalog, and hosting++| | |+|---|---|+| **Status** | **Phase 1 built, v0.2.** Tokens, CORS, persistent gaps, the egress policy, the catalog client and the console are in this repository and tested. The site and the catalog repository are described in [`../deploy/pyrtan.com/README.md`](../deploy/pyrtan.com/README.md). Hosting is designed, not built. |+| **Date** | 2026-10-25 |+| **Decision** | pyrtan.com is three separate things on one invariant: a static catalog of trees anyone can pull, a static console that talks only to the public hub API, and (later) paid hosting that runs the unmodified open-source hub per tenant. A caller cannot tell a hosted hub from a self-hosted one, so every piece works for both. |+| **Where** | This repository for the runtime, client and console. `pyrtan-trees` for the catalog. `pyrtan-site` for the pages. A private repository for the control plane, when there is one. |++## 0. The one invariant++[`design.md`](design.md) §2.6 says a caller cannot tell a service from a+hub, because both answer the same endpoints with the same shapes. This+design extends it one step: **a caller cannot tell a hosted Pyrtan from a+self-hosted one.** The console, the command, the catalog client and any+future integration speak only the public API (`/api`, `/health`,+`/capabilities`, `/peers`, `/intent`, `/run`, `/resume`, `/gaps`), with a+bearer token when the operator set one. Nothing hosted gets a private+endpoint.++The consequence for the business: what people pay for lives *around* the+runtime, not in it. The runtime stays whole and AGPL; the paid product is+provisioning, keeping, watching and billing.++## 1. What this design refuses++- **A fork of the hub for the paid product.** One codebase, one image. A+ tenant's hub is `pyrtan hub --token … --cors https://pyrtan.com` in a+ container, the same command a self-hoster runs on a NAS.+- **Billing or tenancy in `src/pyrtan`.** No `plan`, no `tenant_id`, no+ licence check. The control plane talks to hubs over HTTP like any client.+- **An account to read or pull trees.** The catalog is static files. A+ mirror is `cp -r`. A private catalog is the same files on a private host+ with `--catalog` pointed at it.+- **A console built only for the paid product.** The file at pyrtan.com is+ the file the hub serves at `/`. They cannot drift apart because they are+ the same bytes.+- **A model in the catalog's review.** Trees are accepted by `pyrtan check`+ and a replayed fixture, and read by a person.++## 2. The pieces++### 2.1 Access (`service.py`, `hub.py`; built)++`--token` (or `$PYRTAN_TOKEN`) on `serve` and `hub`. With a token, every+request except `GET /health`, the console and `/api` must carry+`Authorization: Bearer`. `/health` without the token answers status and+counts only. A swarm shares one token: a service presents it when it+registers, the hub presents it when it polls and forwards. `/register` is+behind the token, so a stranger cannot join a swarm.++`--cors ORIGIN` names the browser origins allowed to call (repeatable; `*`+for any). Off by default. The console at pyrtan.com needs+`--cors https://pyrtan.com` on the hub it talks to.++Neither does TLS. A hub that leaves the LAN sits behind a proxy that does.+mTLS between services is still open (design.md §7); a shared token is+enough for the first swarm that crosses a network boundary.++### 2.2 Gaps that persist (`gaps.py`; built)++`serve --gaps FILE` appends every gap to a JSON-lines file and reads it+back on start. The same gap seen twice is one entry with a `count`, first+and last seen: the list is what to write next, not a log. The hub's+`/gaps` still collects from peers; a store on the hub itself is deferred+until a swarm needs gaps to outlive its services.++### 2.3 Egress (`transport.py`; built)++`serve --egress public` refuses, before connecting, any URL whose host+resolves to a loopback, private, link-local, multicast, reserved or+unspecified address, and applies the same rule to every redirect. A tree+with `hosts = ["*"]` (the `site-status` example) is otherwise a way to+probe the host's own network. Self-hosters on a LAN leave it off; a host+that runs other people's trees turns it on and firewalls as well, because+the check resolves the name once and the client resolves it again.++### 2.4 The catalog (`catalog.py`; built)++A catalog is `index.json` plus the tree files it points to, on any static+host. `pyrtan index DIR` builds the index: one entry per tree with its+capability, licence, path, size and SHA-256. `pyrtan search WORDS` and+`pyrtan pull NAME[@VERSION]` read one; the default is+`https://pyrtan.com/catalog/`, overridden by `--catalog` or+`$PYRTAN_CATALOG`.++A pull fetches the file, checks its hash against the index, loads it as a+tree, and checks that the name and version inside match the entry before+writing it. An index entry that points outside the catalog (`..`, an+absolute path, another host) is refused. A file that exists locally and+differs is not overwritten without `--force`.++`version` now has one meaning the catalog relies on: it is the integer an+author bumps when the file changes in a way a puller should notice, and+`NAME@VERSION` pins it. Two services holding different versions of a name+is still the hub's open question.++A catalog tree carries a `license` (SPDX). The public catalog requires+one; `pyrtan index` leaves out a tree without one unless told otherwise.+Trees are data, not part of the program, so the AGPL does not reach them;+a declared licence is what lets a stranger use what they pulled.++### 2.5 The console (`page.py`; built)++One HTML file, no build, no external assets. Served by a hub or service+at `/` and `/console`, it talks to that origin. Written out by+`pyrtan console --out` and hosted anywhere, it finds out on load that its+own origin is not a Pyrtan (`/api` does not answer as one), shows a+connect box for a hub URL and a token, keeps them in local storage, and+sends the token as a bearer. A hub that requires a token and serves the+console itself shows the same box with only the token field.++Tabs: Ask (sentence, answer, output, trace, and a pointer when the run met+a gap), Trees (capabilities, with a link to the catalog), Peers (hub only;+`link` per peer), Gaps, API.++A browser will refuse a `http://` hub from an `https://` page. The console+says so and points at the hub's own `/`, which is the same console.++### 2.6 Hosting (designed)++A tenant gets a hub and one or more services as containers from this+repository's image, with a token, `--cors https://pyrtan.com`, `--gaps` on+a volume, `--egress public`, and TLS from the ingress. The control plane+(separate, private) does what the runtime does not: sign-up and teams,+provisioning, the secrets that reach trees as `env`, trace retention, a+scheduler that re-runs sentences and reports when a path changes (the+`pyrtan watch` of design.md §6), uptime from `link`, and billing.++Tenants are isolated by process and network, never by a flag in the+runtime. A tenant who leaves takes their trees, their gaps file and their+fixtures and runs the same commands on their own box.++## 3. Sequencing++1. **Phase 1 (this change).** Everything in §2.1–2.5, in the open-source+ repository. The site: pyrtan.com as static pages, `/console/` from+ `pyrtan console --out`, `/catalog/` from the catalog repository's CI.+ Hosting needs nothing yet.+2. **Phase 2.** Namespaces in the catalog (`someone/tree`), publishing+ through pull requests with the checker as the gate, and opt-in gap+ reporting: a hub that holds a catalog tree can send that tree's gaps to+ the catalog, so an author learns where reality disagreed with their+ tree across every hub that runs it. This needs the first small API.+3. **Phase 3.** The control plane: per-tenant hubs, secrets, hosted watch,+ retention, billing.++## 4. What to settle before Phase 2++Not engineering, and not legal advice:++- **A contributor licence agreement**, or a standing rule that nothing+ closed ever enters `src/pyrtan`. Without one of the two, contributed+ runtime code cannot be relicensed later. The second is this design's+ choice; a CLA keeps the first open.+- **A licence requirement for catalog trees.** This design requires an+ SPDX identifier and recommends permissive ones (MIT, Apache-2.0, CC0).+- **The trademark.** "Pyrtan" as a mark is what stops someone else hosting+ a lookalike under the name; the AGPL does not.+- **The word.** The code says *trees* and compares them to skills. The+ site should too; calling them skills sells the comparison and loses the+ claim that no model is in the loop.diff --git a/_meta/tree-format.md b/_meta/tree-format.md index ea344b6..18fa28c 100644 --- a/_meta/tree-format.md +++ b/_meta/tree-format.md@@ -12,6 +12,7 @@ A tree is one TOML file. This is the whole of it.| `hosts` | if any node speaks HTTP | The hosts the tree may reach: exact names, `*.example.com`, or `*` for a tree whose host comes from its input. | | `start` | yes | The first node. | | `tags` | no | Free strings, for people and for the hub's listing. |+| `license` | for a catalog | An SPDX identifier such as `MIT`, `Apache-2.0` or `CC0-1.0`: what others may do with the file. A tree without one runs anywhere but is left out of a catalog. || `triggers` | no | Sentences the tree answers to (below). | | `inputs` | no | What the tree needs (below). | | `nodes` | yes | The nodes (below). | diff --git a/deploy/README.md b/deploy/README.md index a1a16fb..a0edc85 100644 --- a/deploy/README.md +++ b/deploy/README.md@@ -51,8 +51,16 @@ tree you dropped into `trees/` before it is live:demo has met: `http://<nas-ip>:8700/gaps`. The services do not expose their own ports; everything goes through the-hub on 8700. Nothing authenticates: this is a demo for your LAN. Do not-forward 8700 to the internet.+hub on 8700. As shipped nothing authenticates: this is a demo for your+LAN. Before forwarding 8700 anywhere, set a token: uncomment+`PYRTAN_TOKEN` in the compose file's `x-pyrtan` block (every container+reads it, so the swarm shares it), put TLS in front, and open+`https://pyrtan.com/console/` with the hub's URL and the token; the+compose file already passes `--cors https://pyrtan.com` to the hub. The+`mine` service keeps its gaps in `trees/.gaps.jsonl` so they survive a+restart; the container runs as uid 10001, so `chown 10001 trees` (or+`chmod o+w trees`) on the NAS, else the gaps stay in memory and `/health`+reports `gaps_file_error`.### Without the plugin diff --git a/deploy/openmediavault/docker-compose.yml b/deploy/openmediavault/docker-compose.yml index 2fad1c3..c36020e 100644 --- a/deploy/openmediavault/docker-compose.yml +++ b/deploy/openmediavault/docker-compose.yml@@ -27,12 +27,20 @@ x-pyrtan: &pyrtanlogging: driver: json-file options: { max-size: "5m", max-file: "3" }+ environment: &env+ # Uncomment to require a bearer token on every request but /health and+ # the console. Every container reads it, so the swarm shares it. Do+ # this before the hub is reachable from outside your LAN.+ # PYRTAN_TOKEN: "change-me-to-something-long"+ PYRTAN_EGRESS: anyservices: hub: <<: *pyrtan- command: ["hub", "--host", "0.0.0.0", "--port", "8700", "--poll", "10"]+ # --cors lets the console at pyrtan.com talk to this hub from a browser.+ command: ["hub", "--host", "0.0.0.0", "--port", "8700", "--poll", "10", "--cors", "https://pyrtan.com"]environment:+ <<: *envPYRTAN_PORT: "8700" ports: - "8700:8700"@@ -50,9 +58,11 @@ services:# Your own trees. Edit files in this folder from a share; the service # reloads them without a restart. `pyrtan check` them first: # docker compose -p pyrtan exec mine pyrtan check /trees+ # Gaps are kept in /trees/.gaps.jsonl (the one file the service writes),+ # so the list of branches to write survives a restart.mine: <<: *pyrtan- command: ["serve", "/trees", "--host", "0.0.0.0", "--port", "8701", "--name", "mine", "--hub", "http://hub:8700", "--advertise", "http://mine:8701"]+ command: ["serve", "/trees", "--host", "0.0.0.0", "--port", "8701", "--name", "mine", "--hub", "http://hub:8700", "--advertise", "http://mine:8701", "--gaps", "/trees/.gaps.jsonl"]volumes:- - /srv/dev-disk-by-uuid-CHANGE-ME/pyrtan/trees:/trees:ro+ - /srv/dev-disk-by-uuid-CHANGE-ME/pyrtan/trees:/treesdepends_on: [hub] diff --git a/deploy/pyrtan.com/README.md b/deploy/pyrtan.com/README.md new file mode 100644 index 0000000..7cc0fed --- /dev/null +++ b/deploy/pyrtan.com/README.md@@ -0,0 +1,162 @@+# Putting up pyrtan.com++pyrtan.com is static files: a landing page, the console, and the catalog.+Nothing runs server-side until there is paid hosting, so the whole site is+two GitHub repositories deployed to GitHub Pages (or Cloudflare Pages; the+files are the same). The design is in+[`_meta/platform.md`](../../_meta/platform.md).++What to create, in order:++| | Repository | Serves | From |+| :-- | :-- | :-- | :-- |+| 1 | `b-i-t-8/pyrtan-trees` | `pyrtan.com/catalog/` | its own `trees/` folder, checked and indexed by CI |+| 2 | `b-i-t-8/pyrtan-site` | `pyrtan.com/` and `pyrtan.com/console/` | `index.html` here, and `pyrtan console --out` |++Both can be one repository if you prefer; two keeps "I wrote a tree" pull+requests apart from "I changed the site" ones, and lets the catalog's+contributors never touch the site.++Files in this folder, ready to copy:++- [`catalog.yml`](catalog.yml): the workflow for the trees repository.+- [`site.yml`](site.yml): the workflow for the site repository.+- [`index.html`](index.html): the landing page.+- [`trees-README.md`](trees-README.md): the README for the trees repository.++## 1. The catalog repository (`pyrtan-trees`)++```+pyrtan-trees/+ README.md from trees-README.md here+ LICENSE what the *repository's* own files are under; each tree declares its own+ .github/workflows/catalog.yml+ trees/+ weather/+ weather.toml license = "MIT" in the file+ fixture.json recorded responses, from pyrtan run --record+ hacker-news/+ hacker-news.toml+ hacker-news-item.toml+ fixture.json+ tests/+ test_trees.py replays each tree against its fixture+```++Steps:++1. Create the repository, empty.+2. Copy `trees-README.md` to `README.md` and `catalog.yml` to+ `.github/workflows/catalog.yml`.+3. Seed `trees/` from this repository's `examples/` (each already carries+ `license = "MIT"`), and `tests/fixtures/examples.json` as the fixtures,+ split per folder or kept as one file; the workflow only needs `pyrtan+ check` and `pytest` to pass.+4. In the repository's *Settings › Pages*, set *Source* to **GitHub+ Actions**.+5. Push. The workflow runs `pyrtan check trees`, the tests, then+ `pyrtan index trees --out site/index.json`, copies the tree files next+ to it, and publishes `site/` to Pages. The result is at+ `https://b-i-t-8.github.io/pyrtan-trees/`.++What the workflow enforces on every pull request: every tree loads and+checks clean, every tree has a `license`, no two trees share a name, and+the tests pass. `pyrtan index` exits 1 on any of those, which fails the+check. A reviewer reads the tree; nothing else is needed to accept one.++Bump `version` in the tree when a change should be noticed by people who+pulled it; `pyrtan pull NAME@VERSION` pins one.++## 2. The site repository (`pyrtan-site`)++```+pyrtan-site/+ index.html from index.html here+ CNAME "pyrtan.com"+ .github/workflows/site.yml+```++Steps:++1. Create the repository; copy `index.html` and `site.yml` (to+ `.github/workflows/site.yml`); add a `CNAME` file containing+ `pyrtan.com`.+2. *Settings › Pages*: *Source* **GitHub Actions**; *Custom domain*+ `pyrtan.com`; tick *Enforce HTTPS* once DNS resolves.+3. Push. The workflow installs `pyrtan` from this repository (pinned to a+ tag; update the tag to ship a new console), writes+ `console/index.html` with `pyrtan console --out`, and publishes.++### DNS++At your DNS provider, for `pyrtan.com`:++| Record | Name | Value |+| :-- | :-- | :-- |+| `A` | `@` | `185.199.108.153`, `185.199.109.153`, `185.199.110.153`, `185.199.111.153` |+| `AAAA` | `@` | `2606:50c0:8000::153`, `2606:50c0:8001::153`, `2606:50c0:8002::153`, `2606:50c0:8003::153` |+| `CNAME` | `www` | `b-i-t-8.github.io` |++(GitHub's current addresses; check *Pages › Custom domain* in the+repository settings if they have changed.)++### `/catalog/` on the same domain++The catalog is published from the other repository, at+`b-i-t-8.github.io/pyrtan-trees/`. The command's default is+`https://pyrtan.com/catalog/`, so the catalog must answer there. Two ways:++- **Proxy it (Cloudflare).** If pyrtan.com's DNS is on Cloudflare, add a+ *Worker route* or an *Origin rule* that serves+ `pyrtan.com/catalog/*` from `b-i-t-8.github.io/pyrtan-trees/*`. A+ ten-line Worker does it:++ ```js+ export default {+ async fetch(req) {+ const u = new URL(req.url);+ if (!u.pathname.startsWith("/catalog/")) return fetch(req);+ const up = "https://b-i-t-8.github.io/pyrtan-trees/" + u.pathname.slice("/catalog/".length);+ const r = await fetch(up, { headers: { "User-Agent": "pyrtan.com" } });+ return new Response(r.body, { status: r.status, headers: { "Content-Type": r.headers.get("Content-Type") || "application/octet-stream", "Cache-Control": "public, max-age=300", "Access-Control-Allow-Origin": "*" } });+ },+ };+ ```++- **Or publish the catalog into the site.** Have `site.yml` check out+ `pyrtan-trees` too, run `pyrtan index` there, and copy the output to+ `catalog/` in the site's artifact; then one Pages deployment holds+ everything, at the cost of the site rebuilding when a tree merges. The+ commented block in `site.yml` shows this.++Either way, `curl https://pyrtan.com/catalog/index.json` should return the+index, and `pyrtan search weather` from anywhere should find it.++### `/console/`++Works as soon as the site is up. A visitor enters a hub URL and a token;+the hub must have been started with `--cors https://pyrtan.com`. A hub on+a LAN over plain `http://` cannot be reached from an `https://` page (the+browser blocks it); the console says so and points at the hub's own `/`,+which is the same console served locally.++## 3. Later: hosting++Nothing here needs to change. A tenant's hub is this repository's image+started with `--token`, `--cors https://pyrtan.com`, `--gaps` on a volume+and `--egress public`, behind TLS; the console at pyrtan.com connects to it+like to any other. The control plane that provisions it is a separate,+private repository, per `_meta/platform.md` §2.6.++## Checklist++- [ ] `pyrtan-trees` created; Pages source = Actions; first run green;+ `…github.io/pyrtan-trees/index.json` answers.+- [ ] `pyrtan-site` created with `CNAME`; Pages custom domain set; DNS+ records in; HTTPS enforced.+- [ ] `https://pyrtan.com/` shows the landing page; `/console/` loads.+- [ ] `/catalog/index.json` answers on pyrtan.com (proxy or merged build).+- [ ] From a laptop: `pyrtan search weather` finds it; `pyrtan pull weather`+ writes `trees/weather.toml`.+- [ ] A hub started with `--token … --cors https://pyrtan.com` connects+ from `https://pyrtan.com/console/`.diff --git a/deploy/pyrtan.com/catalog.yml b/deploy/pyrtan.com/catalog.yml new file mode 100644 index 0000000..91bf3d9 --- /dev/null +++ b/deploy/pyrtan.com/catalog.yml@@ -0,0 +1,65 @@+# The catalog repository's workflow (pyrtan-trees/.github/workflows/catalog.yml).+#+# On every push and pull request: every tree must load, check clean, carry+# a license and have a unique name, and the tests must pass. On a push to+# main: build index.json, put the tree files next to it, publish to Pages.+#+# Pin PYRTAN_REF to a tag of b-i-t-8/pyrtan; bump it to adopt a newer+# checker or index format.++name: catalog++on:+ push:+ branches: [main]+ pull_request:++env:+ PYRTAN_REF: v0.2.0++permissions:+ contents: read++jobs:+ check:+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v4+ - uses: astral-sh/setup-uv@v5+ - name: Install pyrtan+ run: uv tool install --python 3.12 "git+https://github.com/b-i-t-8/pyrtan@${PYRTAN_REF}"+ - name: Check every tree+ run: pyrtan check trees+ - name: Build the index (fails on a tree with no license or a duplicate name)+ run: |+ mkdir -p site+ pyrtan index trees --out site/index.json+ - name: Replay the tests+ if: hashFiles('tests/**') != ''+ run: |+ uv venv --python 3.12 .venv+ uv pip install --python .venv/bin/python pytest "git+https://github.com/b-i-t-8/pyrtan@${PYRTAN_REF}"+ .venv/bin/python -m pytest -q+ - name: Lay out the catalog+ run: |+ # index.json plus every tree file at the path the index names.+ (cd trees && find . -name '*.toml' -not -name 'pyproject.toml' | cpio -pdm ../site)+ cp -n README.md site/ 2>/dev/null || true+ ls -R site | head -50+ - uses: actions/upload-pages-artifact@v3+ with:+ path: site++ publish:+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'+ needs: check+ runs-on: ubuntu-latest+ permissions:+ pages: write+ id-token: write+ environment:+ name: github-pages+ url: ${{ steps.deployment.outputs.page_url }}+ steps:+ - id: deployment+ uses: actions/deploy-pages@v4diff --git a/deploy/pyrtan.com/index.html b/deploy/pyrtan.com/index.html new file mode 100644 index 0000000..3aac98b --- /dev/null +++ b/deploy/pyrtan.com/index.html@@ -0,0 +1,75 @@+<!doctype html>+<html lang="en">+<head>+<meta charset="utf-8">+<meta name="viewport" content="width=device-width, initial-scale=1">+<title>Pyrtan</title>+<meta name="description" content="Open-source total automation: a core set of expandable services that act on all forms of I/O through hardcoded, inspectable decision trees. No model in the loop.">+<style>+ :root { --bg:#fbfaf7; --fg:#1d1d1b; --mute:#6b6a64; --line:#e3e1da; --card:#ffffff; --accent:#2b5db8; --accent-fg:#fff; --soft:#f1efe9 }+ @media (prefers-color-scheme: dark) { :root { --bg:#141513; --fg:#ecebe6; --mute:#9d9c95; --line:#2b2c29; --card:#1c1d1b; --accent:#7fa6f0; --accent-fg:#101318; --soft:#222420 } }+ * { box-sizing:border-box }+ body { margin:0; background:var(--bg); color:var(--fg); font:17px/1.55 system-ui,-apple-system,"Segoe UI",sans-serif }+ main { max-width:820px; margin:0 auto; padding:48px 16px 64px }+ h1 { font-size:34px; margin:0 0 8px; letter-spacing:-.01em }+ p.lede { font-size:20px; color:var(--mute); margin:0 0 28px }+ h2 { font-size:19px; margin:36px 0 10px }+ .actions { display:flex; flex-wrap:wrap; gap:10px; margin:0 0 8px }+ .actions a { display:inline-block; padding:10px 16px; border-radius:8px; text-decoration:none; font-weight:600; border:1px solid var(--line); background:var(--card); color:var(--fg) }+ .actions a.primary { background:var(--accent); color:var(--accent-fg); border-color:var(--accent) }+ pre { padding:12px 14px; background:var(--card); border:1px solid var(--line); border-radius:10px; overflow:auto; font:13.5px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace; white-space:pre-wrap }+ code { font:14px ui-monospace,SFMono-Regular,Menlo,monospace; background:var(--soft); padding:1px 5px; border-radius:4px }+ ul { padding-left:20px } li { margin:4px 0 }+ .grid { display:grid; grid-template-columns:repeat(auto-fit,minmax(220px,1fr)); gap:12px }+ .card { background:var(--card); border:1px solid var(--line); border-radius:10px; padding:14px 16px }+ .card h3 { margin:0 0 6px; font-size:16px } .card p { margin:0; color:var(--mute); font-size:15px }+ a { color:var(--accent) }+ footer { color:var(--mute); font-size:14px; margin-top:48px; border-top:1px solid var(--line); padding-top:16px }+</style>+</head>+<body>+<main>+ <h1>Pyrtan</h1>+ <p class="lede">Open-source total automation: a core set of expandable services that act on all forms of I/O through hardcoded, inspectable decision trees. It answers like an assistant. Inside, there is no model.</p>++ <div class="actions">+ <a class="primary" href="/console/">Open the console</a>+ <a href="/catalog/">Browse the catalog</a>+ <a href="https://github.com/b-i-t-8/pyrtan">Source on GitHub</a>+ </div>++<pre>$ pyrtan run "what's the weather in Oslo" --trees examples+trace: weather+ geocode (observe): GET https://geocoding-api.open-meteo.com/v1/search?name=Oslo… -> 200 -> found+ found (decide) [exists True]: state.hit = {…} -> forecast+ forecast (observe): GET https://api.open-meteo.com/v1/forecast?latitude=59.91… -> 200 -> have-temp+ sky (decide) [between [51, 67]]: state.code = 61 -> rain+ reply (done): Oslo, Norway: 4.2°C, raining, humidity 80%, wind 12.3 km/h.</pre>++ <h2>The trade</h2>+ <p>A model generalises to sentences nobody wrote a tree for; Pyrtan does not, and says so. In return every run is deterministic, costs nothing, works offline against recorded responses, cannot reach a host its author did not list, and can always explain itself: the trace names each observation and the condition that chose each branch.</p>++ <div class="grid">+ <div class="card"><h3>Trees</h3><p>One TOML file: the sentences it answers to, the hosts it may touch, what it observes, how it decides. No code, no loops, no model.</p></div>+ <div class="card"><h3>Swarm</h3><p>Services hold trees; a hub routes between them and shows each peer's <code>link</code>. One process or many, the callers never change.</p></div>+ <div class="card"><h3>Gaps</h3><p>A decision that meets a value no branch covers is recorded, not guessed at. The list of gaps is the list of what to write next.</p></div>+ <div class="card"><h3>Catalog</h3><p>Trees others wrote, as static files anyone can mirror. <code>pyrtan pull NAME</code> verifies the file against the index before writing it.</p></div>+ </div>++ <h2>Install</h2>+<pre>uv tool install git+https://github.com/b-i-t-8/pyrtan # or: pip install git+https://github.com/b-i-t-8/pyrtan+pyrtan search weather+pyrtan pull weather --dir trees+pyrtan serve trees --port 8701 # then open http://127.0.0.1:8701/</pre>+ <p>Python 3.12 or later, no runtime dependencies. Runs on a laptop, a NAS or a container; <a href="https://github.com/b-i-t-8/pyrtan/blob/main/deploy/README.md">deployment notes</a>.</p>++ <h2>The console</h2>+ <p>Every hub and service serves the console at <code>/</code>. The same console is at <a href="/console/">pyrtan.com/console</a>: point it at your own hub, started with <code>--token</code> and <code>--cors https://pyrtan.com</code>, and ask it from anywhere. Nothing about your swarm passes through this site; the page talks to your hub directly.</p>++ <h2>Hosting</h2>+ <p>Self-hosting is the product. A hosted option, running the same open-source hub per customer with secrets, retention, scheduled re-runs and alerts around it, is planned; it will speak the same API, so a swarm can move either way without changing a caller.</p>++ <footer>Pyrtan is free software under the AGPL-3.0-or-later, a product of <a href="https://hillvon.io">Hillvon.io</a>. Trees you write are your own.</footer>+</main>+</body>+</html>diff --git a/deploy/pyrtan.com/site.yml b/deploy/pyrtan.com/site.yml new file mode 100644 index 0000000..451bbd4 --- /dev/null +++ b/deploy/pyrtan.com/site.yml@@ -0,0 +1,66 @@+# The site repository's workflow (pyrtan-site/.github/workflows/site.yml).+#+# Publishes index.html, CNAME and the console (written by `pyrtan console+# --out` from a pinned release of b-i-t-8/pyrtan) to GitHub Pages at+# pyrtan.com. Bump PYRTAN_REF to ship a newer console.++name: site++on:+ push:+ branches: [main]+ workflow_dispatch:++env:+ PYRTAN_REF: v0.2.0++permissions:+ contents: read+ pages: write+ id-token: write++concurrency:+ group: pages+ cancel-in-progress: true++jobs:+ build:+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v4+ - uses: astral-sh/setup-uv@v5+ - name: Install pyrtan+ run: uv tool install --python 3.12 "git+https://github.com/b-i-t-8/pyrtan@${PYRTAN_REF}"+ - name: Assemble the site+ run: |+ mkdir -p site/console+ cp index.html CNAME site/+ pyrtan console --out site/console/index.html+ touch site/.nojekyll++ # To serve the catalog from this same deployment instead of proxying+ # b-i-t-8.github.io/pyrtan-trees, uncomment this block (and give the+ # job `contents: read` on that repository if it is private).+ # - uses: actions/checkout@v4+ # with:+ # repository: b-i-t-8/pyrtan-trees+ # path: trees-repo+ # - name: Build the catalog into /catalog+ # run: |+ # mkdir -p site/catalog+ # pyrtan index trees-repo/trees --out site/catalog/index.json+ # (cd trees-repo/trees && find . -name '*.toml' | cpio -pdm ../../site/catalog)++ - uses: actions/upload-pages-artifact@v3+ with:+ path: site++ deploy:+ needs: build+ runs-on: ubuntu-latest+ environment:+ name: github-pages+ url: ${{ steps.deployment.outputs.page_url }}+ steps:+ - id: deployment+ uses: actions/deploy-pages@v4diff --git a/deploy/pyrtan.com/trees-README.md b/deploy/pyrtan.com/trees-README.md new file mode 100644 index 0000000..29ef6a4 --- /dev/null +++ b/deploy/pyrtan.com/trees-README.md@@ -0,0 +1,46 @@+# Pyrtan trees++The catalog behind `pyrtan search` and `pyrtan pull`: trees people wrote+for [Pyrtan](https://github.com/b-i-t-8/pyrtan), checked by CI and+published as static files at <https://pyrtan.com/catalog/>.++A tree is one TOML file that says which sentences it answers to, which+hosts it may touch, what it observes, and exactly how it decides. No model+is involved; see the Pyrtan README for the format.++## Using a tree++```+pyrtan search weather+pyrtan pull weather --dir trees+pyrtan serve trees+```++A pull verifies the file against this catalog's index before writing it.++## Adding a tree++1. Fork, then `pyrtan new my-tree --dir trees/my-tree` and write it. Keep+ `hosts` to exactly what it needs, and set `license` (an SPDX+ identifier; `MIT`, `Apache-2.0` or `CC0-1.0`). CI refuses a tree+ without one.+2. `pyrtan check trees/my-tree` until it is clean.+3. Record its responses for the tests:+ `pyrtan run "the sentence" --trees trees/my-tree --record trees/my-tree/fixture.json`.+ Trim the recording to what the test needs; never commit a credential.+4. Add a test under `tests/` that replays the sentence against the fixture+ and asserts the `say`, the `output` and the branches taken.+5. Open a pull request. CI runs the checker, the index build and the+ tests; a maintainer reads the tree.++To change a tree others may have pulled, bump its `version`.++## Mirroring++The catalog is `index.json` and the files it names. Copy the published+site anywhere and point `pyrtan --catalog` (or `$PYRTAN_CATALOG`) at it.++## Licences++Each tree carries its own licence in the file. The repository's own files+(this README, the workflow, the tests) are under the licence in `LICENSE`.diff --git a/examples/github/github-release.toml b/examples/github/github-release.toml index 1ce20ef..d32f374 100644 --- a/examples/github/github-release.toml +++ b/examples/github/github-release.toml@@ -5,6 +5,7 @@ name = "github-release"version = 1 description = "The latest release of a GitHub repository." tags = ["github", "read-only"]+license = "MIT"hosts = ["api.github.com"] start = "fetch" diff --git a/examples/hacker-news/hacker-news-item.toml b/examples/hacker-news/hacker-news-item.toml index 19e50d5..eadbb09 100644 --- a/examples/hacker-news/hacker-news-item.toml +++ b/examples/hacker-news/hacker-news-item.toml@@ -2,6 +2,7 @@ name = "hacker-news-item"version = 1 description = "One Hacker News item by id: title, url, score, author." tags = ["news", "read-only"]+license = "MIT"hosts = ["hacker-news.firebaseio.com"] start = "fetch" diff --git a/examples/hacker-news/hacker-news.toml b/examples/hacker-news/hacker-news.toml index 37ba031..826c223 100644 --- a/examples/hacker-news/hacker-news.toml +++ b/examples/hacker-news/hacker-news.toml@@ -6,6 +6,7 @@ name = "hacker-news"version = 1 description = "Top stories on Hacker News, from its public API." tags = ["news", "read-only"]+license = "MIT"hosts = ["hacker-news.firebaseio.com"] start = "fetch-ids" diff --git a/examples/site-status/site-status.toml b/examples/site-status/site-status.toml index 49a59c3..52a57f7 100644 --- a/examples/site-status/site-status.toml +++ b/examples/site-status/site-status.toml@@ -6,6 +6,7 @@ name = "site-status"version = 1 description = "Whether a web site answers, and how quickly." tags = ["ops", "read-only"]+license = "MIT"hosts = ["*"] start = "probe" diff --git a/examples/weather/weather.toml b/examples/weather/weather.toml index 032d624..4c9ccd6 100644 --- a/examples/weather/weather.toml +++ b/examples/weather/weather.toml@@ -6,6 +6,7 @@ name = "weather"version = 1 description = "Current weather for a place, from Open-Meteo." tags = ["weather", "read-only"]+license = "MIT"hosts = ["geocoding-api.open-meteo.com", "api.open-meteo.com"] start = "geocode" diff --git a/pyproject.toml b/pyproject.toml index 5f22a04..115dfed 100644 --- a/pyproject.toml +++ b/pyproject.toml@@ -1,6 +1,6 @@[project] name = "pyrtan"-version = "0.1.0"+version = "0.2.0"description = "Open-source total automation: a core set of expandable services that act on all forms of I/O through hardcoded, inspectable decision trees." readme = "README.md" license = "AGPL-3.0-or-later" diff --git a/src/pyrtan/__init__.py b/src/pyrtan/__init__.py index 038ac85..ee50ff2 100644 --- a/src/pyrtan/__init__.py +++ b/src/pyrtan/__init__.py@@ -33,4 +33,4 @@ __all__ = ["Response", ]-__version__ = "0.1.0"+__version__ = "0.2.0"diff --git a/src/pyrtan/catalog.py b/src/pyrtan/catalog.py new file mode 100644 index 0000000..2cd8bcb --- /dev/null +++ b/src/pyrtan/catalog.py@@ -0,0 +1,251 @@+"""The catalog: a directory of trees people can pull by name.++A catalog is static files behind any web server (or a directory on disk):+an ``index.json`` and the tree files it points to. There is no account, no+API and no server-side code; a mirror is a copy of the files, and a+private catalog is the same files on a private host. The one at+``https://pyrtan.com/catalog/`` is built from a public repository whose CI+runs ``pyrtan check`` on every tree and this module's :func:`build_index`+on every merge.++``index.json``::++ {"pyrtan-catalog": 1, "generated": "2026-10-25T12:00:00Z",+ "trees": [{"name": "weather", "version": 3, "description": "...",+ "tags": [...], "hosts": [...], "triggers": [...],+ "license": "MIT", "path": "weather/weather.toml",+ "sha256": "...", "size": 1234}]}++``path`` is relative to the index. A pull fetches the file, checks its+``sha256`` against the index, loads it as a tree, checks that the name and+version inside match the entry, and only then writes it. A catalog cannot+hand out a file other than the one its index describes, and a tree a+service loads is still checked like any other: the catalog is a place to+find trees, not a channel that bypasses the checker.++Not confused with :class:`pyrtan.registry.Registry`, which is the set of+trees one running service holds. A catalog is where a registry's files+may have come from.+"""++from __future__ import annotations++import datetime as _dt+import hashlib+import json+import os+import re+import urllib.error+import urllib.parse+import urllib.request+from dataclasses import dataclass+from pathlib import Path+from typing import Any++from .tree import Tree, TreeError, check, load_tree, load_tree_text++DEFAULT_CATALOG = "https://pyrtan.com/catalog/"+INDEX = "index.json"+FORMAT = 1+NAME_AT_VERSION = re.compile(r"^([a-z0-9][a-z0-9-]*)(?:@(\d+))?$")+++class CatalogError(RuntimeError):+ pass+++def catalog_url() -> str:+ """The catalog to use when none is given: ``$PYRTAN_CATALOG`` or the public one."""+ return os.environ.get("PYRTAN_CATALOG") or DEFAULT_CATALOG+++# ---------------------------------------------------------------- building+++def sha256(data: bytes) -> str:+ return hashlib.sha256(data).hexdigest()+++def entry_for(tree: Tree, data: bytes, path: str) -> dict[str, Any]:+ """One index entry: the tree's capability plus where its file is and what it hashes to."""+ cap = tree.capability()+ return {+ "name": tree.name,+ "version": tree.version,+ "description": tree.description,+ "tags": tree.tags,+ "hosts": tree.hosts,+ "triggers": cap["triggers"],+ "inputs": cap["inputs"],+ "license": tree.license,+ "path": path,+ "sha256": sha256(data),+ "size": len(data),+ }+++@dataclass+class Problem:+ path: str+ message: str++ def __str__(self) -> str:+ return f"{self.path}: {self.message}"+++def build_index(root: str | Path, *, require_license: bool = True) -> tuple[dict[str, Any], list[Problem]]:+ """Walk ``root`` for tree files and describe each in an index.++ Returns the index and the problems that kept a file out of it: a file+ that does not load, two files with the same name, or (when required) a+ tree with no ``license``. The caller decides whether problems fail the+ build; the catalog's CI does.+ """+ root = Path(root)+ trees: list[dict[str, Any]] = []+ problems: list[Problem] = []+ seen: dict[str, str] = {}+ for path in sorted(root.rglob("*.toml")):+ if path.name == "pyproject.toml" or any(part.startswith(".") for part in path.relative_to(root).parts):+ continue+ rel = path.relative_to(root).as_posix()+ try:+ tree = load_tree(path)+ except TreeError as e:+ problems.append(Problem(rel, str(e)))+ continue+ errors = [p for p in check(tree) if p.level == "error"]+ if errors:+ problems.append(Problem(rel, "; ".join(f"{p.where}: {p.message}" for p in errors)))+ continue+ if tree.name in seen:+ problems.append(Problem(rel, f"a tree named {tree.name!r} is already at {seen[tree.name]}"))+ continue+ if require_license and not tree.license:+ problems.append(Problem(rel, "no 'license': a catalog tree must say what others may do with it"))+ continue+ seen[tree.name] = rel+ trees.append(entry_for(tree, path.read_bytes(), rel))+ index = {+ "pyrtan-catalog": FORMAT,+ "generated": _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),+ "count": len(trees),+ "trees": trees,+ }+ return index, problems+++# ---------------------------------------------------------------- reading+++def _fetch(url: str, timeout: float = 20.0) -> bytes:+ parts = urllib.parse.urlsplit(url)+ if parts.scheme == "file" or not parts.scheme:+ path = Path(urllib.request.url2pathname(parts.path) if parts.scheme else url)+ try:+ return path.read_bytes()+ except OSError as e:+ raise CatalogError(f"{url}: {e}") from e+ if parts.scheme not in ("http", "https"):+ raise CatalogError(f"{url}: a catalog is reached over https (or a file: path)")+ req = urllib.request.Request(url, headers={"User-Agent": "pyrtan-catalog/0.2", "Accept": "application/json, application/toml, text/plain, */*"})+ try:+ with urllib.request.urlopen(req, timeout=timeout) as resp:+ return resp.read()+ except urllib.error.HTTPError as e:+ raise CatalogError(f"{url}: {e.code}") from e+ except (urllib.error.URLError, OSError) as e:+ raise CatalogError(f"{url}: {e}") from e+++def _join(base: str, rel: str) -> str:+ base = base if base.endswith("/") else base + "/"+ if urllib.parse.urlsplit(base).scheme in ("http", "https", "file"):+ if "://" in rel or rel.startswith("/") or ".." in rel.split("/"):+ raise CatalogError(f"index points outside the catalog: {rel!r}")+ return urllib.parse.urljoin(base, rel)+ return str(Path(base) / rel)+++class Catalog:+ """A catalog at a URL (or a directory), read lazily."""++ def __init__(self, base: str | None = None) -> None:+ self.base = (base or catalog_url()).rstrip("/") + "/"+ self._index: dict[str, Any] | None = None++ @property+ def index(self) -> dict[str, Any]:+ if self._index is None:+ raw = _fetch(_join(self.base, INDEX))+ try:+ index = json.loads(raw.decode("utf-8"))+ except (json.JSONDecodeError, UnicodeDecodeError) as e:+ raise CatalogError(f"{self.base}{INDEX}: not JSON: {e}") from e+ if not isinstance(index, dict) or index.get("pyrtan-catalog") != FORMAT or not isinstance(index.get("trees"), list):+ raise CatalogError(f"{self.base}{INDEX}: not a pyrtan catalog index (format {FORMAT})")+ self._index = index+ return self._index++ def entries(self) -> list[dict[str, Any]]:+ return [e for e in self.index["trees"] if isinstance(e, dict) and isinstance(e.get("name"), str)]++ def search(self, query: str = "") -> list[dict[str, Any]]:+ """Entries whose name, description, tags, hosts or triggers mention+ every word of ``query``. An empty query lists everything."""+ words = [w.lower() for w in query.split()]+ out: list[dict[str, Any]] = []+ for e in self.entries():+ hay = " ".join([e["name"], str(e.get("description", "")), " ".join(e.get("tags", [])), " ".join(e.get("hosts", [])), " ".join(e.get("triggers", []))]).lower()+ if all(w in hay for w in words):+ out.append(e)+ return out++ def find(self, spec: str) -> dict[str, Any]:+ """The entry for ``name`` or ``name@version``."""+ m = NAME_AT_VERSION.match(spec)+ if not m:+ raise CatalogError(f"{spec!r}: expected NAME or NAME@VERSION")+ name, version = m.group(1), m.group(2)+ for e in self.entries():+ if e["name"] == name and (version is None or e.get("version") == int(version)):+ return e+ have = [e for e in self.entries() if e["name"] == name]+ if have and version is not None:+ raise CatalogError(f"{name}: the catalog has version {have[0].get('version')}, not {version}")+ raise CatalogError(f"{name!r} is not in the catalog at {self.base}")++ def fetch(self, entry: dict[str, Any]) -> tuple[Tree, bytes]:+ """The tree file an entry describes, verified against the entry."""+ rel = entry.get("path")+ if not isinstance(rel, str) or not rel:+ raise CatalogError(f"{entry['name']}: the index entry has no path")+ data = _fetch(_join(self.base, rel))+ digest = sha256(data)+ if entry.get("sha256") and digest != entry["sha256"]:+ raise CatalogError(f"{entry['name']}: the file at {rel} does not match the index (sha256 {digest[:12]}… vs {str(entry['sha256'])[:12]}…); refusing it")+ try:+ tree = load_tree_text(data.decode("utf-8"), source=f"{self.base}{rel}")+ except (TreeError, UnicodeDecodeError) as e:+ raise CatalogError(f"{entry['name']}: the file does not load as a tree: {e}") from e+ if tree.name != entry["name"]:+ raise CatalogError(f"{entry['name']}: the file says it is {tree.name!r}; refusing it")+ if entry.get("version") is not None and tree.version != entry["version"]:+ raise CatalogError(f"{entry['name']}: the file is version {tree.version}, the index says {entry['version']}; refusing it")+ return tree, data++ def pull(self, spec: str, into: str | Path, *, force: bool = False) -> Path:+ """Fetch ``spec`` and write it as ``into/NAME.toml``. Refuses to+ overwrite a file unless ``force``; a tree someone edited locally is+ theirs."""+ entry = self.find(spec)+ tree, data = self.fetch(entry)+ into = Path(into)+ target = into / f"{tree.name}.toml"+ if target.exists() and not force:+ if target.read_bytes() == data:+ return target+ raise CatalogError(f"{target} exists and differs from the catalog's; pass --force to replace it")+ into.mkdir(parents=True, exist_ok=True)+ target.write_bytes(data)+ return targetdiff --git a/src/pyrtan/cli.py b/src/pyrtan/cli.py index 8a0e655..76aacd1 100644 --- a/src/pyrtan/cli.py +++ b/src/pyrtan/cli.py@@ -10,6 +10,15 @@pyrtan gaps --at URL branches waiting to be written pyrtan unanswered --at URL [--scaffold DIR] trees waiting to be written pyrtan new [NAME] [--from TEXT] [--dir PATH] scaffold a tree file+ pyrtan search [WORDS] [--catalog URL] find trees in a catalog+ pyrtan pull NAME[@VERSION] [--dir] fetch a tree from a catalog, verified+ pyrtan index DIR [--out FILE] build a catalog index from a directory+ pyrtan console --out FILE write the console as one static HTML file++``serve`` and ``hub`` take ``--token`` (or ``$PYRTAN_TOKEN``) to require a+bearer token, ``--cors ORIGIN`` to let a browser elsewhere call them, and+``serve`` takes ``--gaps FILE`` to keep gaps across restarts and+``--egress public`` to refuse private addresses.""" from __future__ import annotations@@ -22,11 +31,13 @@ from pathlib import Pathfrom typing import Any from . import __version__+from .catalog import Catalog, CatalogError, build_index, catalog_urlfrom .grow import Proposal, propose+from .page import PAGEfrom .registry import Registry, default_paths from .runtime import Outcome, Runner from .service import RemoteResolver, http_json, outcome_from_dict, serve-from .transport import RecordedTransport, RecordingTransport, UrllibTransport+from .transport import EGRESS_POLICIES, RecordedTransport, RecordingTransport, UrllibTransportfrom .tree import check, load_tree@@ -55,12 +66,15 @@ def main(argv: list[str] | None = None) -> int:p.add_argument("--record", help="save every HTTP response to this fixture") p.add_argument("--hub", help="forward trees not held here to this hub or service") p.add_argument("--env", action="append", default=[], metavar="K=V", help="a value trees may read as {{ env.K }} (repeatable)")+ p.add_argument("--egress", choices=EGRESS_POLICIES, default="any", help="'public' refuses private, loopback and link-local addresses")+ _token_arg(p, "the token the hub (--hub) requires, if any")p = sub.add_parser("ask", help="answer a sentence through a running service or hub") p.add_argument("text") p.add_argument("--at", default=os.environ.get("PYRTAN_AT", "http://127.0.0.1:8700"), help="service or hub URL (default $PYRTAN_AT or the local hub)") p.add_argument("--json", action="store_true") p.add_argument("--no-trace", action="store_true")+ _token_arg(p, "the token the service or hub requires, if any")p = sub.add_parser("serve", help="host trees as a service") p.add_argument("paths", nargs="*")@@ -70,6 +84,9 @@ def main(argv: list[str] | None = None) -> int:p.add_argument("--hub", help="register with this hub, and forward unknown trees to it") p.add_argument("--advertise", help="the URL the hub should reach this service at, if not host:port") p.add_argument("--env", action="append", default=[], metavar="K=V")+ p.add_argument("--gaps", metavar="FILE", help="keep gaps in this file (JSON lines) so they survive a restart")+ p.add_argument("--egress", choices=EGRESS_POLICIES, default=os.environ.get("PYRTAN_EGRESS", "any"), help="'public' refuses private, loopback and link-local addresses (default $PYRTAN_EGRESS or any)")+ _access_args(p)p.add_argument("--quiet", action="store_true") p = sub.add_parser("hub", help="route between services")@@ -77,24 +94,59 @@ def main(argv: list[str] | None = None) -> int:p.add_argument("--port", type=int, default=8700) p.add_argument("--peer", action="append", default=[], help="a service URL to start with (repeatable; services may also register themselves)") p.add_argument("--poll", type=float, default=10.0, help="seconds between health polls")+ _access_args(p)p.add_argument("--quiet", action="store_true") p = sub.add_parser("gaps", help="decisions that met a value with no branch") p.add_argument("--at", help="a running service or hub") p.add_argument("--json", action="store_true")+ _token_arg(p, "the token the service or hub requires, if any")p = sub.add_parser("unanswered", help="sentences nobody answered, each with the tree that would") p.add_argument("--at", help="a running service or hub") p.add_argument("--json", action="store_true") p.add_argument("--scaffold", metavar="DIR", help="write a tree file for each sentence into DIR (existing files are kept)")+ _token_arg(p, "the token the service or hub requires, if any")p = sub.add_parser("new", help="scaffold a tree file") p.add_argument("name", nargs="?", help="the tree's name (derived from --from when omitted)") p.add_argument("--from", dest="sentence", metavar="TEXT", help="the sentence the tree should answer; numbers and hosts in it become slots") p.add_argument("--dir", default=".")+ p = sub.add_parser("search", help="find trees in a catalog")+ p.add_argument("words", nargs="*", help="words that must all appear in the name, description, tags, hosts or triggers")+ p.add_argument("--catalog", default=None, help=f"catalog URL or directory (default $PYRTAN_CATALOG or {catalog_url()})")+ p.add_argument("--json", action="store_true")++ p = sub.add_parser("pull", help="fetch a tree from a catalog, verified against its index")+ p.add_argument("spec", nargs="+", metavar="NAME[@VERSION]")+ p.add_argument("--dir", default=None, help="where to write (default: the first of $PYRTAN_TREES or ./trees)")+ p.add_argument("--catalog", default=None, help=f"catalog URL or directory (default $PYRTAN_CATALOG or {catalog_url()})")+ p.add_argument("--force", action="store_true", help="replace a file that exists and differs")++ p = sub.add_parser("index", help="build a catalog index.json from a directory of trees")+ p.add_argument("dir")+ p.add_argument("--out", default=None, help="write the index here (default: DIR/index.json; '-' for stdout)")+ p.add_argument("--allow-unlicensed", action="store_true", help="index trees with no 'license' too")++ p = sub.add_parser("console", help="write the console as one static HTML file, to host anywhere")+ p.add_argument("--out", required=True, help="the file to write ('-' for stdout)")+args = parser.parse_args(argv)- return {"check": cmd_check, "list": cmd_list, "run": cmd_run, "ask": cmd_ask, "serve": cmd_serve, "hub": cmd_hub, "gaps": cmd_gaps, "unanswered": cmd_unanswered, "new": cmd_new}[args.command](args)+ commands = {+ "check": cmd_check, "list": cmd_list, "run": cmd_run, "ask": cmd_ask, "serve": cmd_serve, "hub": cmd_hub, "gaps": cmd_gaps, "unanswered": cmd_unanswered, "new": cmd_new,+ "search": cmd_search, "pull": cmd_pull, "index": cmd_index, "console": cmd_console,+ }+ return commands[args.command](args)+++def _token_arg(p: argparse.ArgumentParser, help_text: str) -> None:+ p.add_argument("--token", default=os.environ.get("PYRTAN_TOKEN"), help=f"{help_text} (default $PYRTAN_TOKEN)")+++def _access_args(p: argparse.ArgumentParser) -> None:+ _token_arg(p, "require this bearer token on every request but /health and the console")+ p.add_argument("--cors", action="append", default=[], metavar="ORIGIN", help="let a browser at this origin call (repeatable; '*' for any; e.g. https://pyrtan.com)")# ---------------------------------------------------------------- commands@@ -175,12 +227,12 @@ def cmd_run(args: argparse.Namespace) -> int:reg = Registry.from_paths(_paths(args.trees)) for path, err in reg.errors.items(): print(f"pyrtan: {path}: {err}", file=sys.stderr)- transport: Any = RecordedTransport.from_file(args.replay) if args.replay else UrllibTransport()+ transport: Any = RecordedTransport.from_file(args.replay) if args.replay else UrllibTransport(egress=args.egress)recorder = None if args.record: recorder = RecordingTransport(transport) transport = recorder- runner = Runner(reg, transport, resolver=RemoteResolver(args.hub) if args.hub else None, env=_kv(args.env))+ runner = Runner(reg, transport, resolver=RemoteResolver(args.hub, args.token) if args.hub else None, env=_kv(args.env))if args.tree: outcome = runner.run_tree(args.tree, _kv(args.input)) else:@@ -206,17 +258,20 @@ def _answer_questions(outcome: Outcome, resume) -> Outcome:def cmd_ask(args: argparse.Namespace) -> int: at = args.at.rstrip("/") try:- status, body = http_json("POST", f"{at}/intent", {"text": args.text, "trace": not args.no_trace})+ status, body = http_json("POST", f"{at}/intent", {"text": args.text, "trace": not args.no_trace}, token=args.token)except Exception as e: print(f"pyrtan: {at}: {e}", file=sys.stderr) return 3+ if status == 401:+ print(f"pyrtan: {at} needs a token: pass --token or set $PYRTAN_TOKEN", file=sys.stderr)+ return 3if status != 200 or not isinstance(body, dict): print(f"pyrtan: {at} answered {status}: {body}", file=sys.stderr) return 3 outcome = outcome_from_dict(body) def remote_resume(snapshot: dict[str, Any], answer: Any) -> Outcome:- st, bd = http_json("POST", f"{at}/resume", {"snapshot": snapshot, "answer": answer, "trace": not args.no_trace})+ st, bd = http_json("POST", f"{at}/resume", {"snapshot": snapshot, "answer": answer, "trace": not args.no_trace}, token=args.token)return outcome_from_dict(bd) if isinstance(bd, dict) else Outcome("error", "", reason="remote", say=f"{at} answered {st}") outcome = _answer_questions(outcome, remote_resume)@@ -242,12 +297,12 @@ def _report(outcome: Outcome, as_json: bool, with_trace: bool) -> int:def cmd_serve(args: argparse.Namespace) -> int: paths = _paths(args.paths) name = args.name or (paths[0].resolve().name if paths else "pyrtan")- service, server = serve(paths, args.host, args.port, name=name, hub=args.hub, env=_kv(args.env), advertise=args.advertise, quiet=args.quiet)- print(f"pyrtan service {name!r}: {len(service.registry)} tree(s) at {server.url}" + (f", registered with {args.hub}" if args.hub else ""), flush=True)+ service, server = serve(paths, args.host, args.port, name=name, hub=args.hub, env=_kv(args.env), advertise=args.advertise, quiet=args.quiet, token=args.token, cors=args.cors, gaps=args.gaps, egress=args.egress)+ print(f"pyrtan service {name!r}: {len(service.registry)} tree(s) at {server.url}" + (f", registered with {args.hub}" if args.hub else "") + _access_note(args), flush=True)for path, err in service.registry.errors.items(): print(f" not loaded: {path}: {err}", file=sys.stderr) try:- server.serve_forever()+ server.wait()except KeyboardInterrupt: pass finally:@@ -258,8 +313,8 @@ def cmd_serve(args: argparse.Namespace) -> int:def cmd_hub(args: argparse.Namespace) -> int: from .hub import serve_hub- hub, server = serve_hub(args.host, args.port, peers=args.peer, poll_every=args.poll, quiet=args.quiet)- print(f"pyrtan hub at {server.url}: {len(hub.peers)} peer(s)", flush=True)+ hub, server = serve_hub(args.host, args.port, peers=args.peer, poll_every=args.poll, quiet=args.quiet, token=args.token, cors=args.cors)+ print(f"pyrtan hub at {server.url}: {len(hub.peers)} peer(s)" + _access_note(args), flush=True)try: server.serve_forever() except KeyboardInterrupt:@@ -275,10 +330,13 @@ def cmd_gaps(args: argparse.Namespace) -> int:print("pyrtan gaps: give --at URL of a running service or hub", file=sys.stderr) return 2 try:- status, body = http_json("GET", f"{args.at.rstrip('/')}/gaps")+ status, body = http_json("GET", f"{args.at.rstrip('/')}/gaps", token=args.token)except Exception as e: print(f"pyrtan: {args.at}: {e}", file=sys.stderr) return 3+ if status == 401:+ print(f"pyrtan: {args.at} needs a token: pass --token or set $PYRTAN_TOKEN", file=sys.stderr)+ return 3gaps = (body or {}).get("gaps", []) if isinstance(body, dict) else [] if args.json: print(json.dumps({"gaps": gaps}, indent=2, ensure_ascii=False))@@ -288,7 +346,88 @@ def cmd_gaps(args: argparse.Namespace) -> int:return 0 for g in gaps: peer = f" ({g['peer']})" if g.get("peer") else ""- print(f"{g['tree']}.{g['node']}: {g['on']} = {json.dumps(g['observed'], ensure_ascii=False)}{peer}")+ times = f" ×{g['count']}" if g.get("count", 1) > 1 else ""+ print(f"{g['tree']}.{g['node']}: {g['on']} = {json.dumps(g['observed'], ensure_ascii=False)}{times}{peer}")+ return 0+++def _access_note(args: argparse.Namespace) -> str:+ notes = []+ if getattr(args, "token", None):+ notes.append("token required")+ if getattr(args, "cors", None):+ notes.append("cors: " + ", ".join(args.cors))+ if getattr(args, "egress", "any") != "any":+ notes.append(f"egress: {args.egress}")+ return f" ({'; '.join(notes)})" if notes else ""+++# ---------------------------------------------------------------- catalog+++def cmd_search(args: argparse.Namespace) -> int:+ cat = Catalog(args.catalog)+ try:+ found = cat.search(" ".join(args.words))+ except CatalogError as e:+ print(f"pyrtan: {e}", file=sys.stderr)+ return 3+ if args.json:+ print(json.dumps({"catalog": cat.base, "trees": found}, indent=2, ensure_ascii=False))+ return 0+ if not found:+ print(f"nothing in {cat.base} matches {' '.join(args.words)!r}")+ return 1+ for e in found:+ hosts = ", ".join(e.get("hosts", []))+ print(f"{e['name']}@{e.get('version', '?')} {e.get('description', '')}" + (f" [{hosts}]" if hosts else ""))+ for t in e.get("triggers", [])[:3]:+ print(f" {t}")+ return 0+++def cmd_pull(args: argparse.Namespace) -> int:+ cat = Catalog(args.catalog)+ into = Path(args.dir) if args.dir else default_paths()[0]+ worst = 0+ for spec in args.spec:+ try:+ target = cat.pull(spec, into, force=args.force)+ except CatalogError as e:+ print(f"pyrtan: {e}", file=sys.stderr)+ worst = 1+ continue+ tree = load_tree(target)+ print(f"{tree.name}@{tree.version} -> {target}" + (f" (hosts: {', '.join(tree.hosts)})" if tree.hosts else ""))+ return worst+++def cmd_index(args: argparse.Namespace) -> int:+ root = Path(args.dir)+ if not root.is_dir():+ print(f"pyrtan: {root}: not a directory", file=sys.stderr)+ return 2+ index, problems = build_index(root, require_license=not args.allow_unlicensed)+ for p in problems:+ print(f"{p}", file=sys.stderr)+ text = json.dumps(index, indent=2, ensure_ascii=False) + "\n"+ if args.out == "-":+ sys.stdout.write(text)+ else:+ out = Path(args.out) if args.out else root / "index.json"+ out.write_text(text, encoding="utf-8")+ print(f"wrote {out}: {index['count']} tree(s)" + (f", {len(problems)} left out" if problems else ""))+ return 1 if problems else 0+++def cmd_console(args: argparse.Namespace) -> int:+ if args.out == "-":+ sys.stdout.write(PAGE)+ return 0+ out = Path(args.out)+ out.parent.mkdir(parents=True, exist_ok=True)+ out.write_text(PAGE, encoding="utf-8")+ print(f"wrote {out}; serve it from any static host, and point it at a hub with --cors set for that host")return 0@@ -297,10 +436,13 @@ def cmd_unanswered(args: argparse.Namespace) -> int:print("pyrtan unanswered: give --at URL of a running service or hub", file=sys.stderr) return 2 try:- status, body = http_json("GET", f"{args.at.rstrip('/')}/unanswered")+ status, body = http_json("GET", f"{args.at.rstrip('/')}/unanswered", token=args.token)except Exception as e: print(f"pyrtan: {args.at}: {e}", file=sys.stderr) return 3+ if status == 401:+ print(f"pyrtan: {args.at} needs a token: pass --token or set $PYRTAN_TOKEN", file=sys.stderr)+ return 3entries = (body or {}).get("unanswered", []) if isinstance(body, dict) else [] if args.json: print(json.dumps({"unanswered": entries}, indent=2, ensure_ascii=False)) diff --git a/src/pyrtan/gaps.py b/src/pyrtan/gaps.py new file mode 100644 index 0000000..17d9887 --- /dev/null +++ b/src/pyrtan/gaps.py@@ -0,0 +1,95 @@+"""Where a service keeps its gaps.++A gap is a decision that met a value no branch covered. The runtime records+one on the trace; the service collects them here so an author can read the+list and write the missing branches. Without a path the store lives in+memory and is lost with the process, which is enough to try the loop (run,+see the gap, add the branch, reload). With a path it appends every new gap+to a JSON-lines file and reads the file back on start, so a fleet's gaps+outlive restarts and a hub can collect them later.++The same gap seen twice is one entry with a count: the list is what to+write next, not a log of every run.+"""++from __future__ import annotations++import json+import sys+import threading+import time+from pathlib import Path+from typing import Any, Iterable+++def _key(gap: dict[str, Any]) -> str:+ return json.dumps([gap.get("tree"), gap.get("node"), gap.get("on"), gap.get("observed")], sort_keys=True, ensure_ascii=False, default=str)+++class GapStore:+ def __init__(self, path: str | Path | None = None) -> None:+ self.path = Path(path) if path else None+ self.error: str | None = None # the last failure to write the file; a run never fails over it+ self._gaps: dict[str, dict[str, Any]] = {}+ self._lock = threading.Lock()+ if self.path is not None and self.path.exists():+ try:+ self._load()+ except OSError as e:+ self.error = str(e)+ print(f"pyrtan: cannot read gaps from {self.path}: {e}; starting empty", file=sys.stderr)++ def _load(self) -> None:+ assert self.path is not None+ for line in self.path.read_text(encoding="utf-8").splitlines():+ line = line.strip()+ if not line:+ continue+ try:+ gap = json.loads(line)+ except json.JSONDecodeError:+ continue+ if isinstance(gap, dict) and "tree" in gap:+ self._merge(gap, persist=False)++ def _merge(self, gap: dict[str, Any], *, persist: bool) -> None:+ key = _key(gap)+ now = time.time()+ current = self._gaps.get(key)+ if current is None:+ entry = {+ "tree": gap.get("tree"),+ "node": gap.get("node"),+ "on": gap.get("on"),+ "observed": gap.get("observed"),+ "count": int(gap.get("count", 1)),+ "first_seen": gap.get("first_seen", now),+ "last_seen": gap.get("last_seen", now),+ }+ self._gaps[key] = entry+ else:+ current["count"] += int(gap.get("count", 1))+ current["last_seen"] = max(current["last_seen"], gap.get("last_seen", now))+ entry = current+ if persist and self.path is not None:+ try:+ self.path.parent.mkdir(parents=True, exist_ok=True)+ with self.path.open("a", encoding="utf-8") as f:+ f.write(json.dumps({**entry, "count": 1, "last_seen": now}, ensure_ascii=False, default=str) + "\n")+ self.error = None+ except OSError as e:+ if self.error is None:+ print(f"pyrtan: cannot write gaps to {self.path}: {e}; keeping them in memory", file=sys.stderr)+ self.error = str(e)++ def add(self, gaps: Iterable[dict[str, Any]]) -> None:+ with self._lock:+ for gap in gaps:+ self._merge(gap, persist=True)++ def all(self) -> list[dict[str, Any]]:+ with self._lock:+ return sorted((dict(g) for g in self._gaps.values()), key=lambda g: (-g["count"], g["last_seen"]))++ def __len__(self) -> int:+ return len(self._gaps)diff --git a/src/pyrtan/grow.py b/src/pyrtan/grow.py index f659210..71e03bb 100644 --- a/src/pyrtan/grow.py +++ b/src/pyrtan/grow.py@@ -48,6 +48,7 @@ name = "{name}"version = 1 description = "" tags = []+license = "" # an SPDX identifier such as "MIT", if you share ithosts = {hosts} start = "fetch" diff --git a/src/pyrtan/hub.py b/src/pyrtan/hub.py index 75ed25f..b9f5ae9 100644 --- a/src/pyrtan/hub.py +++ b/src/pyrtan/hub.py@@ -17,12 +17,13 @@ Endpoints:``POST /resume`` forwarded to the peer named in the snapshot ``GET /gaps`` every peer's gaps, with the peer that saw them ``GET /unanswered`` sentences no peer answered, each with a proposed tree-``GET /`` the demo page; ``GET /api`` is the JSON index+``GET /`` the console; ``GET /api`` is the JSON indexA service that holds several trees and a hub that routes between services answer the same requests, so a client need not know which it is talking to, and a swarm can start as one process and split later without the callers-changing.+changing. A hub given a token requires it of callers (``/register``+included, so a stranger cannot join the swarm) and presents it to peers.""" from __future__ import annotations@@ -32,6 +33,7 @@ import timefrom dataclasses import dataclass, field from typing import Any+from . import __version__from .grow import Unanswered, no_match_outcome from .intent import Trigger, best_matches from .page import PAGE@@ -66,9 +68,10 @@ class Peer:class Hub:- def __init__(self, *, name: str = "hub", poll_every: float = 10.0) -> None:+ def __init__(self, *, name: str = "hub", poll_every: float = 10.0, token: str | None = None) -> None:self.name = name self.poll_every = poll_every+ self.token = token or Noneself.peers: dict[str, Peer] = {} self.unanswered = Unanswered() self.started = time.time()@@ -92,7 +95,7 @@ class Hub:def refresh(self, peer: Peer) -> None: try:- status, caps = http_json("GET", f"{peer.url}/capabilities", timeout=5.0)+ status, caps = http_json("GET", f"{peer.url}/capabilities", timeout=5.0, token=self.token)except Exception as e: # urllib raises URLError subclasses of OSError status, caps = 0, None peer.last_error = str(e)@@ -207,7 +210,7 @@ class Hub:if peer is None: return 200, {"status": "error", "reason": "no-peer"} try:- status, answer = http_json("POST", f"{peer.url}{path}", body, timeout=120.0)+ status, answer = http_json("POST", f"{peer.url}{path}", body, timeout=120.0, token=self.token)except Exception as e: peer.last_error = str(e) return 200, {"status": "error", "reason": "peer-unreachable", "say": f"{peer.name or peer.url} did not answer: {e}", "output": {"peer": peer.url}}@@ -221,7 +224,7 @@ class Hub:out: list[dict[str, Any]] = [] for peer in list(self.peers.values()): try:- status, body = http_json("GET", f"{peer.url}/gaps", timeout=5.0)+ status, body = http_json("GET", f"{peer.url}/gaps", timeout=5.0, token=self.token)except Exception: continue if status == 200 and isinstance(body, dict):@@ -236,7 +239,7 @@ class Hub:if peer.link == "down": continue try:- status, body = http_json("GET", f"{peer.url}/unanswered", timeout=5.0)+ status, body = http_json("GET", f"{peer.url}/unanswered", timeout=5.0, token=self.token)except Exception: continue if status == 200 and isinstance(body, dict):@@ -256,10 +259,10 @@ class Hub:return 200, self.gaps() if method == "GET" and path == "/unanswered": return 200, self.unanswered_all()- if method == "GET" and path in ("/", "/index.html"):+ if method == "GET" and path in ("/", "/index.html", "/console"):return 200, PAGE if method == "GET" and path == "/api":- return 200, {"pyrtan": "hub", "name": self.name, "endpoints": ["/health", "/peers", "/capabilities", "/intent", "/run", "/resume", "/gaps", "/unanswered", "/register"]}+ return 200, {"pyrtan": "hub", "name": self.name, "version": __version__, "auth": "bearer" if self.token else "none", "endpoints": ["/health", "/peers", "/capabilities", "/intent", "/run", "/resume", "/gaps", "/unanswered", "/register"]}if method == "POST" and path == "/register": url = body.get("url") if not isinstance(url, str) or not url.startswith(("http://", "https://")):@@ -279,9 +282,9 @@ def _rank(peer: Peer) -> int:return {"up": 0, "pending": 1, "stale": 2, "down": 3}[peer.link]-def serve_hub(host: str = "127.0.0.1", port: int = 8700, *, name: str = "hub", peers: list[str] | None = None, poll_every: float = 10.0, quiet: bool = False) -> tuple[Hub, Server]:- hub = Hub(name=name, poll_every=poll_every)- server = Server(hub.route, host, port, quiet)+def serve_hub(host: str = "127.0.0.1", port: int = 8700, *, name: str = "hub", peers: list[str] | None = None, poll_every: float = 10.0, quiet: bool = False, token: str | None = None, cors: list[str] | None = None) -> tuple[Hub, Server]:+ hub = Hub(name=name, poll_every=poll_every, token=token)+ server = Server(hub.route, host, port, quiet, token=token, cors=cors)for url in peers or []: hub.add_peer(url) hub.start_polling() diff --git a/src/pyrtan/page.py b/src/pyrtan/page.py index 9298fbd..ae57c0f 100644 --- a/src/pyrtan/page.py +++ b/src/pyrtan/page.py@@ -1,32 +1,50 @@-"""The demo page: one HTML file a service or hub serves at ``/``.--It lists what the swarm can answer, takes a sentence, posts it to-``/intent`` on the same origin, and shows the answer, the output and the-trace. When nothing answers, it shows the tree that would, and the command-that scaffolds it; the sentences nobody answered are listed beside what the-swarm can do. No external assets, no build step; it exists so that someone-can point a browser at a running Pyrtan and understand it in a minute.+"""The console: one HTML file a service or hub serves at ``/``, and the same+file hosted anywhere else (``pyrtan console --out``).++It asks a sentence and shows the answer, the output and the trace; lists+what the swarm can answer; shows the hub's peers and their ``link``; and+lists the gaps. Everything it does goes through the public endpoints+(``/api``, ``/health``, ``/capabilities``, ``/peers``, ``/gaps``,+``/intent``, ``/resume``), so it works the same against a hub and a+service, and the same whether it was served by that hub or by a static+host somewhere else.++Served by a hub or service, it talks to the origin it came from. Served+from elsewhere (it finds out by asking ``api`` on its own origin), it shows+a connect box for a hub URL and a token, keeps them in the browser's local+storage, and sends the token as a bearer. The hub must then allow the+console's origin with ``--cors``. No external assets, no build step.""" from __future__ import annotations+from . import __version__+PAGE = r"""<!doctype html> <html lang="en"> <head> <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>Pyrtan</title>+<meta name="description" content="Pyrtan console: ask the swarm, read the trace, see what it can answer, and find the gaps."><style>- :root { --bg:#fbfaf7; --fg:#1d1d1b; --mute:#6b6a64; --line:#e3e1da; --card:#ffffff; --ok:#1f7a4d; --bad:#a3382b; --ask:#8a6400; --accent:#2b5db8; }- @media (prefers-color-scheme: dark) { :root { --bg:#141513; --fg:#ecebe6; --mute:#9d9c95; --line:#2b2c29; --card:#1c1d1b; --ok:#5fcf93; --bad:#f08a7a; --ask:#e2b84a; --accent:#7fa6f0; } }+ :root { --bg:#fbfaf7; --fg:#1d1d1b; --mute:#6b6a64; --line:#e3e1da; --card:#ffffff; --ok:#1f7a4d; --bad:#a3382b; --ask:#8a6400; --accent:#2b5db8; --accent-fg:#fff; --soft:#f1efe9 }+ @media (prefers-color-scheme: dark) { :root { --bg:#141513; --fg:#ecebe6; --mute:#9d9c95; --line:#2b2c29; --card:#1c1d1b; --ok:#5fcf93; --bad:#f08a7a; --ask:#e2b84a; --accent:#7fa6f0; --accent-fg:#101318; --soft:#222420 } }* { box-sizing:border-box } body { margin:0; background:var(--bg); color:var(--fg); font:16px/1.5 system-ui,-apple-system,"Segoe UI",sans-serif }- main { max-width:880px; margin:0 auto; padding:32px 16px 64px }- h1 { font-size:22px; margin:0 0 4px } h1 small { color:var(--mute); font-weight:normal; font-size:14px; margin-left:8px }- p.lede { color:var(--mute); margin:0 0 24px }- form { display:flex; gap:8px; margin:0 0 16px }- input[type=text] { flex:1; font:inherit; padding:10px 12px; border:1px solid var(--line); border-radius:8px; background:var(--card); color:var(--fg) }- button { font:inherit; padding:10px 16px; border:0; border-radius:8px; background:var(--accent); color:#fff; cursor:pointer }+ main { max-width:920px; margin:0 auto; padding:24px 16px 64px }+ header { display:flex; flex-wrap:wrap; align-items:baseline; gap:8px 16px; margin-bottom:12px }+ h1 { font-size:22px; margin:0 } h1 small { color:var(--mute); font-weight:normal; font-size:14px; margin-left:8px }+ h2 { font-size:17px; margin:24px 0 8px }+ nav { display:flex; gap:4px; flex-wrap:wrap; margin:12px 0 16px }+ nav a { padding:6px 12px; border-radius:999px; color:var(--fg); text-decoration:none; border:1px solid var(--line); background:var(--card); font-size:14px }+ nav a[aria-current=page] { background:var(--accent); color:var(--accent-fg); border-color:var(--accent) }+ section[hidden] { display:none }+ p.lede { color:var(--mute); margin:0 0 16px }+ form.row { display:flex; gap:8px; margin:0 0 12px; flex-wrap:wrap }+ form.row input[type=text], form.row input[type=url], form.row input[type=password] { flex:1 1 180px; min-width:0; font:inherit; padding:10px 12px; border:1px solid var(--line); border-radius:8px; background:var(--card); color:var(--fg) }+ button { font:inherit; padding:10px 16px; border:0; border-radius:8px; background:var(--accent); color:var(--accent-fg); cursor:pointer }+ button.quiet { background:var(--soft); color:var(--fg) }button:disabled { opacity:.6; cursor:wait } .card { background:var(--card); border:1px solid var(--line); border-radius:10px; padding:14px 16px; margin:0 0 14px } .status { font-weight:600 } .done{color:var(--ok)} .failed,.error{color:var(--bad)} .asked{color:var(--ask)}@@ -38,32 +56,99 @@ PAGE = r"""<!doctype html>table { width:100%; border-collapse:collapse; font-size:14px } td,th { text-align:left; padding:6px 8px; border-top:1px solid var(--line); vertical-align:top } th { color:var(--mute); font-weight:600; border-top:0 } code { font:13px ui-monospace,SFMono-Regular,Menlo,monospace } .mute { color:var(--mute) } .gap { color:var(--ask) }+ .link { display:inline-block; width:9px; height:9px; border-radius:50%; margin-right:6px; vertical-align:middle; background:var(--mute) }+ .link.up { background:var(--ok) } .link.stale { background:var(--ask) } .link.down { background:var(--bad) }+ .notice { border-left:3px solid var(--ask); padding:8px 12px; background:var(--soft); border-radius:6px; margin:0 0 14px; font-size:14px }+ .notice.bad { border-color:var(--bad) }footer { color:var(--mute); font-size:13px; margin-top:32px }+ footer a, .card a { color:var(--accent) }+ @media (max-width:560px) { table thead { display:none } table td { display:block; border-top:0 } table tr { display:block; border-top:1px solid var(--line); padding:6px 0 } }</style> </head> <body> <main>- <h1>Pyrtan <small id="who">…</small></h1>- <p class="lede">A core set of expandable services that act on all forms of I/O through hardcoded, inspectable decision trees. Type a sentence a tree answers to; the answer comes back with the exact path that produced it. If nothing answers, you get the tree that would. No model is involved.</p>+ <header>+ <h1>Pyrtan <small id="who">…</small></h1>+ <span class="mute" id="where"></span>+ </header>+ <p class="lede">A core set of expandable services that act on all forms of I/O through hardcoded, inspectable decision trees. Ask a sentence a tree answers to; the answer comes back with the exact path that produced it. If nothing answers, you get the tree that would. No model is involved.</p>++ <section id="connect" hidden>+ <div class="card">+ <div style="margin-bottom:8px" id="connect-lede">This console is not being served by a Pyrtan. Point it at one:</div>+ <form class="row" id="connect-form">+ <input type="url" id="hub-url" placeholder="https://hub.example.net or http://192.168.1.20:8700" autocomplete="off">+ <input type="password" id="hub-token" placeholder="token, if the hub needs one" autocomplete="off">+ <button type="submit">Connect</button>+ <button type="button" class="quiet" id="forget">Forget</button>+ </form>+ <div class="mute" style="font-size:14px" id="connect-help">Kept in this browser only. The hub must allow this page's origin: <code>pyrtan hub --cors <span id="origin"></span></code>. A hub on your LAN over <code>http://</code> may be refused by the browser when this page is <code>https://</code>; open the hub's own <code>/</code> instead, which is this same console.</div>+ </div>+ </section>++ <div id="problem"></div>- <form id="ask">- <input type="text" id="text" placeholder="what's the weather in Oslo" autofocus autocomplete="off">- <button type="submit" id="go">Ask</button>- </form>- <div class="card" style="margin-bottom:24px">- <div class="mute" style="margin-bottom:8px">Try one</div>- <ul class="tries" id="tries"></ul>- </div>+ <nav id="tabs">+ <a href="#ask" data-tab="ask">Ask</a>+ <a href="#trees" data-tab="trees">Trees</a>+ <a href="#peers" data-tab="peers" id="peers-tab" hidden>Peers</a>+ <a href="#gaps" data-tab="gaps">Gaps</a>+ <a href="#unanswered" data-tab="unanswered">Unanswered</a>+ <a href="#api" data-tab="api">API</a>+ </nav>- <div id="out"></div>+ <section id="tab-ask">+ <form class="row" id="ask">+ <input type="text" id="text" placeholder="what's the weather in Oslo" autofocus autocomplete="off">+ <button type="submit" id="go">Ask</button>+ </form>+ <div class="card" style="margin-bottom:24px">+ <div class="mute" style="margin-bottom:8px">Try one</div>+ <ul class="tries" id="tries"></ul>+ </div>+ <div id="out"></div>+ </section>- <h2 style="font-size:17px;margin:32px 0 8px">What this swarm can answer</h2>- <div class="card"><table id="caps"><thead><tr><th>Tree</th><th>Answers to</th><th>From</th></tr></thead><tbody></tbody></table></div>+ <section id="tab-trees" hidden>+ <h2>What this swarm can answer</h2>+ <div class="card"><table id="caps"><thead><tr><th>Tree</th><th>Answers to</th><th>From</th></tr></thead><tbody></tbody></table></div>+ <p class="mute" style="font-size:14px">Need a tree that is not here? Browse the <a href="https://pyrtan.com/catalog/" id="catalog-link">catalog</a>, then <code>pyrtan pull NAME</code> into a folder a service serves.</p>+ </section>- <h2 style="font-size:17px;margin:32px 0 8px">Waiting to be written</h2>- <div class="card"><table id="unanswered"><thead><tr><th>Nobody answered</th><th>Would be</th><th>To write it</th></tr></thead><tbody><tr><td colspan="3" class="mute">Nothing yet: every sentence so far found a tree.</td></tr></tbody></table></div>+ <section id="tab-peers" hidden>+ <h2>Peers</h2>+ <div class="card"><table id="peers"><thead><tr><th>Link</th><th>Peer</th><th>Trees</th><th>Last</th></tr></thead><tbody></tbody></table></div>+ <p class="mute" style="font-size:14px"><span class="link up"></span>up: answered within 30 s · <span class="link stale"></span>stale: within 90 s · <span class="link down"></span>down: longer, or never.</p>+ </section>- <footer>The same endpoints the page uses: <code>POST /intent {"text": "…"}</code>, <code>POST /run {"tree": "…", "inputs": {…}}</code>, <code>GET /capabilities</code>, <code>GET /health</code>, <code>GET /gaps</code>, <code>GET /unanswered</code>. <a href="https://github.com/b-i-t-8/pyrtan">Source</a>.</footer>+ <section id="tab-gaps" hidden>+ <h2>Gaps</h2>+ <p class="mute" style="font-size:14px;margin-top:0">A gap is a decision that met a value no branch covers. Each one is the next branch to write, in the tree and node named.</p>+ <div class="card"><table id="gaps"><thead><tr><th>Tree · node</th><th>Saw</th><th>Times</th><th>Where</th></tr></thead><tbody></tbody></table></div>+ </section>++ <section id="tab-unanswered" hidden>+ <h2>Waiting to be written</h2>+ <p class="mute" style="font-size:14px;margin-top:0">Sentences nobody answered, each with the tree that would. <code>pyrtan unanswered --at <span class="at">URL</span> --scaffold DIR</code> writes the files.</p>+ <div class="card"><table id="unanswered"><thead><tr><th>Nobody answered</th><th>Would be</th><th>To write it</th></tr></thead><tbody></tbody></table></div>+ </section>++ <section id="tab-api" hidden>+ <h2>The same endpoints this page uses</h2>+ <div class="card">+ <pre id="api-pre">POST /intent {"text": "…"}+POST /run {"tree": "…", "inputs": {…}}+POST /resume {"snapshot": {…}, "answer": …}+GET /capabilities+GET /health+GET /gaps+GET /unanswered+GET /peers (hub only)</pre>+ <div class="mute" style="font-size:14px;margin-top:8px">With a token: <code>Authorization: Bearer …</code>. From the command line: <code>pyrtan ask "…" --at <span class="at">URL</span></code>, <code>pyrtan gaps --at <span class="at">URL</span></code>.</div>+ </div>+ </section>++ <footer><a href="https://pyrtan.com">pyrtan.com</a> · <a href="https://github.com/b-i-t-8/pyrtan">source</a> · AGPL-3.0-or-later · console v__VERSION__<span id="hub-version"></span></footer></main> <script> const $ = (s) => document.querySelector(s);@@ -75,54 +160,157 @@ const examples = {"github-release": "latest release of astral-sh/uv", "hacker-news-item": "hacker news item 1", };+// Served by a Pyrtan, only its token is kept, under its own origin; served elsewhere, the hub URL and token.+const KEY = "pyrtan.console:" + location.origin;+const store = {+ get() { try { return JSON.parse(localStorage.getItem(KEY) || "{}"); } catch (e) { return {}; } },+ set(v) { try { localStorage.setItem(KEY, JSON.stringify(v)); } catch (e) {} },+ clear() { try { localStorage.removeItem(KEY); } catch (e) {} },+};+// base: "" when this page is served by a Pyrtan (relative URLs); otherwise the hub URL the person gave.+let base = "", token = "", remote = false, isHub = false;++function api(path, init) {+ const headers = Object.assign({"Accept": "application/json"}, (init && init.headers) || {});+ if (token) headers["Authorization"] = "Bearer " + token;+ const url = base ? base.replace(/\/+$/, "") + "/" + path : path;+ return fetch(url, Object.assign({}, init, {headers}));+}+async function json(path, init) {+ const r = await api(path, init);+ if (r.status === 401) throw new Error("401: this Pyrtan needs a token");+ return r.json();+}+function problem(msg, bad) { $("#problem").innerHTML = msg ? `<div class="notice${bad ? " bad" : ""}">${esc(msg)}</div>` : ""; }++function showTab(name) {+ for (const a of document.querySelectorAll("#tabs a")) { if (a.dataset.tab === name) a.setAttribute("aria-current", "page"); else a.removeAttribute("aria-current"); }+ for (const s of document.querySelectorAll("main > section[id^=tab-]")) s.hidden = s.id !== "tab-" + name;+ if (name === "peers") loadPeers();+ if (name === "gaps") loadGaps();+ if (name === "unanswered") loadUnanswered();+}+window.addEventListener("hashchange", () => showTab((location.hash || "#ask").slice(1)));+function sample(t) { if (examples[t.name]) return examples[t.name]; const pat = (t.triggers || [])[0]; if (!pat) return null; return pat.replace(/\[[^\]]*\]/g, "").replace(/\((\w+)\|[^)]*\)/g, "$1").replace(/\{(\w+)(?::\w+)?\}/g, (m, n) => (t.inputs && t.inputs[n] && t.inputs[n].default != null) ? t.inputs[n].default : "…").replace(/\s+/g, " ").trim(); }++async function detect() {+ // Same origin first: a Pyrtan answers /api with {"pyrtan": "service"|"hub"}.+ try {+ const r = await fetch("api", {headers: {"Accept": "application/json"}});+ const a = r.ok ? await r.json() : null;+ if (a && a.pyrtan) {+ base = ""; remote = false; isHub = a.pyrtan === "hub";+ if (a.auth === "bearer") {+ // This Pyrtan wants a token: show the connect box with only the token field.+ token = store.get().token || "";+ $("#connect").hidden = false;+ $("#hub-url").hidden = true;+ $("#connect-lede").textContent = "This Pyrtan needs a token:";+ $("#connect-help").innerHTML = "Kept in this browser only. The token is the one the hub or service was started with (<code>--token</code> or <code>$PYRTAN_TOKEN</code>).";+ $("#hub-token").value = token;+ if (!token) problem("Enter the token to see what this Pyrtan can answer.");+ }+ return a;+ }+ } catch (e) {}+ remote = true;+ const saved = store.get();+ base = saved.url || ""; token = saved.token || "";+ $("#connect").hidden = false;+ $("#origin").textContent = location.origin;+ $("#hub-url").value = base; $("#hub-token").value = token;+ if (!base) return null;+ try {+ const a = await json("api");+ if (a && a.pyrtan) { isHub = a.pyrtan === "hub"; return a; }+ problem(`${base} answered, but not as a Pyrtan.`, true);+ } catch (e) { problem(`${base}: ${e.message}. Is the hub up, and started with --cors ${location.origin}?`, true); }+ return null;+}+async function load() {+ const a = await detect();+ for (const el of document.querySelectorAll(".at")) el.textContent = base || location.origin;+ if (!a) { $("#who").textContent = remote ? "not connected" : "unreachable"; return; }+ $("#peers-tab").hidden = !isHub;+ $("#where").textContent = (base || location.origin) + (a.auth === "bearer" ? " · token" : "");+ $("#hub-version").textContent = a.version ? ` · ${a.pyrtan} v${a.version}` : "";try {- const h = await (await fetch("health")).json();- $("#who").textContent = (h.name ? h.name + " · " : "") + (h.peers != null ? `${h.up}/${h.peers} peers up, ${h.trees} trees` : `${(h.trees||[]).length} trees`);+ const h = await json("health");+ $("#who").textContent = (h.name ? h.name + " · " : "") + (h.peers != null ? `${h.up}/${h.peers} peers up` + (h.trees != null ? `, ${h.trees} trees` : "") : (h.trees ? `${h.trees.length} trees` : ""));} catch (e) { $("#who").textContent = "unreachable"; }- const c = await (await fetch("capabilities")).json();+ if (a.auth === "bearer" && !token) return;+ let c;+ try { c = await json("capabilities"); } catch (e) { problem(e.message + (a.auth === "bearer" ? " (is it the right one?)" : ""), true); return; }+ problem("");const body = $("#caps tbody"); const tries = $("#tries");+ body.innerHTML = ""; tries.innerHTML = "";for (const t of c.trees || []) { const tr = document.createElement("tr");- tr.innerHTML = `<td><code>${esc(t.name)}</code><div class="mute">${esc(t.description||"")}</div></td><td>${(t.triggers||[]).map(p => `<div><code>${esc(p)}</code></div>`).join("")}</td><td class="mute">${esc((t.hosts||[]).join(", "))}${t.peer ? `<div>${esc(t.peer)} · ${esc(t.link||"")}</div>` : ""}</td>`;+ tr.innerHTML = `<td><code>${esc(t.name)}</code> <span class="mute">v${esc(t.version||1)}</span><div class="mute">${esc(t.description||"")}</div></td><td>${(t.triggers||[]).map(p => `<div><code>${esc(p)}</code></div>`).join("")}</td><td class="mute">${esc((t.hosts||[]).join(", "))}${t.peer ? `<div><span class="link ${esc(t.link||"")}"></span>${esc(t.peer)}</div>` : ""}</td>`;body.appendChild(tr); const s = sample(t); if (s && !s.includes("…")) { const li = document.createElement("li"); li.innerHTML = `<a href="#" data-q="${esc(s)}">${esc(s)}</a>`; tries.appendChild(li); } }- tries.addEventListener("click", (e) => { const a = e.target.closest("a[data-q]"); if (!a) return; e.preventDefault(); $("#text").value = a.dataset.q; ask(); });- loadUnanswered();+ showTab((location.hash || "#ask").slice(1));}++async function loadPeers() {+ if (!isHub) return;+ let p; try { p = await json("peers"); } catch (e) { problem(e.message, true); return; }+ const body = $("#peers tbody"); body.innerHTML = "";+ for (const peer of p.peers || []) {+ const tr = document.createElement("tr");+ const last = peer.last_ok ? new Date(peer.last_ok * 1000).toLocaleTimeString() : "never";+ tr.innerHTML = `<td><span class="link ${esc(peer.link)}"></span>${esc(peer.link)}</td><td><code>${esc(peer.name||"")}</code><div class="mute">${esc(peer.url)}</div></td><td>${(peer.trees||[]).map(n => `<code>${esc(n)}</code>`).join(" ")}</td><td class="mute">${esc(last)}${peer.last_error ? `<div class="failed">${esc(peer.last_error)}</div>` : ""}</td>`;+ body.appendChild(tr);+ }+ if (!(p.peers || []).length) body.innerHTML = `<tr><td colspan="4" class="mute">No peers yet. Start a service with <code>--hub ${esc(base || location.origin)}</code>.</td></tr>`;+}++async function loadGaps() {+ let g; try { g = await json("gaps"); } catch (e) { problem(e.message, true); return; }+ const body = $("#gaps tbody"); body.innerHTML = "";+ for (const gap of g.gaps || []) {+ const tr = document.createElement("tr");+ tr.innerHTML = `<td><code>${esc(gap.tree)}</code> · <code>${esc(gap.node)}</code></td><td><code>${esc(gap.on)}</code> = <code class="gap">${esc(JSON.stringify(gap.observed))}</code></td><td>${esc(gap.count||1)}</td><td class="mute">${esc(gap.peer||"")}</td>`;+ body.appendChild(tr);+ }+ if (!(g.gaps || []).length) body.innerHTML = `<tr><td colspan="4" class="mute">No gaps: every decision so far had a branch.</td></tr>`;+}+async function loadUnanswered() {- let u; try { u = await (await fetch("unanswered")).json(); } catch (e) { return; }- const rows = (u.unanswered || []);- if (!rows.length) return;+ let u; try { u = await json("unanswered"); } catch (e) { problem(e.message, true); return; }const body = $("#unanswered tbody"); body.innerHTML = "";- for (const e of rows) {+ for (const e of u.unanswered || []) {const p = e.proposal || {}; const tr = document.createElement("tr"); tr.innerHTML = `<td>${esc(e.text)} <span class="mute">×${esc(e.count||1)}${e.peer ? " · " + esc(e.peer) : ""}</span></td><td><code>${esc(p.name||"")}</code><div class="mute"><code>${esc(p.trigger||"")}</code></div></td><td><code>${esc(p.scaffold||"")}</code></td>`; body.appendChild(tr); }+ if (!(u.unanswered || []).length) body.innerHTML = `<tr><td colspan="3" class="mute">Nothing yet: every sentence so far found a tree.</td></tr>`;}+function renderTrace(tr) { if (!tr || !tr.steps) return ""; const lines = tr.steps.map(s => `${" ".repeat((s.depth||0)+1)}${s.node} (${s.kind})${s.chosen ? " [" + s.chosen + "]" : ""}: ${s.summary}${s.next ? " -> " + s.next : ""}`); for (const g of tr.gaps || []) lines.push(` gap: ${g.node} saw ${g.on} = ${JSON.stringify(g.observed)} and had no branch for it`); return `trace: ${tr.tree}\n` + lines.join("\n"); }+async function ask(snapshot, answer) { const text = $("#text").value.trim(); if (!text && !snapshot) return; $("#go").disabled = true; const t0 = performance.now(); let r; try {- const res = snapshot ? await fetch("resume", {method:"POST", headers:{"Content-Type":"application/json"}, body: JSON.stringify({snapshot, answer})})- : await fetch("intent", {method:"POST", headers:{"Content-Type":"application/json"}, body: JSON.stringify({text})});- r = await res.json();+ const res = snapshot ? await api("resume", {method:"POST", headers:{"Content-Type":"application/json"}, body: JSON.stringify({snapshot, answer})})+ : await api("intent", {method:"POST", headers:{"Content-Type":"application/json"}, body: JSON.stringify({text})});+ r = res.status === 401 ? {status:"error", reason:"unauthorised", say:"This Pyrtan needs a token."} : await res.json();} catch (e) { r = {status:"error", reason:"unreachable", say:String(e)}; } const ms = Math.round(performance.now() - t0); const out = $("#out");@@ -131,11 +319,12 @@ async function ask(snapshot, answer) {if (r.say) html += `<div class="say">${esc(r.say)}</div>`; if (r.reason === "no-match" && r.output && r.output.proposal) { const p = r.output.proposal;- html += `<div class="gap">No tree answers this yet. The one that would: <code>${esc(p.name)}</code>, answering <code>${esc(p.trigger)}</code>.</div><pre>${esc(p.scaffold)}</pre>`;+ html += `<div class="gap">No tree answers this yet. The one that would: <code>${esc(p.name)}</code>, answering <code>${esc(p.trigger)}</code>. It is listed under <a href="#unanswered">Unanswered</a>.</div><pre>${esc(p.scaffold)}</pre>`;}- if (r.status === "asked") html += `<form class="resume"><input type="text" placeholder="${esc(r.question||"")}" autofocus><button>Answer</button></form>`;+ if (r.status === "asked") html += `<form class="row resume"><input type="text" placeholder="${esc(r.question||"")}" autofocus><button>Answer</button></form>`;if (r.output != null) html += `<details open><summary>output</summary><pre>${esc(JSON.stringify(r.output, null, 2))}</pre></details>`; if (r.trace) html += `<details open><summary>trace</summary><pre>${esc(renderTrace(r.trace))}</pre></details>`;+ if (r.reason === "gap" && r.output && r.output.gap) html += `<div class="mute" style="font-size:14px;margin-top:8px">This run met a gap. It is listed under <a href="#gaps">Gaps</a>; the branch to write is in <code>${esc(r.output.gap.tree)}</code>, node <code>${esc(r.output.gap.node)}</code>.</div>`;card.innerHTML = html; out.prepend(card); const rf = card.querySelector("form.resume");@@ -143,9 +332,18 @@ async function ask(snapshot, answer) {if (r.reason === "no-match") loadUnanswered(); $("#go").disabled = false; }+$("#ask").addEventListener("submit", (e) => { e.preventDefault(); ask(); });+$("#tries").addEventListener("click", (e) => { const a = e.target.closest("a[data-q]"); if (!a) return; e.preventDefault(); $("#text").value = a.dataset.q; location.hash = "#ask"; ask(); });+$("#connect-form").addEventListener("submit", (e) => {+ e.preventDefault();+ token = $("#hub-token").value;+ store.set(remote ? {url: $("#hub-url").value.trim(), token} : {token});+ load();+});+$("#forget").addEventListener("click", () => { store.clear(); base = ""; token = ""; $("#hub-url").value = ""; $("#hub-token").value = ""; $("#who").textContent = "not connected"; $("#caps tbody").innerHTML = ""; $("#tries").innerHTML = ""; problem(""); });load(); </script> </body> </html>-"""+""".replace("__VERSION__", __version__)diff --git a/src/pyrtan/runtime.py b/src/pyrtan/runtime.py index 5932203..6b1058e 100644 --- a/src/pyrtan/runtime.py +++ b/src/pyrtan/runtime.py@@ -37,7 +37,7 @@ from .grow import no_match_outcomefrom .registry import Ambiguous, NoMatch, Registry from .template import MISSING, TemplateError, render, resolve from .trace import Step, Trace-from .transport import Transport, TransportError, UrllibTransport+from .transport import EgressRefused, Transport, TransportError, UrllibTransportfrom .tree import Node, Tree, TreeError MAX_STEPS = 500@@ -229,15 +229,18 @@ class Runner:timeout = float(node.get("timeout", DEFAULT_TIMEOUT)) attempts = 1 + int(node.get("retries", DEFAULT_RETRIES)) if retry else 1 last_error: str | None = None+ resp = Nonefor attempt in range(attempts): try: resp = self.transport.request(method, url, headers, body, timeout) break except TransportError as e: last_error = str(e)+ if isinstance(e, EgressRefused):+ break # a policy refusal does not change on retryif attempt + 1 < attempts: time.sleep(min(0.2 * (attempt + 1), 1.0))- else:+ if resp is None:trace.add(Step(node.id, node.kind, f"{method} {url} failed: {last_error}")) return Outcome("error", tree.name, reason="transport", say=f"{node.id}: {last_error}") observed = resp.to_state() diff --git a/src/pyrtan/service.py b/src/pyrtan/service.py index 023cfce..1bcc883 100644 --- a/src/pyrtan/service.py +++ b/src/pyrtan/service.py@@ -19,19 +19,29 @@ Endpoints, all JSON:Sentences no tree answered, counted, each with a proposed tree. Each is a tree waiting to be written; ``pyrtan unanswered --at URL --scaffold DIR`` writes the files.-``GET /``- The demo page (:mod:`pyrtan.page`); ``GET /api`` is the JSON index.+``GET /`` and ``GET /console``+ The console (:mod:`pyrtan.page`); ``GET /api`` is the JSON index.A service may be given a hub (``--hub``). Then it registers itself on start, and a ``delegate`` to a tree it does not hold is forwarded to the hub.-The server is the standard library's threading HTTP server. It is meant for-a set of small processes on a trusted network, behind whatever ingress-the operator already runs; it does not do TLS or authentication itself.+**Access.** With no token, everything is open: the swarm is assumed to sit+on a trusted network behind the operator's ingress. With a token+(``--token`` or ``$PYRTAN_TOKEN``), every request except ``GET /health``,+the console and ``/api`` must carry ``Authorization: Bearer <token>``;+``/health`` then answers with status and counts only. A swarm shares one+token: a service sends it to its hub when it registers and the hub sends it+to peers when it forwards. **CORS** is off unless ``--cors`` names the+origins (or ``*``) a browser may call from; the console at pyrtan.com is+one such origin. The server does not do TLS; put it behind a proxy that+does when it leaves the LAN.++The server is the standard library's threading HTTP server.""" from __future__ import annotations+import hmacimport json import threading import time@@ -41,6 +51,8 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServerfrom pathlib import Path from typing import Any, Callable+from . import __version__+from .gaps import GapStorefrom .page import PAGE from .grow import Unanswered from .registry import Registry@@ -50,10 +62,18 @@ from .transport import Transport, UrllibTransportMAX_BODY = 1 << 20+# What answers without a token: the console, its index, and liveness.+PUBLIC_PATHS = frozenset({"/", "/index.html", "/console", "/api", "/health"})+# What /health says to a caller without a token, when one is required.+PUBLIC_HEALTH_KEYS = ("status", "name", "peers", "up", "stale", "down", "uptime_s")-def http_json(method: str, url: str, payload: Any = None, timeout: float = 15.0) -> tuple[int, Any]:++def http_json(method: str, url: str, payload: Any = None, timeout: float = 15.0, token: str | None = None) -> tuple[int, Any]:data = json.dumps(payload).encode("utf-8") if payload is not None else None- req = urllib.request.Request(url, data=data, method=method, headers={"Content-Type": "application/json", "Accept": "application/json"})+ headers = {"Content-Type": "application/json", "Accept": "application/json"}+ if token:+ headers["Authorization"] = f"Bearer {token}"+ req = urllib.request.Request(url, data=data, method=method, headers=headers)try: with urllib.request.urlopen(req, timeout=timeout) as resp: return resp.status, json.loads(resp.read().decode("utf-8") or "null")@@ -67,15 +87,16 @@ def http_json(method: str, url: str, payload: Any = None, timeout: float = 15.0)class RemoteResolver: """Sends what a local runner cannot answer to a hub or another service."""- def __init__(self, base_url: str) -> None:+ def __init__(self, base_url: str, token: str | None = None) -> None:self.base_url = base_url.rstrip("/")+ self.token = tokendef run_tree(self, name: str, inputs: dict[str, Any], depth: int) -> Outcome:- status, body = http_json("POST", f"{self.base_url}/run", {"tree": name, "inputs": inputs, "depth": depth})+ status, body = http_json("POST", f"{self.base_url}/run", {"tree": name, "inputs": inputs, "depth": depth}, token=self.token)return outcome_from_dict(body, name) if isinstance(body, dict) else Outcome("error", name, reason="remote", say=f"{self.base_url} answered {status}") def run_intent(self, text: str, depth: int) -> Outcome:- status, body = http_json("POST", f"{self.base_url}/intent", {"text": text, "depth": depth})+ status, body = http_json("POST", f"{self.base_url}/intent", {"text": text, "depth": depth}, token=self.token)return outcome_from_dict(body, "") if isinstance(body, dict) else Outcome("error", "", reason="remote", say=f"{self.base_url} answered {status}")@@ -101,14 +122,28 @@ def outcome_from_dict(d: dict[str, Any], fallback_tree: str = "") -> Outcome:class Service:- def __init__(self, registry: Registry, *, name: str = "pyrtan", transport: Transport | None = None, hub: str | None = None, env: dict[str, Any] | None = None, advertise: str | None = None) -> None:+ def __init__(+ self,+ registry: Registry,+ *,+ name: str = "pyrtan",+ transport: Transport | None = None,+ hub: str | None = None,+ env: dict[str, Any] | None = None,+ advertise: str | None = None,+ token: str | None = None,+ gaps: str | Path | GapStore | None = None,+ egress: str = "any",+ ) -> None:self.registry = registry self.name = name self.hub = hub.rstrip("/") if hub else None self.advertise = advertise+ self.token = token or None+ self.egress = egressself.started = time.time()- self.runner = Runner(registry, transport or UrllibTransport(), resolver=RemoteResolver(self.hub) if self.hub else None, env=env or {})- self.gaps: list[dict[str, Any]] = []+ self.runner = Runner(registry, transport or UrllibTransport(egress=egress), resolver=RemoteResolver(self.hub, self.token) if self.hub else None, env=env or {})+ self.gaps = gaps if isinstance(gaps, GapStore) else GapStore(gaps)self.unanswered = Unanswered() self.runs = 0 self._lock = threading.Lock()@@ -117,7 +152,10 @@ class Service:def health(self) -> dict[str, Any]: self.registry.reload()- return {"status": "ok", "name": self.name, "trees": self.registry.names(), "errors": self.registry.errors, "runs": self.runs, "gaps": len(self.gaps), "unanswered": len(self.unanswered), "uptime_s": int(time.time() - self.started)}+ health = {"status": "ok", "name": self.name, "trees": self.registry.names(), "errors": self.registry.errors, "runs": self.runs, "gaps": len(self.gaps), "unanswered": len(self.unanswered), "egress": self.egress, "uptime_s": int(time.time() - self.started)}+ if self.gaps.error:+ health["gaps_file_error"] = self.gaps.error+ return healthdef capabilities(self) -> dict[str, Any]: self.registry.reload()@@ -150,8 +188,8 @@ class Service:def _finish(self, outcome: Outcome, body: dict[str, Any]) -> dict[str, Any]: with self._lock: self.runs += 1- if outcome.trace is not None:- self.gaps.extend(outcome.trace.gaps)+ if outcome.trace is not None and outcome.trace.gaps:+ self.gaps.add(outcome.trace.gaps)return outcome.to_dict(with_trace=bool(body.get("trace", True))) # ------------------------------------------------------------ hub@@ -160,7 +198,7 @@ class Service:if not self.hub: return False try:- status, _ = http_json("POST", f"{self.hub}/register", {"url": my_url, "name": self.name})+ status, _ = http_json("POST", f"{self.hub}/register", {"url": my_url, "name": self.name}, token=self.token)return status == 200 except Exception: return False@@ -174,13 +212,13 @@ class Service:if method == "GET" and path == "/capabilities": return 200, self.capabilities() if method == "GET" and path == "/gaps":- return 200, {"gaps": self.gaps}+ return 200, {"gaps": self.gaps.all()}if method == "GET" and path == "/unanswered": return 200, {"unanswered": self.unanswered.to_list()}- if method == "GET" and path in ("/", "/index.html"):+ if method == "GET" and path in ("/", "/index.html", "/console"):return 200, PAGE if method == "GET" and path == "/api":- return 200, {"pyrtan": "service", "name": self.name, "endpoints": ["/health", "/capabilities", "/intent", "/run", "/resume", "/gaps", "/unanswered"]}+ return 200, {"pyrtan": "service", "name": self.name, "version": __version__, "auth": "bearer" if self.token else "none", "endpoints": ["/health", "/capabilities", "/intent", "/run", "/resume", "/gaps", "/unanswered"]}if method == "POST" and path == "/intent": return 200, self.intent(body) if method == "POST" and path == "/run":@@ -190,15 +228,71 @@ class Service:return 404, {"status": "error", "reason": "not-found", "say": f"No {method} {path} here."}-def make_handler(route: Callable[[str, str, dict[str, Any] | None], tuple[int, Any]], quiet: bool = False):+Route = Callable[[str, str, dict[str, Any] | None], tuple[int, Any]]+++def bearer_ok(header: str | None, token: str) -> bool:+ """Whether an ``Authorization`` header carries ``token`` (compared in constant time)."""+ if not header:+ return False+ scheme, _, value = header.strip().partition(" ")+ return scheme.lower() == "bearer" and hmac.compare_digest(value.strip().encode("utf-8"), token.encode("utf-8"))+++def make_handler(route: Route, quiet: bool = False, token: str | None = None, cors: list[str] | None = None):+ token = token or None+ origins = [o.rstrip("/") for o in (cors or []) if o]+ any_origin = "*" in origins+class Handler(BaseHTTPRequestHandler):- server_version = "pyrtan/0.1"+ server_version = f"pyrtan/{__version__}"def log_message(self, fmt: str, *args: Any) -> None: if not quiet: super().log_message(fmt, *args)- def _send(self, status: int, payload: Any) -> None:+ # -------------------------------------------------------- cors++ def _origin_allowed(self) -> str | None:+ origin = self.headers.get("Origin")+ if not origin or not origins:+ return None+ return origin if any_origin or origin.rstrip("/") in origins else None++ def _cors_headers(self) -> None:+ origin = self._origin_allowed()+ if origin is None:+ return+ self.send_header("Access-Control-Allow-Origin", origin)+ self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")+ self.send_header("Access-Control-Allow-Headers", "Authorization, Content-Type")+ self.send_header("Access-Control-Max-Age", "600")+ self.send_header("Vary", "Origin")++ def do_OPTIONS(self) -> None:+ self.send_response(204)+ self._cors_headers()+ self.send_header("Content-Length", "0")+ self.end_headers()++ # -------------------------------------------------------- auth++ def _authorised(self, path: str) -> bool:+ """True if the request may proceed; otherwise a 401 has been sent."""+ if token is None or path in PUBLIC_PATHS or bearer_ok(self.headers.get("Authorization"), token):+ return True+ self._send(401, {"status": "error", "reason": "unauthorised", "say": "This Pyrtan needs a token: send Authorization: Bearer <token>."}, extra={"WWW-Authenticate": 'Bearer realm="pyrtan"'})+ return False++ def _public_view(self, path: str, payload: Any) -> Any:+ """Trim /health for a caller with no token, when one is required."""+ if token is not None and path == "/health" and isinstance(payload, dict) and not bearer_ok(self.headers.get("Authorization"), token):+ return {k: v for k, v in payload.items() if k in PUBLIC_HEALTH_KEYS}+ return payload++ # -------------------------------------------------------- i/o++ def _send(self, status: int, payload: Any, extra: dict[str, str] | None = None) -> None:if isinstance(payload, str): data = payload.encode("utf-8") ctype = "text/html; charset=utf-8"@@ -208,14 +302,27 @@ def make_handler(route: Callable[[str, str, dict[str, Any] | None], tuple[int, Aself.send_response(status) self.send_header("Content-Type", ctype) self.send_header("Content-Length", str(len(data)))+ self.send_header("X-Content-Type-Options", "nosniff")+ for k, v in (extra or {}).items():+ self.send_header(k, v)+ self._cors_headers()self.end_headers()- self.wfile.write(data)+ try:+ self.wfile.write(data)+ except (BrokenPipeError, ConnectionResetError):+ pass # the caller went away; nothing to do about itdef do_GET(self) -> None:- status, payload = route("GET", self.path.split("?", 1)[0], None)- self._send(status, payload)+ path = self.path.split("?", 1)[0]+ if not self._authorised(path):+ return+ status, payload = route("GET", path, None)+ self._send(status, self._public_view(path, payload))def do_POST(self) -> None:+ path = self.path.split("?", 1)[0]+ if not self._authorised(path):+ returnlength = int(self.headers.get("Content-Length") or 0) if length > MAX_BODY: self._send(413, {"status": "error", "reason": "too-large"})@@ -229,17 +336,20 @@ def make_handler(route: Callable[[str, str, dict[str, Any] | None], tuple[int, Aif not isinstance(body, dict): self._send(400, {"status": "error", "reason": "bad-json", "say": "The body must be a JSON object."}) return- status, payload = route("POST", self.path.split("?", 1)[0], body)+ status, payload = route("POST", path, body)self._send(status, payload) return Handler class Server:- """A running HTTP server around a route function, for a service or a hub."""+ """A running HTTP server around a route function, for a service or a hub.- def __init__(self, route: Callable[[str, str, dict[str, Any] | None], tuple[int, Any]], host: str = "127.0.0.1", port: int = 0, quiet: bool = False) -> None:- self.httpd = ThreadingHTTPServer((host, port), make_handler(route, quiet))+ ``token`` requires ``Authorization: Bearer`` on everything but the public+ paths; ``cors`` lists the browser origins allowed to call (``*`` for any)."""++ def __init__(self, route: Route, host: str = "127.0.0.1", port: int = 0, quiet: bool = False, *, token: str | None = None, cors: list[str] | None = None) -> None:+ self.httpd = ThreadingHTTPServer((host, port), make_handler(route, quiet, token, cors))self.httpd.daemon_threads = True self.thread: threading.Thread | None = None@@ -260,15 +370,40 @@ class Server:def serve_forever(self) -> None: self.httpd.serve_forever()+ def wait(self) -> None:+ """Block until the server started with :meth:`start` stops."""+ if self.thread is None:+ self.serve_forever()+ return+ while self.thread.is_alive():+ self.thread.join(0.5)+def stop(self) -> None: self.httpd.shutdown() self.httpd.server_close()-def serve(paths: list[str | Path], host: str = "127.0.0.1", port: int = 8701, *, name: str = "pyrtan", hub: str | None = None, env: dict[str, Any] | None = None, advertise: str | None = None, quiet: bool = False) -> tuple[Service, Server]:+def serve(+ paths: list[str | Path],+ host: str = "127.0.0.1",+ port: int = 8701,+ *,+ name: str = "pyrtan",+ hub: str | None = None,+ env: dict[str, Any] | None = None,+ advertise: str | None = None,+ quiet: bool = False,+ token: str | None = None,+ cors: list[str] | None = None,+ gaps: str | Path | None = None,+ egress: str = "any",+) -> tuple[Service, Server]:registry = Registry.from_paths(paths)- service = Service(registry, name=name, hub=hub, env=env, advertise=advertise)- server = Server(service.route, host, port, quiet)+ service = Service(registry, name=name, hub=hub, env=env, advertise=advertise, token=token, gaps=gaps, egress=egress)+ server = Server(service.route, host, port, quiet, token=token, cors=cors).start()+ # The server is already answering when it registers: the hub checks a new+ # peer's /capabilities inside /register, and would otherwise wait on a+ # service that is itself waiting for the hub's answer.if hub: service.register(advertise or server.url) return service, server diff --git a/src/pyrtan/transport.py b/src/pyrtan/transport.py index aa9fa49..b55096e 100644 --- a/src/pyrtan/transport.py +++ b/src/pyrtan/transport.py@@ -10,19 +10,27 @@ network, and :class:`RecordingTransport` wraps a real transport to write sucha fixture. The runtime, not the transport, enforces a tree's host allow-list; a-transport only moves bytes.+transport only moves bytes. The one policy a transport does hold is+*egress*: an operator who hosts trees for other people can tell the real+transport to reach only public addresses, so a tree with ``hosts = ["*"]``+cannot be pointed at the host's own network.""" from __future__ import annotations+import ipaddressimport json+import socketimport time import urllib.error+import urllib.parseimport urllib.request from dataclasses import dataclass, field from pathlib import Path from typing import Any, Protocol+EGRESS_POLICIES = ("any", "public")+@dataclass class Response:@@ -60,21 +68,84 @@ class TransportError(RuntimeError):pass+class EgressRefused(TransportError):+ """The egress policy refused a URL before any connection was made."""+++def check_egress(url: str, policy: str = "any", *, resolve=socket.getaddrinfo) -> None:+ """Raise :class:`EgressRefused` if ``url`` is not allowed under ``policy``.++ ``any`` allows every http(s) URL. ``public`` also refuses a host that+ resolves to a loopback, private, link-local, multicast, reserved or+ unspecified address: the operator's LAN, the container network, and the+ cloud metadata endpoint. The check resolves the name here and the client+ resolves it again to connect, so a name that changes answers between the+ two can slip past; a host that must be sealed should also firewall.+ """+ if policy not in EGRESS_POLICIES:+ raise ValueError(f"egress policy must be one of {EGRESS_POLICIES}, got {policy!r}")+ parts = urllib.parse.urlsplit(url)+ if parts.scheme not in ("http", "https"):+ raise EgressRefused(f"{url}: only http and https are allowed")+ host = parts.hostname+ if not host:+ raise EgressRefused(f"{url}: no host")+ if policy == "any":+ return+ if host.lower() in ("localhost",) or host.lower().endswith(".localhost"):+ raise EgressRefused(f"{url}: {host} is not a public address")+ try:+ addresses = {info[4][0] for info in resolve(host, parts.port or (443 if parts.scheme == "https" else 80), proto=socket.IPPROTO_TCP)}+ except (socket.gaierror, OSError) as e:+ raise EgressRefused(f"{url}: cannot resolve {host}: {e}") from e+ for text in addresses:+ try:+ address = ipaddress.ip_address(text.split("%", 1)[0])+ except ValueError:+ raise EgressRefused(f"{url}: {host} resolved to {text!r}, which is not an address")+ if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped is not None:+ address = address.ipv4_mapped+ if not address.is_global or address.is_multicast:+ raise EgressRefused(f"{url}: {host} resolves to {address}, which is not a public address")+++class _EgressRedirectHandler(urllib.request.HTTPRedirectHandler):+ """Apply the egress policy to every redirect target as well as the first URL."""++ def __init__(self, policy: str) -> None:+ super().__init__()+ self.policy = policy++ def redirect_request(self, req, fp, code, msg, headers, newurl): # type: ignore[override]+ check_egress(newurl, self.policy)+ return super().redirect_request(req, fp, code, msg, headers, newurl)++class UrllibTransport: """The standard library's HTTP client, with a fixed user agent and no- cookie jar: every request is stateless unless a tree sends state itself."""+ cookie jar: every request is stateless unless a tree sends state itself.++ ``egress`` is ``any`` (the default: every host a tree names) or+ ``public`` (refuse private, loopback, link-local and metadata addresses,+ on the first URL and on every redirect)."""- def __init__(self, user_agent: str = "pyrtan/0.1 (+https://github.com/b-i-t-8/pyrtan)") -> None:+ def __init__(self, user_agent: str = "pyrtan/0.2 (+https://github.com/b-i-t-8/pyrtan)", egress: str = "any") -> None:+ if egress not in EGRESS_POLICIES:+ raise ValueError(f"egress policy must be one of {EGRESS_POLICIES}, got {egress!r}")self.user_agent = user_agent+ self.egress = egress+ self._opener = urllib.request.build_opener(_EgressRedirectHandler(egress)) if egress != "any" else Nonedef request(self, method: str, url: str, headers: dict[str, str] | None = None, body: str | None = None, timeout: float = 10.0) -> Response:+ check_egress(url, self.egress)hdrs = {"User-Agent": self.user_agent, "Accept": "application/json, text/html;q=0.9, */*;q=0.8"} hdrs.update(headers or {}) data = body.encode("utf-8") if body is not None else None req = urllib.request.Request(url, data=data, headers=hdrs, method=method.upper()) started = time.monotonic()+ opener = self._opener.open if self._opener is not None else urllib.request.urlopentry:- with urllib.request.urlopen(req, timeout=timeout) as resp:+ with opener(req, timeout=timeout) as resp:raw = resp.read() return Response(resp.status, {k.lower(): v for k, v in resp.headers.items()}, _decode(raw, resp.headers.get_content_charset()), resp.geturl(), int((time.monotonic() - started) * 1000)) except urllib.error.HTTPError as e: diff --git a/src/pyrtan/tree.py b/src/pyrtan/tree.py index 8b889ec..b4850e6 100644 --- a/src/pyrtan/tree.py +++ b/src/pyrtan/tree.py@@ -161,6 +161,7 @@ class Tree:tags: list[str] = field(default_factory=list) source: str | None = None raw: dict[str, Any] = field(default_factory=dict, repr=False)+ license: str = ""def node(self, node_id: str) -> Node: try:@@ -209,6 +210,7 @@ class Tree:"description": self.description, "tags": self.tags, "hosts": self.hosts,+ "license": self.license,"triggers": [t.pattern for t in self.triggers], "inputs": { n: {k: v for k, v in {"type": s.type, "default": s.default, "required": s.required and s.default is None, "description": s.description}.items() if v not in (None, "", False)}@@ -250,6 +252,9 @@ def build(raw: dict[str, Any], source: str | None = None) -> Tree:tags = raw.get("tags", []) if not isinstance(tags, list) or not all(isinstance(t, str) for t in tags): raise TreeError(f"{name}: 'tags' must be a list of strings")+ license_ = raw.get("license", "")+ if not isinstance(license_, str):+ raise TreeError(f"{name}: 'license' must be a string such as \"MIT\" (an SPDX identifier)")triggers: list[Trigger] = [] for i, t in enumerate(raw.get("triggers", [])):@@ -305,7 +310,7 @@ def build(raw: dict[str, Any], source: str | None = None) -> Tree:hint = " (top-level keys such as 'start' must come before the first [table] in TOML)" if _buried(raw, "start") else "" raise TreeError(f"{name}: 'start' must name a node{hint}")- tree = Tree(name, description, version, hosts, triggers, inputs, start, nodes, tags, source, raw)+ tree = Tree(name, description, version, hosts, triggers, inputs, start, nodes, tags, source, raw, license_.strip())errors = [p for p in check(tree) if p.level == "error"] if errors: raise TreeError(f"{name}: " + "; ".join(f"{p.where}: {p.message}" for p in errors)) diff --git a/tests/test_access.py b/tests/test_access.py new file mode 100644 index 0000000..c24ef66 --- /dev/null +++ b/tests/test_access.py@@ -0,0 +1,246 @@+"""Tokens, CORS, persistent gaps and the egress policy: what an operator+turns on when a swarm leaves the LAN or hosts trees for other people."""++import json+import socket+import urllib.error+import urllib.request++import pytest++from pyrtan import Registry, RecordedTransport+from pyrtan.gaps import GapStore+from pyrtan.hub import Hub+from pyrtan.service import Server, Service, http_json+from pyrtan.transport import EgressRefused, UrllibTransport, check_egress+from conftest import EXAMPLES, FIXTURES++GAPPY = """+name = "gappy"+description = "decides on a value it was given, with no else"+start = "look"++[[triggers]]+pattern = "gappy {value:int}"++[inputs.value]+type = "int"++[nodes.look]+kind = "decide"+on = "inputs.value"+branches = [ { when = { eq = 1 }, then = "one" } ]++[nodes.one]+kind = "done"+say = "one"+"""+++def _raw(url: str, headers: dict | None = None, method: str = "GET", data: bytes | None = None):+ req = urllib.request.Request(url, data=data, method=method, headers=headers or {})+ try:+ with urllib.request.urlopen(req) as resp:+ return resp.status, dict(resp.headers), resp.read()+ except urllib.error.HTTPError as e:+ return e.code, dict(e.headers), e.read()+++@pytest.fixture+def locked_service():+ reg = Registry.from_paths([EXAMPLES])+ svc = Service(reg, name="locked", transport=RecordedTransport.from_file(FIXTURES / "examples.json"), token="s3cret")+ server = Server(svc.route, quiet=True, token="s3cret", cors=["https://pyrtan.com"]).start()+ yield svc, server+ server.stop()+++def test_token_required_except_public_paths(locked_service):+ _, server = locked_service+ status, headers, body = _raw(f"{server.url}/capabilities")+ assert status == 401 and headers["WWW-Authenticate"].startswith("Bearer")+ assert json.loads(body)["reason"] == "unauthorised"+ assert _raw(f"{server.url}/intent", {"Content-Type": "application/json"}, "POST", b'{"text":"weather in Oslo"}')[0] == 401+ assert _raw(f"{server.url}/capabilities", {"Authorization": "Bearer wrong"})[0] == 401+ assert _raw(f"{server.url}/capabilities", {"Authorization": "Basic s3cret"})[0] == 401+ status, _, body = _raw(f"{server.url}/capabilities", {"Authorization": "Bearer s3cret"})+ assert status == 200 and {t["name"] for t in json.loads(body)["trees"]} >= {"weather"}+ assert http_json("POST", f"{server.url}/intent", {"text": "weather in Oslo"}, token="s3cret")[1]["status"] == "done"+ # Public: the console, its index, and liveness.+ assert _raw(f"{server.url}/")[0] == 200+ assert _raw(f"{server.url}/console")[0] == 200+ status, _, body = _raw(f"{server.url}/api")+ assert status == 200 and json.loads(body)["auth"] == "bearer"+++def test_health_is_trimmed_without_the_token(locked_service):+ _, server = locked_service+ status, _, body = _raw(f"{server.url}/health")+ public = json.loads(body)+ assert status == 200 and public["status"] == "ok" and "trees" not in public and "errors" not in public+ status, _, body = _raw(f"{server.url}/health", {"Authorization": "Bearer s3cret"})+ assert "weather" in json.loads(body)["trees"]+++def test_cors_only_for_listed_origins(locked_service):+ _, server = locked_service+ status, headers, _ = _raw(f"{server.url}/intent", {"Origin": "https://pyrtan.com", "Access-Control-Request-Method": "POST", "Access-Control-Request-Headers": "authorization,content-type"}, "OPTIONS")+ assert status == 204+ assert headers["Access-Control-Allow-Origin"] == "https://pyrtan.com"+ assert "Authorization" in headers["Access-Control-Allow-Headers"]+ status, headers, _ = _raw(f"{server.url}/intent", {"Origin": "https://evil.example"}, "OPTIONS")+ assert status == 204 and "Access-Control-Allow-Origin" not in headers+ _, headers, _ = _raw(f"{server.url}/api", {"Origin": "https://pyrtan.com"})+ assert headers["Access-Control-Allow-Origin"] == "https://pyrtan.com" and headers["Vary"] == "Origin"+ _, headers, _ = _raw(f"{server.url}/api")+ assert "Access-Control-Allow-Origin" not in headers+++def test_cors_star_echoes_any_origin():+ reg = Registry.from_paths([EXAMPLES])+ server = Server(Service(reg).route, quiet=True, cors=["*"]).start()+ try:+ _, headers, _ = _raw(f"{server.url}/api", {"Origin": "http://localhost:5173"})+ assert headers["Access-Control-Allow-Origin"] == "http://localhost:5173"+ finally:+ server.stop()+++def test_swarm_shares_one_token(tmp_path):+ token = "swarm-token"+ reg_a = Registry.from_paths([EXAMPLES])+ svc_a = Service(reg_a, name="a", transport=RecordedTransport.from_file(FIXTURES / "examples.json"), token=token)+ srv_a = Server(svc_a.route, quiet=True, token=token).start()+ hub = Hub(poll_every=60, token=token)+ hub_srv = Server(hub.route, quiet=True, token=token).start()+ try:+ # A stranger cannot join the swarm or use it.+ assert http_json("POST", f"{hub_srv.url}/register", {"url": srv_a.url})[0] == 401+ assert http_json("POST", f"{hub_srv.url}/intent", {"text": "weather in Oslo"})[0] == 401+ # A service with the token registers; the hub polls it with the token.+ svc_a.hub = hub_srv.url+ assert svc_a.register(srv_a.url)+ _, peers = http_json("GET", f"{hub_srv.url}/peers", token=token)+ assert peers["peers"][0]["link"] == "up" and "weather" in peers["peers"][0]["trees"]+ # And forwards with it.+ _, out = http_json("POST", f"{hub_srv.url}/intent", {"text": "weather in Oslo"}, token=token)+ assert out["status"] == "done" and out["peer"] == srv_a.url+ # A hub without the token sees the peer as down.+ other = Hub(poll_every=60)+ p = other.add_peer(srv_a.url)+ assert p.link == "down" and "401" in p.last_error+ finally:+ hub.stop()+ hub_srv.stop()+ srv_a.stop()+++# ---------------------------------------------------------------- gaps+++def test_gaps_persist_and_count(tmp_path):+ (tmp_path / "gappy.toml").write_text(GAPPY)+ path = tmp_path / "gaps.jsonl"+ reg = Registry.from_paths([tmp_path])+ svc = Service(reg, name="g", gaps=path)+ server = Server(svc.route, quiet=True).start()+ try:+ assert http_json("POST", f"{server.url}/intent", {"text": "gappy 1"})[1]["status"] == "done"+ for _ in range(3):+ out = http_json("POST", f"{server.url}/intent", {"text": "gappy 7"})[1]+ assert out["status"] == "error" and out["reason"] == "gap"+ http_json("POST", f"{server.url}/intent", {"text": "gappy 8"})+ _, gaps = http_json("GET", f"{server.url}/gaps")+ assert [(g["node"], g["observed"], g["count"]) for g in gaps["gaps"]] == [("look", 7, 3), ("look", 8, 1)]+ assert http_json("GET", f"{server.url}/health")[1]["gaps"] == 2+ finally:+ server.stop()+ assert len(path.read_text().splitlines()) == 4+ # A new service over the same file starts with what the old one saw.+ again = GapStore(path)+ assert [(g["observed"], g["count"]) for g in again.all()] == [(7, 3), (8, 1)]+ again.add([{"tree": "gappy", "node": "look", "on": "inputs.value", "observed": 8}])+ assert [(g["observed"], g["count"]) for g in again.all()] == [(7, 3), (8, 2)]+++def test_gap_store_keeps_going_when_the_file_cannot_be_written(tmp_path, capsys):+ blocked = tmp_path / "dir"+ blocked.mkdir()+ store = GapStore(blocked) # a directory, not a file: reading and every append fail+ store.add([{"tree": "t", "node": "n", "on": "x", "observed": 1}])+ err = capsys.readouterr().err+ assert len(store) == 1 and store.error and "cannot read gaps" in err+ # A store over a file that cannot be written warns once and keeps the gap.+ missing_dir = tmp_path / "nope"+ missing_dir.write_text("") # a file where the parent directory should be+ store = GapStore(missing_dir / "gaps.jsonl")+ store.add([{"tree": "t", "node": "n", "on": "x", "observed": 1}] * 2)+ assert store.all()[0]["count"] == 2 and store.error+ assert capsys.readouterr().err.count("cannot write gaps") == 1+++def test_gap_store_in_memory_when_no_path():+ store = GapStore()+ store.add([{"tree": "t", "node": "n", "on": "x", "observed": "a"}] * 2)+ assert len(store) == 1 and store.all()[0]["count"] == 2 and store.path is None+++# ---------------------------------------------------------------- egress+++def _resolver(*answers: str):+ def resolve(host, port, proto=0):+ return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (a, port)) for a in answers]++ return resolve+++def test_egress_any_allows_everything_http():+ check_egress("http://127.0.0.1:8701/health", "any")+ check_egress("http://10.0.0.5/", "any", resolve=_resolver("10.0.0.5"))+ with pytest.raises(EgressRefused):+ check_egress("ftp://example.com/", "any")+++@pytest.mark.parametrize("address", ["127.0.0.1", "10.1.2.3", "172.16.0.9", "192.168.1.20", "169.254.169.254", "0.0.0.0", "::1", "fd00::1", "fe80::1", "::ffff:10.0.0.1", "224.0.0.1"])+def test_egress_public_refuses_private_addresses(address):+ with pytest.raises(EgressRefused):+ check_egress("https://api.example.com/", "public", resolve=_resolver(address))+++def test_egress_public_allows_public_addresses():+ check_egress("https://api.example.com/", "public", resolve=_resolver("93.184.216.34"))+ check_egress("https://api.example.com/", "public", resolve=_resolver("2606:2800:220:1:248:1893:25c8:1946"))+++def test_egress_public_refuses_localhost_names_and_mixed_answers():+ with pytest.raises(EgressRefused):+ check_egress("http://localhost:8700/", "public")+ with pytest.raises(EgressRefused):+ check_egress("http://thing.localhost/", "public")+ # One private answer among public ones is enough to refuse.+ with pytest.raises(EgressRefused):+ check_egress("https://api.example.com/", "public", resolve=_resolver("93.184.216.34", "10.0.0.1"))+ with pytest.raises(EgressRefused):+ check_egress("https://nope.invalid/", "public", resolve=lambda *a, **k: (_ for _ in ()).throw(socket.gaierror("no such host")))+++def test_egress_public_transport_refuses_before_connecting():+ with pytest.raises(ValueError):+ UrllibTransport(egress="strict")+ with pytest.raises(EgressRefused):+ UrllibTransport(egress="public").request("GET", "http://127.0.0.1:9/")+++def test_service_with_public_egress_reports_it_and_refuses_a_lan_host(tmp_path):+ (tmp_path / "status.toml").write_text((EXAMPLES / "site-status" / "site-status.toml").read_text())+ svc = Service(Registry.from_paths([tmp_path]), name="sealed", egress="public")+ server = Server(svc.route, quiet=True).start()+ try:+ assert http_json("GET", f"{server.url}/health")[1]["egress"] == "public"+ _, out = http_json("POST", f"{server.url}/run", {"tree": "site-status", "inputs": {"host": "127.0.0.1"}})+ assert out["status"] == "error" and out["reason"] == "transport" and "not a public address" in out["say"]+ # Refused once, not retried: one step in the trace.+ assert [s["node"] for s in out["trace"]["steps"]] == ["probe"]+ finally:+ server.stop()diff --git a/tests/test_catalog.py b/tests/test_catalog.py new file mode 100644 index 0000000..8319aa4 --- /dev/null +++ b/tests/test_catalog.py@@ -0,0 +1,141 @@+"""The catalog: building an index from a directory, and pulling from one+over HTTP with every check that keeps a bad file out."""++import hashlib+import json+import shutil+import threading+from functools import partial+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer++import pytest++from pyrtan.catalog import Catalog, CatalogError, build_index+from pyrtan.cli import main+from conftest import EXAMPLES+++@pytest.fixture+def catalog_dir(tmp_path):+ root = tmp_path / "catalog"+ shutil.copytree(EXAMPLES, root)+ index, problems = build_index(root)+ assert not problems, problems+ (root / "index.json").write_text(json.dumps(index))+ return root+++@pytest.fixture+def served(catalog_dir):+ class Quiet(SimpleHTTPRequestHandler):+ def log_message(self, *a):+ pass++ httpd = ThreadingHTTPServer(("127.0.0.1", 0), partial(Quiet, directory=str(catalog_dir)))+ threading.Thread(target=httpd.serve_forever, daemon=True).start()+ yield f"http://127.0.0.1:{httpd.server_address[1]}/"+ httpd.shutdown()+++def test_build_index_describes_every_tree(catalog_dir):+ index = json.loads((catalog_dir / "index.json").read_text())+ assert index["pyrtan-catalog"] == 1 and index["count"] == 5+ weather = next(e for e in index["trees"] if e["name"] == "weather")+ assert weather["path"] == "weather/weather.toml" and weather["license"] == "MIT"+ assert weather["sha256"] == hashlib.sha256((catalog_dir / "weather/weather.toml").read_bytes()).hexdigest()+ assert "[what's the] weather in {place}" in weather["triggers"]+ assert weather["inputs"]["place"]["required"] is True+++def test_build_index_leaves_out_what_does_not_belong(tmp_path):+ (tmp_path / "ok.toml").write_text('name = "ok"\nlicense = "MIT"\nstart = "s"\n[[triggers]]\npattern = "ok now"\n[nodes.s]\nkind = "done"\nsay = "ok"\n')+ (tmp_path / "zz-dup.toml").write_text('name = "ok"\nlicense = "MIT"\nstart = "s"\n[[triggers]]\npattern = "ok again"\n[nodes.s]\nkind = "done"\nsay = "ok"\n')+ (tmp_path / "unlicensed.toml").write_text('name = "free"\nstart = "s"\n[[triggers]]\npattern = "free now"\n[nodes.s]\nkind = "done"\nsay = "ok"\n')+ (tmp_path / "broken.toml").write_text("name = \n")+ (tmp_path / ".hidden").mkdir()+ (tmp_path / ".hidden" / "x.toml").write_text("nope")+ index, problems = build_index(tmp_path)+ assert [e["name"] for e in index["trees"]] == ["ok"]+ messages = {p.path: p.message for p in problems}+ assert "already at ok.toml" in messages["zz-dup.toml"]+ assert "license" in messages["unlicensed.toml"]+ assert "broken.toml" in messages+ assert "x.toml" not in " ".join(messages)+ index, problems = build_index(tmp_path, require_license=False)+ assert {e["name"] for e in index["trees"]} == {"ok", "free"}+++def test_search_and_find(served):+ cat = Catalog(served)+ assert {e["name"] for e in cat.search("")} == {"weather", "hacker-news", "hacker-news-item", "site-status", "github-release"}+ assert [e["name"] for e in cat.search("open-meteo")] == ["weather"]+ assert [e["name"] for e in cat.search("news stories")] == ["hacker-news"]+ assert cat.search("nothing like this") == []+ assert cat.find("weather")["version"] == 1+ assert cat.find("weather@1")["name"] == "weather"+ with pytest.raises(CatalogError, match="has version 1, not 4"):+ cat.find("weather@4")+ with pytest.raises(CatalogError, match="not in the catalog"):+ cat.find("nope")+ with pytest.raises(CatalogError, match="NAME"):+ cat.find("Not Valid")+++def test_pull_writes_verified_file_and_refuses_to_clobber(served, tmp_path):+ cat = Catalog(served)+ target = cat.pull("weather", tmp_path / "trees")+ assert target == tmp_path / "trees" / "weather.toml"+ assert target.read_bytes() == (EXAMPLES / "weather" / "weather.toml").read_bytes()+ assert cat.pull("weather", tmp_path / "trees") == target # same bytes: fine+ target.write_text(target.read_text() + "\n# mine\n")+ with pytest.raises(CatalogError, match="--force"):+ cat.pull("weather", tmp_path / "trees")+ cat.pull("weather", tmp_path / "trees", force=True)+ assert "# mine" not in target.read_text()+++def test_pull_refuses_a_file_that_does_not_match_the_index(catalog_dir, served, tmp_path):+ index = json.loads((catalog_dir / "index.json").read_text())+ # Tamper with the served file after the index was built.+ path = catalog_dir / "weather" / "weather.toml"+ path.write_text(path.read_text().replace("api.open-meteo.com", "evil.example"))+ with pytest.raises(CatalogError, match="does not match the index"):+ Catalog(served).pull("weather", tmp_path)+ # An index that points outside the catalog is refused too.+ for e in index["trees"]:+ if e["name"] == "weather":+ e["path"] = "../../etc/passwd"+ (catalog_dir / "index.json").write_text(json.dumps(index))+ with pytest.raises(CatalogError, match="outside the catalog"):+ Catalog(served).pull("weather", tmp_path)+ # An index whose entry says one name while the file says another.+ index = json.loads((catalog_dir / "index.json").read_text())+ hn = next(e for e in index["trees"] if e["name"] == "hacker-news")+ hn["path"] = "github/github-release.toml"+ hn["sha256"] = hashlib.sha256((catalog_dir / "github/github-release.toml").read_bytes()).hexdigest()+ (catalog_dir / "index.json").write_text(json.dumps(index))+ with pytest.raises(CatalogError, match="says it is 'github-release'"):+ Catalog(served).pull("hacker-news", tmp_path)+++def test_not_a_catalog(served, tmp_path):+ (tmp_path / "index.json").write_text("{}")+ with pytest.raises(CatalogError, match="not a pyrtan catalog"):+ Catalog(str(tmp_path)).entries()+ with pytest.raises(CatalogError):+ Catalog(served + "missing/").entries()+++def test_cli_index_search_pull(catalog_dir, served, tmp_path, capsys):+ out = tmp_path / "idx.json"+ assert main(["index", str(catalog_dir), "--out", str(out)]) == 0+ assert json.loads(out.read_text())["count"] == 5+ assert main(["search", "weather", "--catalog", served]) == 0+ assert "weather@1" in capsys.readouterr().out+ assert main(["search", "zzz", "--catalog", served]) == 1+ assert main(["pull", "site-status", "weather@1", "--catalog", served, "--dir", str(tmp_path / "t")]) == 0+ assert (tmp_path / "t" / "site-status.toml").exists() and (tmp_path / "t" / "weather.toml").exists()+ assert main(["pull", "weather@9", "--catalog", served, "--dir", str(tmp_path / "t")]) == 1+ assert "not 9" in capsys.readouterr().err+ assert main(["console", "--out", str(tmp_path / "site" / "console.html")]) == 0+ assert "<title>Pyrtan</title>" in (tmp_path / "site" / "console.html").read_text()diff --git a/tests/test_service_hub.py b/tests/test_service_hub.py index 6493b03..4ab46ef 100644 --- a/tests/test_service_hub.py +++ b/tests/test_service_hub.py@@ -7,7 +7,7 @@ import urllib.requestimport pytest-from pyrtan import Registry, RecordedTransport+from pyrtan import Registry, RecordedTransport, __version__from pyrtan.hub import Hub from pyrtan.service import Server, Service, http_json from pyrtan.tree import load_tree_text@@ -218,7 +218,8 @@ def test_demo_page_is_served_by_service_and_hub(example_service):_, server = example_service with urllib.request.urlopen(f"{server.url}/") as resp: assert resp.headers.get_content_type() == "text/html"- assert b"<title>Pyrtan</title>" in resp.read()+ page = resp.read()+ assert b"<title>Pyrtan</title>" in page and f"console v{__version__}".encode() in pagestatus, api = http_json("GET", f"{server.url}/api") assert status == 200 and api["pyrtan"] == "service" hub = Hub() diff --git a/uv.lock b/uv.lock index 20629ab..4beb188 100644 --- a/uv.lock +++ b/uv.lock@@ -147,7 +147,7 @@ wheels = [[[package]] name = "pyrtan"-version = "0.1.0"+version = "0.2.0"source = { editable = "." } [package.optional-dependencies]