pyrtan / src / pyrtan / extract.py
4,563 bytes · last changed 2026-10-25 in Initial public release
| 1 | """Pulling values out of what was observed. |
| 2 | |
| 3 | An ``extract`` entry names where the value goes (``into``) and one way of |
| 4 | finding it: |
| 5 | |
| 6 | { into = "title", json = "json.items[0].title" } |
| 7 | { into = "price", regex = "\\$([0-9.]+)", group = 1 } |
| 8 | { into = "heading", css = "h1" } # text of first match |
| 9 | { into = "links", css = "a.story", attr = "href", all = true } |
| 10 | { into = "rows", css = "table tr", all = true, fields = { name = "td:nth-child(1)", qty = "td:nth-child(2)" } } |
| 11 | |
| 12 | The source is the observation's state (``json``, ``body``, ``status`` …) or |
| 13 | ``from`` when given. Extraction never raises on a miss; the value is simply |
| 14 | absent, and a ``decide`` node downstream says what that means. |
| 15 | """ |
| 16 | |
| 17 | from __future__ import annotations |
| 18 | |
| 19 | import re |
| 20 | from typing import Any |
| 21 | |
| 22 | from . import html as htmlmod |
| 23 | from .template import MISSING, resolve |
| 24 | |
| 25 | |
| 26 | class ExtractError(ValueError): |
| 27 | pass |
| 28 | |
| 29 | |
| 30 | WAYS = {"json", "regex", "css", "path"} |
| 31 | |
| 32 | |
| 33 | def validate(entry: Any) -> None: |
| 34 | if not isinstance(entry, dict): |
| 35 | raise ExtractError("an extract entry is a table") |
| 36 | if "into" not in entry or not isinstance(entry["into"], str): |
| 37 | raise ExtractError("extract entry needs a string 'into'") |
| 38 | ways = WAYS & set(entry) |
| 39 | if len(ways) != 1: |
| 40 | raise ExtractError(f"extract entry {entry['into']!r} needs exactly one of {sorted(WAYS)}") |
| 41 | if "regex" in entry: |
| 42 | try: |
| 43 | re.compile(entry["regex"]) |
| 44 | except re.error as e: |
| 45 | raise ExtractError(f"bad regex in extract {entry['into']!r}: {e}") from e |
| 46 | if "css" in entry: |
| 47 | try: |
| 48 | htmlmod.compile_selector(entry["css"]) |
| 49 | except htmlmod.SelectorError as e: |
| 50 | raise ExtractError(f"bad selector in extract {entry['into']!r}: {e}") from e |
| 51 | for sub in (entry.get("fields") or {}).values(): |
| 52 | try: |
| 53 | htmlmod.compile_selector(sub if isinstance(sub, str) else sub["css"]) |
| 54 | except (htmlmod.SelectorError, KeyError, TypeError) as e: |
| 55 | raise ExtractError(f"bad field selector in extract {entry['into']!r}: {e}") from e |
| 56 | |
| 57 | |
| 58 | def extract(entry: dict[str, Any], source: dict[str, Any], doc_cache: dict[int, htmlmod.Element] | None = None) -> Any: |
| 59 | """Return the extracted value, or :data:`MISSING`.""" |
| 60 | if "json" in entry or "path" in entry: |
| 61 | path = entry.get("json", entry.get("path")) |
| 62 | return resolve(path, source) |
| 63 | if "regex" in entry: |
| 64 | text = _text_source(entry, source) |
| 65 | if text is MISSING: |
| 66 | return MISSING |
| 67 | pattern = re.compile(entry["regex"], re.DOTALL if entry.get("dotall") else 0) |
| 68 | group = entry.get("group", 1 if pattern.groups else 0) |
| 69 | if entry.get("all"): |
| 70 | return [m.group(group) for m in pattern.finditer(text)] |
| 71 | m = pattern.search(text) |
| 72 | return m.group(group) if m else MISSING |
| 73 | if "css" in entry: |
| 74 | text = _text_source(entry, source) |
| 75 | if text is MISSING: |
| 76 | return MISSING |
| 77 | key = id(text) |
| 78 | if doc_cache is not None and key in doc_cache: |
| 79 | doc = doc_cache[key] |
| 80 | else: |
| 81 | doc = htmlmod.parse(text) |
| 82 | if doc_cache is not None: |
| 83 | doc_cache[key] = doc |
| 84 | found = doc.select(entry["css"]) |
| 85 | if entry.get("all"): |
| 86 | return [_element_value(entry, el) for el in found] |
| 87 | if not found: |
| 88 | return MISSING |
| 89 | return _element_value(entry, found[0]) |
| 90 | raise ExtractError(f"extract entry {entry.get('into')!r} has no way") |
| 91 | |
| 92 | |
| 93 | def _text_source(entry: dict[str, Any], source: dict[str, Any]) -> Any: |
| 94 | frm = entry.get("from", "body") |
| 95 | value = resolve(frm, source) if isinstance(frm, str) else MISSING |
| 96 | if value is MISSING or value is None: |
| 97 | return MISSING |
| 98 | return value if isinstance(value, str) else str(value) |
| 99 | |
| 100 | |
| 101 | def _element_value(entry: dict[str, Any], el: htmlmod.Element) -> Any: |
| 102 | fields = entry.get("fields") |
| 103 | if fields: |
| 104 | out: dict[str, Any] = {} |
| 105 | for name, spec in fields.items(): |
| 106 | if isinstance(spec, str): |
| 107 | sub = el.select_one(spec) |
| 108 | out[name] = sub.text if sub else None |
| 109 | else: |
| 110 | sub = el.select_one(spec["css"]) |
| 111 | if sub is None: |
| 112 | out[name] = None |
| 113 | elif "attr" in spec: |
| 114 | out[name] = sub.get(spec["attr"]) |
| 115 | else: |
| 116 | out[name] = sub.text |
| 117 | return out |
| 118 | if "attr" in entry: |
| 119 | return el.get(entry["attr"]) |
| 120 | return el.text |