pyrtan / sourcev0.19.0 · main

pyrtan / src / pyrtan / html.py

8,749 bytes · last changed 2026-10-25 in Initial public release

1"""A small HTML document model with a CSS selector subset, on the standard
2library's parser.
3
4The subset is what trees actually need to read a page: tag, ``#id``,
5``.class``, ``[attr]``, ``[attr=value]``, ``[attr^=v]``, ``[attr$=v]``,
6``[attr*=v]``, ``:first-child``, ``:nth-child(n)``, ``:contains(text)``,
7the descendant (space) and child (``>``) combinators, and ``,`` groups.
8Anything else is a :class:`SelectorError` at tree check time, not a silent
9empty match at run time.
10"""
11
12from __future__ import annotations
13
14import re
15from dataclasses import dataclass, field
16from html.parser import HTMLParser
17from typing import Iterator
18
19VOID = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}
20INLINE = {"a", "abbr", "b", "bdi", "bdo", "cite", "code", "data", "dfn", "em", "i", "kbd", "label", "mark", "q", "s", "samp", "small", "span", "strong", "sub", "sup", "time", "u", "var", "del", "ins"}
21
22
23@dataclass
24class Element:
25 tag: str
26 attrs: dict[str, str] = field(default_factory=dict)
27 children: list["Element | str"] = field(default_factory=list)
28 parent: "Element | None" = field(default=None, repr=False, compare=False)
29
30 @property
31 def text(self) -> str:
32 """All descendant text, whitespace collapsed. Inline elements join
33 their neighbours directly; any other element boundary counts as a
34 space, so two table cells never run together."""
35 parts: list[str] = []
36 for child in self.children:
37 if isinstance(child, str):
38 parts.append(child)
39 elif child.tag in INLINE:
40 parts.append(child.text)
41 else:
42 parts.append(" " + child.text + " ")
43 return re.sub(r"\s+", " ", "".join(parts)).strip()
44
45 @property
46 def classes(self) -> list[str]:
47 return self.attrs.get("class", "").split()
48
49 def get(self, name: str, default: str | None = None) -> str | None:
50 return self.attrs.get(name, default)
51
52 def elements(self) -> Iterator["Element"]:
53 for child in self.children:
54 if isinstance(child, Element):
55 yield child
56
57 def descendants(self) -> Iterator["Element"]:
58 for child in self.elements():
59 yield child
60 yield from child.descendants()
61
62 def select(self, selector: str) -> list["Element"]:
63 return select(self, selector)
64
65 def select_one(self, selector: str) -> "Element | None":
66 found = select(self, selector)
67 return found[0] if found else None
68
69 def to_dict(self) -> dict:
70 return {"tag": self.tag, "attrs": self.attrs, "text": self.text}
71
72
73class _Builder(HTMLParser):
74 def __init__(self) -> None:
75 super().__init__(convert_charrefs=True)
76 self.root = Element("document")
77 self.stack = [self.root]
78
79 def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
80 el = Element(tag.lower(), {k.lower(): (v if v is not None else "") for k, v in attrs}, parent=self.stack[-1])
81 self.stack[-1].children.append(el)
82 if tag.lower() not in VOID:
83 self.stack.append(el)
84
85 def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
86 el = Element(tag.lower(), {k.lower(): (v if v is not None else "") for k, v in attrs}, parent=self.stack[-1])
87 self.stack[-1].children.append(el)
88
89 def handle_endtag(self, tag: str) -> None:
90 tag = tag.lower()
91 for i in range(len(self.stack) - 1, 0, -1):
92 if self.stack[i].tag == tag:
93 del self.stack[i:]
94 return
95
96 def handle_data(self, data: str) -> None:
97 if data:
98 self.stack[-1].children.append(data)
99
100
101def parse(text: str) -> Element:
102 b = _Builder()
103 b.feed(text)
104 b.close()
105 return b.root
106
107
108class SelectorError(ValueError):
109 pass
110
111
112_TOKEN = re.compile(
113 r"""
114 (?P<tag>[a-zA-Z][a-zA-Z0-9\-]*|\*)
115 | \#(?P<id>[\w\-]+)
116 | \.(?P<cls>[\w\-]+)
117 | \[(?P<attr>[\w\-:]+)(?:(?P<op>[\^\$\*]?=)(?P<q>["']?)(?P<val>[^\]"']*)(?P=q))?\]
118 | :(?P<pseudo>first-child|last-child|nth-child\(\d+\)|contains\((?P<cq>["']?)(?P<ctext>[^)"']*)(?P=cq)\))
119 """,
120 re.VERBOSE,
121)
122
123
124@dataclass
125class _Simple:
126 tag: str | None = None
127 id: str | None = None
128 classes: list[str] = field(default_factory=list)
129 attrs: list[tuple[str, str, str | None]] = field(default_factory=list)
130 pseudos: list[str] = field(default_factory=list)
131
132 def matches(self, el: Element) -> bool:
133 if self.tag and self.tag != "*" and el.tag != self.tag:
134 return False
135 if self.id and el.attrs.get("id") != self.id:
136 return False
137 if self.classes and not set(self.classes) <= set(el.classes):
138 return False
139 for name, op, val in self.attrs:
140 if name not in el.attrs:
141 return False
142 have = el.attrs[name]
143 if op == "=" and have != val:
144 return False
145 if op == "^=" and not have.startswith(val or ""):
146 return False
147 if op == "$=" and not have.endswith(val or ""):
148 return False
149 if op == "*=" and (val or "") not in have:
150 return False
151 for p in self.pseudos:
152 if not _pseudo(p, el):
153 return False
154 return True
155
156
157def _pseudo(p: str, el: Element) -> bool:
158 if p.startswith("contains:"):
159 return p[len("contains:"):] in el.text
160 siblings = list(el.parent.elements()) if el.parent else [el]
161 if p == "first-child":
162 return siblings[0] is el
163 if p == "last-child":
164 return siblings[-1] is el
165 if p.startswith("nth-child("):
166 n = int(p[len("nth-child("):-1])
167 return 0 < n <= len(siblings) and siblings[n - 1] is el
168 raise SelectorError(f"unknown pseudo-class :{p}")
169
170
171def _parse_compound(text: str) -> _Simple:
172 simple = _Simple()
173 pos = 0
174 text = text.strip()
175 if not text:
176 raise SelectorError("empty selector part")
177 while pos < len(text):
178 m = _TOKEN.match(text, pos)
179 if not m:
180 raise SelectorError(f"cannot parse selector near {text[pos:]!r}")
181 pos = m.end()
182 if m.group("tag"):
183 if simple.tag:
184 raise SelectorError(f"two tags in one compound selector: {text!r}")
185 simple.tag = m.group("tag").lower()
186 elif m.group("id"):
187 simple.id = m.group("id")
188 elif m.group("cls"):
189 simple.classes.append(m.group("cls"))
190 elif m.group("attr"):
191 simple.attrs.append((m.group("attr").lower(), m.group("op") or "", m.group("val")))
192 elif m.group("pseudo"):
193 p = m.group("pseudo")
194 if p.startswith("contains("):
195 simple.pseudos.append("contains:" + m.group("ctext"))
196 else:
197 simple.pseudos.append(p)
198 return simple
199
200
201def compile_selector(selector: str) -> list[list[tuple[str, _Simple]]]:
202 """Compile to groups of (combinator, compound) pairs. Raises on anything
203 outside the supported subset."""
204 groups = []
205 for alt in selector.split(","):
206 alt = alt.strip()
207 if not alt:
208 raise SelectorError(f"empty alternative in {selector!r}")
209 parts = re.split(r"\s*(>)\s*|\s+", alt)
210 chain: list[tuple[str, _Simple]] = []
211 comb = " "
212 for part in parts:
213 if part is None or part == "":
214 continue
215 if part == ">":
216 comb = ">"
217 continue
218 chain.append((comb, _parse_compound(part)))
219 comb = " "
220 if not chain:
221 raise SelectorError(f"nothing to match in {selector!r}")
222 groups.append(chain)
223 return groups
224
225
226def select(root: Element, selector: str) -> list[Element]:
227 compiled = compile_selector(selector)
228 out: list[Element] = []
229 seen: set[int] = set()
230 for el in root.descendants():
231 for chain in compiled:
232 if _chain_matches(el, chain) and id(el) not in seen:
233 seen.add(id(el))
234 out.append(el)
235 break
236 return out
237
238
239def _chain_matches(el: Element, chain: list[tuple[str, _Simple]]) -> bool:
240 comb, simple = chain[-1]
241 if not simple.matches(el):
242 return False
243 if len(chain) == 1:
244 return True
245 rest = chain[:-1]
246 if comb == ">":
247 return el.parent is not None and el.parent.tag != "document" and _chain_matches(el.parent, rest)
248 anc = el.parent
249 while anc is not None and anc.tag != "document":
250 if _chain_matches(anc, rest):
251 return True
252 anc = anc.parent
253 return False