"""Small HTML tree used by official metadata adapters.""" from html.parser import HTMLParser class Node: def __init__(self,tag='',attrs=None): self.tag=tag self.attrs={key:value or '' for key,value in attrs or []} self.children=[] def text(self): if self.tag in ('script','style'): return '' content=''.join(child.text() if isinstance(child,Node) else child for child in self.children) return ' '+content+' ' if self.tag in ('p','div','section','li','br') else content def walk(self): yield self for child in self.children: if isinstance(child,Node): yield from child.walk() class Document(HTMLParser): VOID={'area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr'} def __init__(self,html): super().__init__(convert_charrefs=True) self.root=Node() self.stack=[self.root] self.feed(html) def handle_starttag(self,tag,attrs): node=Node(tag,attrs) self.stack[-1].children.append(node) if tag not in self.VOID: self.stack.append(node) def handle_endtag(self,tag): for index in range(len(self.stack)-1,0,-1): if self.stack[index].tag==tag: del self.stack[index:] break def handle_data(self,data): self.stack[-1].children.append(data) def clean(text): return ' '.join(text.split())