"""A parser for HTML and XHTML."""
# This file is based on sgmllib.py, but the API is slightly different.
# XXX There should be a way to distinguish between PCDATA (parsed
# character data -- the normal case), RCDATA (replaceable character
# data -- only char and entity references and end tags are special)
# and CDATA (character data -- only end tags are special).
import re
import _markupbase
from html import unescape
from html.entities import html5 as html5_entities
__all__ = ['HTMLParser']
# Regular expressions used for parsing
interesting_normal = re.compile('[&')
# Note:
# 1) if you change tagfind/attrfind remember to update locatetagend too;
# 2) if you change tagfind/attrfind and/or locatetagend the parser will
# explode, so don't do it.
# see the HTML5 specs section "13.2.5.6 Tag open state",
# "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state".
# https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
# https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state
tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*')
attrfind_tolerant = re.compile(r"""
(
(?\t\n\r\f ]* # bare value
)
)?
(?:[\t\n\r\f ]|/(?!>))* # possibly followed by a space
""", re.VERBOSE)
locatetagend = re.compile(r"""
[a-zA-Z][^\t\n\r\f />]* # tag name
[\t\n\r\f /]* # optional whitespace before attribute name
(?:(?\t\n\r\f ]* # bare value
)
)?
[\t\n\r\f /]* # possibly followed by a space
)*
>?
""", re.VERBOSE)
# The following variables are not used, but are temporarily left for
# backward compatibility.
locatestarttagend_tolerant = re.compile(r"""
\x00]* # tag name
(?:[\s/]* # optional whitespace before attribute name
(?:(?\s]* # bare value
)
\s* # possibly followed by a space
)?(?:\s|/(?!>))*
)*
)?
\s* # trailing whitespace
""", re.VERBOSE)
endendtag = re.compile('>')
endtagfind = re.compile(r'')
# Character reference processing logic specific to attribute values
# See: https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
def _replace_attr_charref(match):
ref = match.group(0)
# Numeric / hex char refs must always be unescaped
if ref.startswith('&#'):
return unescape(ref)
# Named character / entity references must only be unescaped
# if they are an exact match, and they are not followed by an equals sign
if not ref.endswith('=') and ref[1:] in html5_entities:
return unescape(ref)
# Otherwise do not unescape
return ref
def _unescape_attrvalue(s):
return attr_charref.sub(_replace_attr_charref, s)
class HTMLParser(_markupbase.ParserBase):
"""Find tags and other markup and call handler functions.
Usage:
p = HTMLParser()
p.feed(data)
...
p.close()
Start tags are handled by calling self.handle_starttag() or
self.handle_startendtag(); end tags by self.handle_endtag(). The
data between tags is passed from the parser to the derived class
by calling self.handle_data() with the data as argument (the data
may be split up in arbitrary chunks). If convert_charrefs is
True the character references are converted automatically to the
corresponding Unicode character (and self.handle_data() is no
longer split in chunks), otherwise they are passed by calling
self.handle_entityref() or self.handle_charref() with the string
containing respectively the named or numeric reference as the
argument.
"""
# See the HTML5 specs section "13.4 Parsing HTML fragments".
# https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments
# CDATA_CONTENT_ELEMENTS are parsed in RAWTEXT mode
CDATA_CONTENT_ELEMENTS = ("script", "style", "xmp", "iframe", "noembed", "noframes")
RCDATA_CONTENT_ELEMENTS = ("textarea", "title")
def __init__(self, *, convert_charrefs=True, scripting=False):
"""Initialize and reset this instance.
If convert_charrefs is true (the default), all character references
are automatically converted to the corresponding Unicode characters.
If *scripting* is false (the default), the content of the
``noscript`` element is parsed normally; if it's true,
it's returned as is without being parsed.
"""
super().__init__()
self.convert_charrefs = convert_charrefs
self.scripting = scripting
self.reset()
def reset(self):
"""Reset this instance. Loses all unprocessed data."""
self.rawdata = ''
self.lasttag = '???'
self.interesting = interesting_normal
self.cdata_elem = None
self._support_cdata = True
self._escapable = True
self._pending = []
self._pending_len = 0
self._parse_threshold = 1
super().reset()
def feed(self, data):
r"""Feed data to the parser.
Call this as often as you want, with as little or as much text
as you want (may include '\n').
"""
# Accumulate new data in a list and only join and parse it once
# enough has piled up. Rescanning an unparsed buffer (e.g. an
# unterminated tag) and concatenating onto it on every call would
# both be quadratic in the input size.
self._pending_len += len(data)
if self._pending_len < self._parse_threshold:
self._pending.append(data)
else:
if not self._pending:
self.rawdata += data
else:
self._pending.append(data)
self.rawdata += ''.join(self._pending)
self._pending.clear()
self._pending_len = 0
n = len(self.rawdata)
self.goahead(0)
if len(self.rawdata) < n:
# Some data was parsed; resume on the next call.
self._parse_threshold = 1
else:
# Nothing was parsed; wait until the buffer doubles.
self._parse_threshold = len(self.rawdata)
def close(self):
"""Handle any buffered data."""
if self._pending:
self.rawdata += ''.join(self._pending)
self._pending.clear()
self._pending_len = 0
self.goahead(1)
__starttag_text = None
def get_starttag_text(self):
"""Return full source of start tag: ''."""
return self.__starttag_text
def set_cdata_mode(self, elem, *, escapable=False):
self.cdata_elem = elem.lower()
self._escapable = escapable
if self.cdata_elem == 'plaintext':
self.interesting = re.compile(r'\z')
elif escapable and not self.convert_charrefs:
self.interesting = re.compile(r'&|])' % self.cdata_elem,
re.IGNORECASE|re.ASCII)
else:
self.interesting = re.compile(r'])' % self.cdata_elem,
re.IGNORECASE|re.ASCII)
def clear_cdata_mode(self):
self.interesting = interesting_normal
self.cdata_elem = None
self._escapable = True
def _set_support_cdata(self, flag=True):
"""Enable or disable support of the CDATA sections.
If enabled, "".
If disabled, "".
This method is not called by default. Its purpose is to be called
in custom handle_starttag() and handle_endtag() methods, with
value that depends on the adjusted current node.
See https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state
for details.
"""
self._support_cdata = flag
# Internal -- handle data as far as reasonable. May leave state
# and data to be processed by a subsequent call. If 'end' is
# true, force handling all data as if followed by EOF marker.
def goahead(self, end):
rawdata = self.rawdata
i = 0
n = len(rawdata)
while i < n:
if self.convert_charrefs and not self.cdata_elem:
j = rawdata.find('