html5lib-python/utils/entities.py at patch-1 · charmander/html5lib-python · GitHub

History

100 lines (72 loc) · 2.66 KB

Raw

100

import json

import html5lib

def parse(path="html5ents.xml"):

return html5lib.parse(open(path), treebuilder="lxml")

def entity_table(tree):

return dict((entity_name("".join(tr[0].xpath(".//text()"))),

entity_characters(tr[1].text))

for tr in tree.xpath("//h:tbody/h:tr",

namespaces={"h": "http://www.w3.org/1999/xhtml"}))

def entity_name(inp):

return inp.strip()

def entity_characters(inp):

return "".join(codepoint_to_character(item)

for item in inp.split()

if item)

def codepoint_to_character(inp):

return ("\\U000" + inp[2:]).decode("unicode-escape")

def make_tests_json(entities):

test_list = make_test_list(entities)

tests_json = {"tests":

[make_test(*item) for item in test_list]

}

return tests_json

def make_test(name, characters, good):

return {

"description": test_description(name, good),

"input": "&%s" % name,

"output": test_expected(name, characters, good)

}

def test_description(name, good):

with_semicolon = name.endswith(";")

semicolon_text = {True: "with a semi-colon",

False: "without a semi-colon"}[with_semicolon]

if good:

text = "Named entity: %s %s" % (name, semicolon_text)

else:

text = "Bad named entity: %s %s" % (name, semicolon_text)

return text

def test_expected(name, characters, good):

rv = []

if not good or not name.endswith(";"):

rv.append("ParseError")

rv.append(["Character", characters])

return rv

def make_test_list(entities):

tests = []

for entity_name, characters in entities.items():

if entity_name.endswith(";") and not subentity_exists(entity_name, entities):

tests.append((entity_name[:-1], "&" + entity_name[:-1], False))

tests.append((entity_name, characters, True))

return sorted(tests)

def subentity_exists(entity_name, entities):

for i in range(1, len(entity_name)):

if entity_name[:-i] in entities:

return True

return False

def make_entities_code(entities):

entities_text = "\n".join(" \"%s\": u\"%s\"," % (

name, entities[name].encode(

"unicode-escape").replace("\"", "\\\""))

for name in sorted(entities.keys()))

return """entities = {

}""" % entities_text

def main():

entities = entity_table(parse())

tests_json = make_tests_json(entities)

json.dump(tests_json, open("namedEntities.test", "w"), indent=4)

code = make_entities_code(entities)

open("entities_constants.py", "w").write(code)

if __name__ == "__main__":

main()

html5lib-python/utils/entities.py at patch-1 · charmander/html5lib-python · GitHub

Expand file tree

Breadcrumbs

entities.py

Latest commit

History

Breadcrumbs

entities.py

File metadata and controls