codeql/python/extractor/tests/test_tokenizer.py at codeql-cli-2.27.1 · github/codeql · GitHub

History

66 lines (55 loc) · 2.37 KB

Raw

import sys

import os.path

import shutil

import unittest

import semmle.populator

from tests import test_utils

from semmle.python.parser import tokenizer

from blib2to3.pgen2.token import tok_name

def unescape(s):

return u"'" + s.replace(u"\\", u"\\\\").replace(u"\n", u"\\n").replace(u"\t", u"\\t").replace(u"\'", u"\\'") + u"'"

def format_token(token):

type, text, start, end = token

# Use Python 3 tokenize style output, regardless of version

token_range = u"%d,%d-%d,%d:" % (start + end)

return u"%-20s%-15s%s" % (token_range, tok_name[type], unescape(text))

class TokenizerTest(unittest.TestCase):

def __init__(self, name):

super(TokenizerTest, self).__init__(name)

self.test_folder = os.path.join(os.path.dirname(__file__), "tokenizer")

def setUp(self):

pass

def tearDown(self):

pass

def compare_tokens(self, filename):

pyfile = os.path.join(self.test_folder, filename)

tokenfile = os.path.join(self.test_folder, filename[:-3]+".tokens")

with open(tokenfile, "rb") as tkns:

expected = [ line.strip().decode("utf8") for line in tkns if line.strip() ]

try:

with open(pyfile, "rb") as srcfile:

srcbytes = srcfile.read()

encoding, srcbytes = tokenizer.encoding_from_source(srcbytes)

text = srcbytes.decode(encoding)

actual = [format_token(tkn) for tkn in tokenizer.Tokenizer(text).tokens()]

except Exception as ex:

print(ex)

self.fail("Failed to tokenize " + filename)

if expected == actual:

return

actualfile = os.path.join(self.test_folder, filename[:-3]+".actual")

with open(actualfile, "wb") as out:

for line in actual:

out.write(line.encode("utf8"))

out.write(b"\n")

lineno = 1

for expected_tkn, actual_tkn in zip(expected, actual):

assert type(expected_tkn) is str

assert type(actual_tkn) is str

self.assertEqual(expected_tkn, actual_tkn, " at %s:%d" % (filename[:-3]+".tokens", lineno))

lineno += 1

self.assertTrue(len(expected) == len(actual), "Too few or too many tokens for %s" % filename)

def test_tokens(self):

for file in os.listdir(self.test_folder):

if file.endswith(".py"):

self.compare_tokens(file)

codeql/python/extractor/tests/test_tokenizer.py at codeql-cli-2.27.1 · github/codeql · GitHub

Expand file tree

Breadcrumbs

test_tokenizer.py

Latest commit

History

Breadcrumbs

test_tokenizer.py

File metadata and controls