codeql/python/extractor/lark/parser_frontends.py at codeql-cli/v2.25.1 · github/codeql · GitHub

History

189 lines (152 loc) · 6.62 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

import re

from functools import partial

from .utils import get_regexp_width

from .parsers.grammar_analysis import GrammarAnalyzer

from .lexer import TraditionalLexer, ContextualLexer, Lexer, Token

from .parsers import lalr_parser, earley, xearley, resolve_ambig, cyk

from .tree import Tree

class WithLexer:

lexer = None

parser = None

lexer_conf = None

def init_traditional_lexer(self, lexer_conf):

self.lexer_conf = lexer_conf

self.lexer = TraditionalLexer(lexer_conf.tokens, ignore=lexer_conf.ignore, user_callbacks=lexer_conf.callbacks)

def init_contextual_lexer(self, lexer_conf):

self.lexer_conf = lexer_conf

states = {idx:list(t.keys()) for idx, t in self.parser._parse_table.states.items()}

always_accept = lexer_conf.postlex.always_accept if lexer_conf.postlex else ()

self.lexer = ContextualLexer(lexer_conf.tokens, states,

ignore=lexer_conf.ignore,

always_accept=always_accept,

user_callbacks=lexer_conf.callbacks)

def lex(self, text):

stream = self.lexer.lex(text)

if self.lexer_conf.postlex:

return self.lexer_conf.postlex.process(stream)

return stream

def parse(self, text):

token_stream = self.lex(text)

sps = self.lexer.set_parser_state

return self.parser.parse(token_stream, *[sps] if sps is not NotImplemented else [])

class LALR_TraditionalLexer(WithLexer):

def __init__(self, lexer_conf, parser_conf, options=None):

self.parser = lalr_parser.Parser(parser_conf)

self.init_traditional_lexer(lexer_conf)

class LALR_ContextualLexer(WithLexer):

def __init__(self, lexer_conf, parser_conf, options=None):

self.parser = lalr_parser.Parser(parser_conf)

self.init_contextual_lexer(lexer_conf)

class LALR_CustomLexer(WithLexer):

def __init__(self, lexer_cls, lexer_conf, parser_conf, options=None):

self.parser = lalr_parser.Parser(parser_conf)

self.lexer_conf = lexer_conf

self.lexer = lexer_cls(lexer_conf)

def get_ambiguity_resolver(options):

if not options or options.ambiguity == 'resolve':

return resolve_ambig.standard_resolve_ambig

elif options.ambiguity == 'resolve__antiscore_sum':

return resolve_ambig.antiscore_sum_resolve_ambig

elif options.ambiguity == 'explicit':

return None

raise ValueError(options)

def tokenize_text(text):

line = 1

col_start_pos = 0

for i, ch in enumerate(text):

if '\n' in ch:

line += ch.count('\n')

col_start_pos = i + ch.rindex('\n')

yield Token('CHAR', ch, line=line, column=i - col_start_pos)

class Earley(WithLexer):

def __init__(self, lexer_conf, parser_conf, options=None):

self.init_traditional_lexer(lexer_conf)

self.parser = earley.Parser(parser_conf, self.match,

resolve_ambiguity=get_ambiguity_resolver(options))

def match(self, term, token):

return term.name == token.type

class XEarley:

def __init__(self, lexer_conf, parser_conf, options=None, **kw):

self.token_by_name = {t.name:t for t in lexer_conf.tokens}

self._prepare_match(lexer_conf)

self.parser = xearley.Parser(parser_conf,

self.match,

resolve_ambiguity=get_ambiguity_resolver(options),

ignore=lexer_conf.ignore,

predict_all=options.earley__predict_all,

**kw

)

def match(self, term, text, index=0):

return self.regexps[term.name].match(text, index)

def _prepare_match(self, lexer_conf):

self.regexps = {}

for t in lexer_conf.tokens:

regexp = t.pattern.to_regexp()

try:

width = get_regexp_width(regexp)[0]

except ValueError:

raise ValueError("Bad regexp in token %s: %s" % (t.name, regexp))

else:

if width == 0:

raise ValueError("Dynamic Earley doesn't allow zero-width regexps", t)

self.regexps[t.name] = re.compile(regexp)

def parse(self, text):

return self.parser.parse(text)

class XEarley_CompleteLex(XEarley):

def __init__(self, *args, **kw):

super(self).__init__(*args, complete_lex=True, **kw)

class CYK(WithLexer):

def __init__(self, lexer_conf, parser_conf, options=None):

self.init_traditional_lexer(lexer_conf)

self._analysis = GrammarAnalyzer(parser_conf)

self._parser = cyk.Parser(parser_conf.rules, parser_conf.start)

self._postprocess = {}

for rule in parser_conf.rules:

a = rule.alias

self._postprocess[a] = a if callable(a) else (a and getattr(parser_conf.callback, a))

def parse(self, text):

tokens = list(self.lex(text))

parse = self._parser.parse(tokens)

parse = self._transform(parse)

return parse

def _transform(self, tree):

subtrees = list(tree.iter_subtrees())

for subtree in subtrees:

subtree.children = [self._apply_callback(c) if isinstance(c, Tree) else c for c in subtree.children]

return self._apply_callback(tree)

def _apply_callback(self, tree):

children = tree.children

callback = self._postprocess[tree.rule.alias]

assert callback, tree.rule.alias

r = callback(children)

return r

def get_frontend(parser, lexer):

if parser=='lalr':

if lexer is None:

raise ValueError('The LALR parser requires use of a lexer')

elif lexer == 'standard':

return LALR_TraditionalLexer

elif lexer == 'contextual':

return LALR_ContextualLexer

elif issubclass(lexer, Lexer):

return partial(LALR_CustomLexer, lexer)

else:

raise ValueError('Unknown lexer: %s' % lexer)

elif parser=='earley':

if lexer=='standard':

return Earley

elif lexer=='dynamic':

return XEarley

elif lexer=='dynamic_complete':

return XEarley_CompleteLex

elif lexer=='contextual':

raise ValueError('The Earley parser does not support the contextual parser')

else:

raise ValueError('Unknown lexer: %s' % lexer)

elif parser == 'cyk':

if lexer == 'standard':

return CYK

else:

raise ValueError('CYK parser requires using standard parser.')

else:

raise ValueError('Unknown parser: %s' % parser)

codeql/python/extractor/lark/parser_frontends.py at codeql-cli/v2.25.1 · github/codeql · GitHub

Uh oh!

Expand file tree

Breadcrumbs

parser_frontends.py

Latest commit

History

Breadcrumbs

parser_frontends.py

File metadata and controls