vscode-python/pythonFiles/parso/python/fstring.py at master · miturchi/vscode-python · GitHub

History

211 lines (180 loc) · 7.1 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

import re

from itertools import count

from parso.utils import PythonVersionInfo

from parso.utils import split_lines

from parso.python.tokenize import Token

from parso import parser

from parso.tree import TypedLeaf, ErrorNode, ErrorLeaf

version36 = PythonVersionInfo(3, 6)

class TokenNamespace:

_c = count()

LBRACE = next(_c)

RBRACE = next(_c)

ENDMARKER = next(_c)

COLON = next(_c)

CONVERSION = next(_c)

PYTHON_EXPR = next(_c)

EXCLAMATION_MARK = next(_c)

UNTERMINATED_STRING = next(_c)

token_map = dict((v, k) for k, v in locals().items() if not k.startswith('_'))

@classmethod

def generate_token_id(cls, string):

if string == '{':

return cls.LBRACE

elif string == '}':

return cls.RBRACE

elif string == '!':

return cls.EXCLAMATION_MARK

elif string == ':':

return cls.COLON

return getattr(cls, string)

GRAMMAR = """

fstring: expression* ENDMARKER

format_spec: ':' expression*

expression: '{' PYTHON_EXPR [ '!' CONVERSION ] [ format_spec ] '}'

"""

_prefix = r'((?:[^{}]+)*)'

_expr = _prefix + r'(\{|\}|$)'

_in_expr = r'([^{}\[\]:"\'!]*)(.?)'

# There's only one conversion character allowed. But the rules have to be

# checked later anyway, so allow more here. This makes error recovery nicer.

_conversion = r'([^={}:]*)(.?)'

_compiled_expr = re.compile(_expr)

_compiled_in_expr = re.compile(_in_expr)

_compiled_conversion = re.compile(_conversion)

def tokenize(code, start_pos=(1, 0)):

def add_to_pos(string):

lines = split_lines(string)

l = len(lines[-1])

if len(lines) > 1:

start_pos[0] += len(lines) - 1

start_pos[1] = l

else:

start_pos[1] += l

def tok(value, type=None, prefix=''):

if type is None:

type = TokenNamespace.generate_token_id(value)

add_to_pos(prefix)

token = Token(type, value, tuple(start_pos), prefix)

add_to_pos(value)

return token

start = 0

recursion_level = 0

added_prefix = ''

start_pos = list(start_pos)

while True:

match = _compiled_expr.match(code, start)

prefix = added_prefix + match.group(1)

found = match.group(2)

start = match.end()

if not found:

# We're at the end.

break

if found == '}':

if recursion_level == 0 and len(code) > start and code[start] == '}':

# This is a }} escape.

added_prefix = prefix + '}}'

start += 1

continue

recursion_level = max(0, recursion_level - 1)

yield tok(found, prefix=prefix)

added_prefix = ''

else:

assert found == '{'

if recursion_level == 0 and len(code) > start and code[start] == '{':

# This is a {{ escape.

added_prefix = prefix + '{{'

start += 1

continue

recursion_level += 1

yield tok(found, prefix=prefix)

added_prefix = ''

expression = ''

squared_count = 0

curly_count = 0

while True:

expr_match = _compiled_in_expr.match(code, start)

expression += expr_match.group(1)

found = expr_match.group(2)

start = expr_match.end()

if found == '{':

curly_count += 1

expression += found

elif found == '}' and curly_count > 0:

curly_count -= 1

expression += found

elif found == '[':

squared_count += 1

expression += found

elif found == ']':

# Use a max function here, because the Python code might

# just have syntax errors.

squared_count = max(0, squared_count - 1)

expression += found

elif found == ':' and (squared_count or curly_count):

expression += found

elif found in ('"', "'"):

search = found

if len(code) > start + 1 and \

code[start] == found == code[start+1]:

search *= 3

start += 2

index = code.find(search, start)

if index == -1:

yield tok(expression, type=TokenNamespace.PYTHON_EXPR)

yield tok(

found + code[start:],

type=TokenNamespace.UNTERMINATED_STRING,

)

start = len(code)

break

expression += found + code[start:index+1]

start = index + 1

elif found == '!' and len(code) > start and code[start] == '=':

# This is a python `!=` and not a conversion.

expression += found

else:

yield tok(expression, type=TokenNamespace.PYTHON_EXPR)

if found:

yield tok(found)

break

if found == '!':

conversion_match = _compiled_conversion.match(code, start)

found = conversion_match.group(2)

start = conversion_match.end()

yield tok(conversion_match.group(1), type=TokenNamespace.CONVERSION)

if found:

yield tok(found)

if found == '}':

recursion_level -= 1

# We don't need to handle everything after ':', because that is

# basically new tokens.

yield tok('', type=TokenNamespace.ENDMARKER, prefix=prefix)

class Parser(parser.BaseParser):

def parse(self, tokens):

node = super(Parser, self).parse(tokens)

if isinstance(node, self.default_leaf): # Is an endmarker.

# If there's no curly braces we get back a non-module. We always

# want an fstring.

node = self.default_node('fstring', [node])

return node

def convert_leaf(self, pgen_grammar, type, value, prefix, start_pos):

# TODO this is so ugly.

leaf_type = TokenNamespace.token_map[type].lower()

return TypedLeaf(leaf_type, value, start_pos, prefix)

def error_recovery(self, pgen_grammar, stack, arcs, typ, value, start_pos, prefix,

add_token_callback):

if not self._error_recovery:

return super(Parser, self).error_recovery(

pgen_grammar, stack, arcs, typ, value, start_pos, prefix,

add_token_callback

)

token_type = TokenNamespace.token_map[typ].lower()

if len(stack) == 1:

error_leaf = ErrorLeaf(token_type, value, start_pos, prefix)

stack[0][2][1].append(error_leaf)

else:

dfa, state, (type_, nodes) = stack[1]

stack[0][2][1].append(ErrorNode(nodes))

stack[1:] = []

add_token_callback(typ, value, start_pos, prefix)

vscode-python/pythonFiles/parso/python/fstring.py at master · miturchi/vscode-python · GitHub

Expand file tree

Breadcrumbs

fstring.py

Latest commit

History

Breadcrumbs

fstring.py

File metadata and controls