cpython/Lib/token.py at 3.5 · python/cpython · GitHub

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

"""Token constants (from "token.h")."""

__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF']

# This file is automatically generated; please don't muck it up!

# To update the symbols in this file, 'cd' to the top directory of

# the python source tree after building the interpreter and run:

# ./python Lib/token.py

#--start constants--

ENDMARKER = 0

NAME = 1

NUMBER = 2

STRING = 3

NEWLINE = 4

INDENT = 5

DEDENT = 6

LPAR = 7

RPAR = 8

LSQB = 9

RSQB = 10

COLON = 11

COMMA = 12

SEMI = 13

PLUS = 14

MINUS = 15

STAR = 16

SLASH = 17

VBAR = 18

AMPER = 19

LESS = 20

GREATER = 21

EQUAL = 22

DOT = 23

PERCENT = 24

LBRACE = 25

RBRACE = 26

EQEQUAL = 27

NOTEQUAL = 28

LESSEQUAL = 29

GREATEREQUAL = 30

TILDE = 31

CIRCUMFLEX = 32

LEFTSHIFT = 33

RIGHTSHIFT = 34

DOUBLESTAR = 35

PLUSEQUAL = 36

MINEQUAL = 37

STAREQUAL = 38

SLASHEQUAL = 39

PERCENTEQUAL = 40

AMPEREQUAL = 41

VBAREQUAL = 42

CIRCUMFLEXEQUAL = 43

LEFTSHIFTEQUAL = 44

RIGHTSHIFTEQUAL = 45

DOUBLESTAREQUAL = 46

DOUBLESLASH = 47

DOUBLESLASHEQUAL = 48

AT = 49

ATEQUAL = 50

RARROW = 51

ELLIPSIS = 52

OP = 53

AWAIT = 54

ASYNC = 55

ERRORTOKEN = 56

N_TOKENS = 57

NT_OFFSET = 256

#--end constants--

tok_name = {value: name

for name, value in globals().items()

if isinstance(value, int) and not name.startswith('_')}

__all__.extend(tok_name.values())

def ISTERMINAL(x):

return x < NT_OFFSET

def ISNONTERMINAL(x):

return x >= NT_OFFSET

def ISEOF(x):

return x == ENDMARKER

def _main():

import re

import sys

args = sys.argv[1:]

inFileName = args and args[0] or "Include/token.h"

outFileName = "Lib/token.py"

if len(args) > 1:

outFileName = args[1]

try:

fp = open(inFileName)

except OSError as err:

sys.stdout.write("I/O error: %s\n" % str(err))

sys.exit(1)

with fp:

lines = fp.read().split("\n")

prog = re.compile(

"#define[ \t][ \t]*([A-Z0-9][A-Z0-9_]*)[ \t][ \t]*([0-9][0-9]*)",

re.IGNORECASE)

tokens = {}

for line in lines:

match = prog.match(line)

if match:

name, val = match.group(1, 2)

val = int(val)

tokens[val] = name # reverse so we can sort them...

keys = sorted(tokens.keys())

# load the output skeleton from the target:

try:

fp = open(outFileName)

except OSError as err:

sys.stderr.write("I/O error: %s\n" % str(err))

sys.exit(2)

with fp:

format = fp.read().split("\n")

try:

start = format.index("#--start constants--") + 1

end = format.index("#--end constants--")

except ValueError:

sys.stderr.write("target does not contain format markers")

sys.exit(3)

lines = []

for val in keys:

lines.append("%s = %d" % (tokens[val], val))

format[start:end] = lines

try:

fp = open(outFileName, 'w')

except OSError as err:

sys.stderr.write("I/O error: %s\n" % str(err))

sys.exit(4)

with fp:

fp.write("\n".join(format))

if __name__ == "__main__":

_main()

cpython/Lib/token.py at 3.5 · python/cpython · GitHub

Repository navigation

Expand file tree

Breadcrumbs

token.py

Latest commit

History

Breadcrumbs

token.py

File metadata and controls