cpython/Lib/token.py at 3.15 · python/cpython · GitHub

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

"""Token constants."""

# Auto-generated by Tools/build/generate_token.py

__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF',

'EXACT_TOKEN_TYPES']

ENDMARKER = 0

NAME = 1

NUMBER = 2

STRING = 3

NEWLINE = 4

INDENT = 5

DEDENT = 6

LPAR = 7

RPAR = 8

LSQB = 9

RSQB = 10

COLON = 11

COMMA = 12

SEMI = 13

PLUS = 14

MINUS = 15

STAR = 16

SLASH = 17

VBAR = 18

AMPER = 19

LESS = 20

GREATER = 21

EQUAL = 22

DOT = 23

PERCENT = 24

LBRACE = 25

RBRACE = 26

EQEQUAL = 27

NOTEQUAL = 28

LESSEQUAL = 29

GREATEREQUAL = 30

TILDE = 31

CIRCUMFLEX = 32

LEFTSHIFT = 33

RIGHTSHIFT = 34

DOUBLESTAR = 35

PLUSEQUAL = 36

MINEQUAL = 37

STAREQUAL = 38

SLASHEQUAL = 39

PERCENTEQUAL = 40

AMPEREQUAL = 41

VBAREQUAL = 42

CIRCUMFLEXEQUAL = 43

LEFTSHIFTEQUAL = 44

RIGHTSHIFTEQUAL = 45

DOUBLESTAREQUAL = 46

DOUBLESLASH = 47

DOUBLESLASHEQUAL = 48

AT = 49

ATEQUAL = 50

RARROW = 51

ELLIPSIS = 52

COLONEQUAL = 53

EXCLAMATION = 54

OP = 55

TYPE_IGNORE = 56

TYPE_COMMENT = 57

SOFT_KEYWORD = 58

FSTRING_START = 59

FSTRING_MIDDLE = 60

FSTRING_END = 61

TSTRING_START = 62

TSTRING_MIDDLE = 63

TSTRING_END = 64

COMMENT = 65

NL = 66

# These aren't used by the C tokenizer but are needed for tokenize.py

ERRORTOKEN = 67

ENCODING = 68

N_TOKENS = 69

# Special definitions for cooperation with parser

NT_OFFSET = 256

tok_name = {value: name

for name, value in globals().items()

if isinstance(value, int) and not name.startswith('_')}

__all__.extend(tok_name.values())

EXACT_TOKEN_TYPES = {

'!': EXCLAMATION,

'!=': NOTEQUAL,

'%': PERCENT,

'%=': PERCENTEQUAL,

'&': AMPER,

'&=': AMPEREQUAL,

'(': LPAR,

')': RPAR,

'*': STAR,

'**': DOUBLESTAR,

'**=': DOUBLESTAREQUAL,

'*=': STAREQUAL,

'+': PLUS,

'+=': PLUSEQUAL,

',': COMMA,

'-': MINUS,

'-=': MINEQUAL,

'->': RARROW,

'.': DOT,

'...': ELLIPSIS,

'/': SLASH,

'//': DOUBLESLASH,

'//=': DOUBLESLASHEQUAL,

'/=': SLASHEQUAL,

':': COLON,

':=': COLONEQUAL,

';': SEMI,

'<': LESS,

'<<': LEFTSHIFT,

'<<=': LEFTSHIFTEQUAL,

'<=': LESSEQUAL,

'=': EQUAL,

'==': EQEQUAL,

'>': GREATER,

'>=': GREATEREQUAL,

'>>': RIGHTSHIFT,

'>>=': RIGHTSHIFTEQUAL,

'@': AT,

'@=': ATEQUAL,

'[': LSQB,

']': RSQB,

'^': CIRCUMFLEX,

'^=': CIRCUMFLEXEQUAL,

'{': LBRACE,

'|': VBAR,

'|=': VBAREQUAL,

'}': RBRACE,

'~': TILDE,

}

def ISTERMINAL(x: int) -> bool:

return x < NT_OFFSET

def ISNONTERMINAL(x: int) -> bool:

return x >= NT_OFFSET

def ISEOF(x: int) -> bool:

return x == ENDMARKER

cpython/Lib/token.py at 3.15 · python/cpython · GitHub

Repository navigation

Expand file tree

Breadcrumbs

token.py

Latest commit

History

Breadcrumbs

token.py

File metadata and controls