FazBrowse GitHub Viewer
|
Trending
|
URL:
|
Home
Tools:
[Download Repo ZIP]
[View Raw Code]
[Original HTTPS Page]
cpython/Parser/token.c at e69d068ad0bd6a25434ea476a647b635da4d82bb · python/cpython · GitHub
python
cpython
Repository navigation
Code
Issues
5k+
(5k+)
Pull requests
2.3k
(2.3k)
Actions
Projects
Security and quality
Insights
Expand file tree
Breadcrumbs
cpython
/
Parser
/
token.c
Copy path
More file actions
More file actions
Latest commit
History
History
History
247 lines (240 loc) · 4.61 KB
Breadcrumbs
cpython
/
Parser
/
token.c
Copy path
File metadata and controls
247 lines (240 loc) · 4.61 KB
Raw
Copy raw file
Download raw file
Open symbols panel
Edit and raw actions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
/* Auto-generated by Tools/build/generate_token.py */
#include
"Python.h"
#include
"pycore_token.h"
/* Token names */
const
char
*
const
_PyParser_TokenNames
[]
=
{
"ENDMARKER"
,
"NAME"
,
"NUMBER"
,
"STRING"
,
"NEWLINE"
,
"INDENT"
,
"DEDENT"
,
"LPAR"
,
"RPAR"
,
"LSQB"
,
"RSQB"
,
"COLON"
,
"COMMA"
,
"SEMI"
,
"PLUS"
,
"MINUS"
,
"STAR"
,
"SLASH"
,
"VBAR"
,
"AMPER"
,
"LESS"
,
"GREATER"
,
"EQUAL"
,
"DOT"
,
"PERCENT"
,
"LBRACE"
,
"RBRACE"
,
"EQEQUAL"
,
"NOTEQUAL"
,
"LESSEQUAL"
,
"GREATEREQUAL"
,
"TILDE"
,
"CIRCUMFLEX"
,
"LEFTSHIFT"
,
"RIGHTSHIFT"
,
"DOUBLESTAR"
,
"PLUSEQUAL"
,
"MINEQUAL"
,
"STAREQUAL"
,
"SLASHEQUAL"
,
"PERCENTEQUAL"
,
"AMPEREQUAL"
,
"VBAREQUAL"
,
"CIRCUMFLEXEQUAL"
,
"LEFTSHIFTEQUAL"
,
"RIGHTSHIFTEQUAL"
,
"DOUBLESTAREQUAL"
,
"DOUBLESLASH"
,
"DOUBLESLASHEQUAL"
,
"AT"
,
"ATEQUAL"
,
"RARROW"
,
"ELLIPSIS"
,
"COLONEQUAL"
,
"EXCLAMATION"
,
"OP"
,
"TYPE_IGNORE"
,
"TYPE_COMMENT"
,
"SOFT_KEYWORD"
,
"FSTRING_START"
,
"FSTRING_MIDDLE"
,
"FSTRING_END"
,
"COMMENT"
,
"NL"
,
"<ERRORTOKEN>"
,
"<ENCODING>"
,
"<N_TOKENS>"
,
};
/* Return the token corresponding to a single character */
int
_PyToken_OneChar
(
int
c1
)
{
switch
(
c1
) {
case
'!'
:
return
EXCLAMATION
;
case
'%'
:
return
PERCENT
;
case
'&'
:
return
AMPER
;
case
'('
:
return
LPAR
;
case
')'
:
return
RPAR
;
case
'*'
:
return
STAR
;
case
'+'
:
return
PLUS
;
case
','
:
return
COMMA
;
case
'-'
:
return
MINUS
;
case
'.'
:
return
DOT
;
case
'/'
:
return
SLASH
;
case
':'
:
return
COLON
;
case
';'
:
return
SEMI
;
case
'<'
:
return
LESS
;
case
'='
:
return
EQUAL
;
case
'>'
:
return
GREATER
;
case
'@'
:
return
AT
;
case
'['
:
return
LSQB
;
case
']'
:
return
RSQB
;
case
'^'
:
return
CIRCUMFLEX
;
case
'{'
:
return
LBRACE
;
case
'|'
:
return
VBAR
;
case
'}'
:
return
RBRACE
;
case
'~'
:
return
TILDE
;
}
return
OP
;
}
int
_PyToken_TwoChars
(
int
c1
,
int
c2
)
{
switch
(
c1
) {
case
'!'
:
switch
(
c2
) {
case
'='
:
return
NOTEQUAL
;
}
break
;
case
'%'
:
switch
(
c2
) {
case
'='
:
return
PERCENTEQUAL
;
}
break
;
case
'&'
:
switch
(
c2
) {
case
'='
:
return
AMPEREQUAL
;
}
break
;
case
'*'
:
switch
(
c2
) {
case
'*'
:
return
DOUBLESTAR
;
case
'='
:
return
STAREQUAL
;
}
break
;
case
'+'
:
switch
(
c2
) {
case
'='
:
return
PLUSEQUAL
;
}
break
;
case
'-'
:
switch
(
c2
) {
case
'='
:
return
MINEQUAL
;
case
'>'
:
return
RARROW
;
}
break
;
case
'/'
:
switch
(
c2
) {
case
'/'
:
return
DOUBLESLASH
;
case
'='
:
return
SLASHEQUAL
;
}
break
;
case
':'
:
switch
(
c2
) {
case
'='
:
return
COLONEQUAL
;
}
break
;
case
'<'
:
switch
(
c2
) {
case
'<'
:
return
LEFTSHIFT
;
case
'='
:
return
LESSEQUAL
;
case
'>'
:
return
NOTEQUAL
;
}
break
;
case
'='
:
switch
(
c2
) {
case
'='
:
return
EQEQUAL
;
}
break
;
case
'>'
:
switch
(
c2
) {
case
'='
:
return
GREATEREQUAL
;
case
'>'
:
return
RIGHTSHIFT
;
}
break
;
case
'@'
:
switch
(
c2
) {
case
'='
:
return
ATEQUAL
;
}
break
;
case
'^'
:
switch
(
c2
) {
case
'='
:
return
CIRCUMFLEXEQUAL
;
}
break
;
case
'|'
:
switch
(
c2
) {
case
'='
:
return
VBAREQUAL
;
}
break
;
}
return
OP
;
}
int
_PyToken_ThreeChars
(
int
c1
,
int
c2
,
int
c3
)
{
switch
(
c1
) {
case
'*'
:
switch
(
c2
) {
case
'*'
:
switch
(
c3
) {
case
'='
:
return
DOUBLESTAREQUAL
;
}
break
;
}
break
;
case
'.'
:
switch
(
c2
) {
case
'.'
:
switch
(
c3
) {
case
'.'
:
return
ELLIPSIS
;
}
break
;
}
break
;
case
'/'
:
switch
(
c2
) {
case
'/'
:
switch
(
c3
) {
case
'='
:
return
DOUBLESLASHEQUAL
;
}
break
;
}
break
;
case
'<'
:
switch
(
c2
) {
case
'<'
:
switch
(
c3
) {
case
'='
:
return
LEFTSHIFTEQUAL
;
}
break
;
}
break
;
case
'>'
:
switch
(
c2
) {
case
'>'
:
switch
(
c3
) {
case
'='
:
return
RIGHTSHIFTEQUAL
;
}
break
;
}
break
;
}
return
OP
;
}
Back
|
FazBrowse Home
|
New Git URL