#include
#include
#include
#include
#include
#include
namespace LCompilers::LPython {
template
bool adddgt(uint64_t &u, uint64_t d)
{
if (u > (std::numeric_limits::max() - d) / base) {
return false;
}
u = u * base + d;
return true;
}
bool lex_oct(const unsigned char *s, const unsigned char *e, uint64_t &u)
{
for (u = 0, ++s; s < e; ++s) {
if (!adddgt(u, *s - 0x30u)) {
return false;
}
}
return true;
}
bool lex_dec(const unsigned char *s, const unsigned char *e, uint64_t &u)
{
for (u = 0; s < e; ++s) {
if (*s == '_') continue;
if (!adddgt(u, *s - 0x30u)) {
return false;
}
}
return true;
}
void lex_dec_int_large(Allocator &al, const unsigned char *s,
const unsigned char *e, BigInt::BigInt &u)
{
uint64_t ui;
if (lex_dec(s, e, ui)) {
if (ui = MAX_PAREN_LEVEL) {
throw parser_local::TokenizerError(
"Too many nested parentheses", {loc});
}
paren_stack[parenlevel] = c;
parenlevel++;
break;
case ')':
case ']':
case '}':
if(parenlevel < 1) {
throw parser_local::TokenizerError(
"Parenthesis unexpected", {loc});
}
parenlevel--;
char prev_paren = paren_stack[parenlevel];
if(!((prev_paren == '(' && c == ')') ||
(prev_paren == '[' && c == ']') ||
(prev_paren == '{' && c == '}'))) {
throw parser_local::TokenizerError(
"Parentheses does not match", {loc});
}
break;
}
return;
}
#define KW(x) token(yylval.string); RET(KW_##x);
#define RET(x) token_loc(loc); last_token=yytokentype::x; return yytokentype::x;
int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnostics &/*diagnostics*/)
{
if(dedent == 1) {
// Removes the indent completely i.e., to level 0
if(!indent_length.empty()) {
indent_length.pop_back();
return yytokentype::TK_DEDENT;
} else {
dedent = 0;
}
} else if(dedent == 2) {
// Reduce the indent to `last_indent_length`
if((long int)indent_length.back() != last_indent_length) {
indent_length.pop_back();
loc.first = loc.last;
return yytokentype::TK_DEDENT;
} else {
dedent = 0;
}
}
for (;;) {
tok = cur;
/*
Re2c has excellent documentation at:
https://re2c.org/manual/manual_c.html
The first paragraph there explains the basics:
* If multiple rules match, the longest match takes precedence
* If multiple rules match the same string, the earlier rule takes
precedence
* Default rule `*` should always be defined, it has the lowest priority
regardless of its place and matches any code unit
* We use the "Sentinel character" method for end of input:
* The end of the input text is denoted with a null character \x00
* Thus the null character cannot be part of the input otherwise
* There is one rule to match \x00 to end the parser
* No other rule is allowed to match \x00, otherwise the re2c block
would parse past the end of the string and segfaults
* A special case of the previous point are negated character
ranges, such as [^"\x00], where one must include \x00 in it to
ensure this rule does not match \x00 (all other rules simply do
not mention \x00)
* See the "Handling the end of input" section in the re2c
documentation for more info
The re2c block interacts with the rest of the code via just one pointer
variable `cur`. On entering the re2c block, the `cur` variable must
point to the first character of the token to be tokenized by the block.
The re2c block below then executes on its own until a rule is matched:
the action in {} is then executed. In that action `cur` points to the
first character of the next token.
Before the re2c block we save the current `cur` into `tok`, so that we
can use `tok` and `cur` in the action in {} to extract the token that
corresponds to the rule that got matched:
* `tok` points to the first character of the token
* `cur-1` points to the last character of the token
* `cur` points to the first character of the next token
* `cur-tok` is the length of the token
In the action, we do one of:
* call `continue` which executes another cycle in the for loop (which
will parse the next token); we use this to skip a token
* call `return` which returns from this function; we return a token
* throw an exception (terminates the tokenizer)
In the first two cases, `cur` points to first character of the next
token, which becomes `tok` at the next iteration of the loop (either
right away after `continue` or after the `lex` function is called again
after `return`).
See the manual for more details.
*/
// These two variables are needed by the re2c block below internally,
// initialization is not needed. One can think of them as local
// variables of the re2c block.
unsigned char *mar; //, *ctxmar;
/*!re2c
re2c:define:YYCURSOR = cur;
re2c:define:YYMARKER = mar;
// re2c:define:YYCTXMARKER = ctxmar;
re2c:yyfill:enable = 0;
re2c:define:YYCTYPE = "unsigned char";
end = "\x00";
whitespace = [ \t\v]+;
newline = "\n" | "\r\n";
digit = [0-9];
int_oct = "0"[oO]([0-7] | "_" [0-7])+;
int_bin = "0"[bB]([01] | "_" [01])+;
int_hex = "0"[xX]([0-9a-fA-F] | "_" [0-9a-fA-F])+;
digits = digit+ (digit | "_" digit)*;
char = [^\x00-\x7F]|[a-zA-Z_];
name = char (char | digit)*;
significand = (digits "." digits?) | ("." digits);
exp = [eE][-+]? digits;
integer = digits | int_oct | int_bin | int_hex;
real = (significand exp?) | (digits exp);
imag_number = (real | digits)[jJ];
string1 = '"' ('\\'[^\x00] | [^"\x00\n\\])* '"';
string2 = "'" ("\\"[^\x00] | [^'\x00\n\\])* "'";
string3 = '"""' ( '\\'[^\x00]
| ('"' | '"' '\\'+ '"' | '"' '\\'+) [^"\x00\\]
| ('""' | '""' '\\'+) [^"\x00\\]
| [^"\x00\\] )*
'"""';
string4 = "'''" ( "\\"[^\x00]
| ("'" | "'" "\\"+ "'" | "'" "\\"+) [^'\x00\\]
| ("''" | "''" "\\"+) [^'\x00\\]
| [^'\x00\\] )*
"'''";
type_ignore = "#" whitespace? "type:" whitespace? "ignore" [^\n\x00]*;
type_comment = "#" whitespace? "type:" whitespace? [^\n\x00]*;
comment = "#" [^\n\x00]*;
// docstring = newline whitespace? string1 | string2;
ws_comment = whitespace? comment? newline;
* { token_loc(loc);
std::string t = token();
throw parser_local::TokenizerError(diag::Diagnostic(
"Token '" + t + "' is not recognized",
diag::Level::Error, diag::Stage::Tokenizer, {
diag::Label("token not recognized", {loc})
})
);
}
end {
token_loc(loc);
if(parenlevel) {
throw parser_local::TokenizerError(
"Parentheses was never closed", {loc});
}
RET(END_OF_FILE);
}
whitespace {
if(cur[0] == '#') { continue; }
if(last_token == yytokentype::TK_NEWLINE && cur[0] == '\n') {
continue;
}
if (indent) {
indent = false;
if (cur[0] != ' ' && cur[0] != '\t'
&& last_indent_length < cur-tok) {
if (last_indent_length == 0) {
last_indent_type = tok[0];
}
if (last_indent_type == tok[0]) {
indent_length.push_back(cur-tok);
last_indent_length = cur-tok;
RET(TK_INDENT);
} else {
token_loc(loc);
throw parser_local::TokenizerError(
"Indentation should be of the same type "
"(either tabs or spaces)", {loc});
}
} else {
token_loc(loc);
throw parser_local::TokenizerError(
"Expected an indented block.", {loc});
}
} else {
if(last_token == yytokentype::TK_NEWLINE
&& cur[0] != ' ' && cur[0] != '\t') {
if (last_indent_type == tok[0]) {
if (last_indent_length > cur-tok) {
last_indent_length = cur-tok;
dedent = 2;
if (!indent_length.empty()) {
indent_length.pop_back();
}
RET(TK_DEDENT);
}
} else {
token_loc(loc);
throw parser_local::TokenizerError(
"Indentation should be of the same type "
"(either tabs or spaces)", {loc});
}
}
}
continue;
}
// Keywords
"as" { KW(AS) }
"assert" { KW(ASSERT) }
"async" { KW(ASYNC) }
"await" { KW(AWAIT) }
"break" { KW(BREAK) }
"class" { KW(CLASS) }
"continue" { KW(CONTINUE) }
"def" { KW(DEF) }
"del" { KW(DEL) }
"elif" { KW(ELIF) }
"else" { KW(ELSE) }
"except" { KW(EXCEPT) }
"finally" { KW(FINALLY) }
"for" { KW(FOR) }
"from" { KW(FROM) }
"global" { KW(GLOBAL) }
"if" { KW(IF) }
"import" { KW(IMPORT) }
"in" { KW(IN) }
"is" { KW(IS) }
"lambda" { KW(LAMBDA) }
"None" { KW(NONE) }
"nonlocal" { KW(NONLOCAL) }
"pass" { KW(PASS) }
"raise" { KW(RAISE) }
"return" { KW(RETURN) }
"try" { KW(TRY) }
"while" { KW(WHILE) }
"with" { KW(WITH) }
"yield" { KW(YIELD) }
"yield" whitespace "from" whitespace { KW(YIELD_FROM) }
// Soft Keywords
"match" / [^:\n\x00] {
if ((last_token == -1
|| last_token == yytokentype::TK_DEDENT
|| last_token == yytokentype::TK_INDENT
|| last_token == yytokentype::TK_NEWLINE)
&& parenlevel == 0) {
bool is_match_keyword = false;
lex_match_or_case(loc, cur, is_match_keyword);
if (is_match_keyword) {
KW(MATCH);
} else {
token(yylval.string);
RET(TK_NAME);
}
} else {
token(yylval.string);
RET(TK_NAME);
}
}
"case" / [^:\n\x00] {
if ((last_token == yytokentype::TK_INDENT
|| last_token == yytokentype::TK_DEDENT)
&& parenlevel == 0) {
bool is_case_keyword = false;
lex_match_or_case(loc, cur, is_case_keyword);
if (is_case_keyword) {
KW(CASE);
} else {
token(yylval.string);
RET(TK_NAME);
}
} else {
token(yylval.string);
RET(TK_NAME);
}
}
// Tokens
newline {
if(parenlevel) { continue; }
if(cur[0] == '#') { RET(TK_NEWLINE); }
if (last_token == yytokentype::TK_COLON
|| colon_actual_last_token) {
colon_actual_last_token = false;
indent = true;
} else if (cur[0] != ' ' && cur[0] != '\t' && cur[0] != '\n'
&& last_indent_length >= cur-tok) {
last_indent_length = 0;
dedent = 1;
}
RET(TK_NEWLINE);
}
"\\" newline { continue; }
// Single character symbols
"(" { token_loc(loc); record_paren(loc, '('); RET(TK_LPAREN) }
"[" { token_loc(loc); record_paren(loc, '['); RET(TK_LBRACKET) }
"{" { token_loc(loc); record_paren(loc, '{'); RET(TK_LBRACE) }
")" { token_loc(loc); record_paren(loc, ')'); RET(TK_RPAREN) }
"]" { token_loc(loc); record_paren(loc, ']'); RET(TK_RBRACKET) }
"}" { token_loc(loc); record_paren(loc, '}'); RET(TK_RBRACE) }
"+" { RET(TK_PLUS) }
"-" { RET(TK_MINUS) }
"=" { RET(TK_EQUAL) }
":" {
if(cur[0] == '\n' && !parenlevel){
colon_actual_last_token = true;
}
RET(TK_COLON);
}
";" { RET(TK_SEMICOLON) }
"/" { RET(TK_SLASH) }
"%" { RET(TK_PERCENT) }
"," { RET(TK_COMMA) }
"*" { RET(TK_STAR) }
"|" { RET(TK_VBAR) }
"&" { RET(TK_AMPERSAND) }
"." { RET(TK_DOT) }
"~" { RET(TK_TILDE) }
"^" { RET(TK_CARET) }
"@" { RET(TK_AT) }
// Multiple character symbols
">>" { RET(TK_RIGHTSHIFT) }
">")
T(TK_LEFTSHIFT, "