GitHub Viewer

#include #include #include #include #include #include namespace LCompilers::LPython { template bool adddgt(uint64_t &u, uint64_t d) { if (u > (std::numeric_limits::max() - d) / base) { return false; } u = u * base + d; return true; } bool lex_oct(const unsigned char *s, const unsigned char *e, uint64_t &u) { for (u = 0, ++s; s < e; ++s) { if (!adddgt(u, *s - 0x30u)) { return false; } } return true; } bool lex_dec(const unsigned char *s, const unsigned char *e, uint64_t &u) { for (u = 0; s < e; ++s) { if (*s == '_') continue; if (!adddgt(u, *s - 0x30u)) { return false; } } return true; } void lex_dec_int_large(Allocator &al, const unsigned char *s, const unsigned char *e, BigInt::BigInt &u) { uint64_t ui; if (lex_dec(s, e, ui)) { if (ui = MAX_PAREN_LEVEL) { throw parser_local::TokenizerError( "Too many nested parentheses", {loc}); } paren_stack[parenlevel] = c; parenlevel++; break; case ')': case ']': case '}': if(parenlevel < 1) { throw parser_local::TokenizerError( "Parenthesis unexpected", {loc}); } parenlevel--; char prev_paren = paren_stack[parenlevel]; if(!((prev_paren == '(' && c == ')') || (prev_paren == '[' && c == ']') || (prev_paren == '{' && c == '}'))) { throw parser_local::TokenizerError( "Parentheses does not match", {loc}); } break; } return; } #define KW(x) token(yylval.string); RET(KW_##x); #define RET(x) token_loc(loc); last_token=yytokentype::x; return yytokentype::x; int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnostics &/*diagnostics*/) { if(dedent == 1) { // Removes the indent completely i.e., to level 0 if(!indent_length.empty()) { indent_length.pop_back(); return yytokentype::TK_DEDENT; } else { dedent = 0; } } else if(dedent == 2) { // Reduce the indent to `last_indent_length` if((long int)indent_length.back() != last_indent_length) { indent_length.pop_back(); loc.first = loc.last; return yytokentype::TK_DEDENT; } else { dedent = 0; } } for (;;) { tok = cur; /* Re2c has excellent documentation at: https://re2c.org/manual/manual_c.html The first paragraph there explains the basics: * If multiple rules match, the longest match takes precedence * If multiple rules match the same string, the earlier rule takes precedence * Default rule `*` should always be defined, it has the lowest priority regardless of its place and matches any code unit * We use the "Sentinel character" method for end of input: * The end of the input text is denoted with a null character \x00 * Thus the null character cannot be part of the input otherwise * There is one rule to match \x00 to end the parser * No other rule is allowed to match \x00, otherwise the re2c block would parse past the end of the string and segfaults * A special case of the previous point are negated character ranges, such as [^"\x00], where one must include \x00 in it to ensure this rule does not match \x00 (all other rules simply do not mention \x00) * See the "Handling the end of input" section in the re2c documentation for more info The re2c block interacts with the rest of the code via just one pointer variable `cur`. On entering the re2c block, the `cur` variable must point to the first character of the token to be tokenized by the block. The re2c block below then executes on its own until a rule is matched: the action in {} is then executed. In that action `cur` points to the first character of the next token. Before the re2c block we save the current `cur` into `tok`, so that we can use `tok` and `cur` in the action in {} to extract the token that corresponds to the rule that got matched: * `tok` points to the first character of the token * `cur-1` points to the last character of the token * `cur` points to the first character of the next token * `cur-tok` is the length of the token In the action, we do one of: * call `continue` which executes another cycle in the for loop (which will parse the next token); we use this to skip a token * call `return` which returns from this function; we return a token * throw an exception (terminates the tokenizer) In the first two cases, `cur` points to first character of the next token, which becomes `tok` at the next iteration of the loop (either right away after `continue` or after the `lex` function is called again after `return`). See the manual for more details. */ // These two variables are needed by the re2c block below internally, // initialization is not needed. One can think of them as local // variables of the re2c block. unsigned char *mar; //, *ctxmar; /*!re2c re2c:define:YYCURSOR = cur; re2c:define:YYMARKER = mar; // re2c:define:YYCTXMARKER = ctxmar; re2c:yyfill:enable = 0; re2c:define:YYCTYPE = "unsigned char"; end = "\x00"; whitespace = [ \t\v]+; newline = "\n" | "\r\n"; digit = [0-9]; int_oct = "0"[oO]([0-7] | "_" [0-7])+; int_bin = "0"[bB]([01] | "_" [01])+; int_hex = "0"[xX]([0-9a-fA-F] | "_" [0-9a-fA-F])+; digits = digit+ (digit | "_" digit)*; char = [^\x00-\x7F]|[a-zA-Z_]; name = char (char | digit)*; significand = (digits "." digits?) | ("." digits); exp = [eE][-+]? digits; integer = digits | int_oct | int_bin | int_hex; real = (significand exp?) | (digits exp); imag_number = (real | digits)[jJ]; string1 = '"' ('\\'[^\x00] | [^"\x00\n\\])* '"'; string2 = "'" ("\\"[^\x00] | [^'\x00\n\\])* "'"; string3 = '"""' ( '\\'[^\x00] | ('"' | '"' '\\'+ '"' | '"' '\\'+) [^"\x00\\] | ('""' | '""' '\\'+) [^"\x00\\] | [^"\x00\\] )* '"""'; string4 = "'''" ( "\\"[^\x00] | ("'" | "'" "\\"+ "'" | "'" "\\"+) [^'\x00\\] | ("''" | "''" "\\"+) [^'\x00\\] | [^'\x00\\] )* "'''"; type_ignore = "#" whitespace? "type:" whitespace? "ignore" [^\n\x00]*; type_comment = "#" whitespace? "type:" whitespace? [^\n\x00]*; comment = "#" [^\n\x00]*; // docstring = newline whitespace? string1 | string2; ws_comment = whitespace? comment? newline; * { token_loc(loc); std::string t = token(); throw parser_local::TokenizerError(diag::Diagnostic( "Token '" + t + "' is not recognized", diag::Level::Error, diag::Stage::Tokenizer, { diag::Label("token not recognized", {loc}) }) ); } end { token_loc(loc); if(parenlevel) { throw parser_local::TokenizerError( "Parentheses was never closed", {loc}); } RET(END_OF_FILE); } whitespace { if(cur[0] == '#') { continue; } if(last_token == yytokentype::TK_NEWLINE && cur[0] == '\n') { continue; } if (indent) { indent = false; if (cur[0] != ' ' && cur[0] != '\t' && last_indent_length < cur-tok) { if (last_indent_length == 0) { last_indent_type = tok[0]; } if (last_indent_type == tok[0]) { indent_length.push_back(cur-tok); last_indent_length = cur-tok; RET(TK_INDENT); } else { token_loc(loc); throw parser_local::TokenizerError( "Indentation should be of the same type " "(either tabs or spaces)", {loc}); } } else { token_loc(loc); throw parser_local::TokenizerError( "Expected an indented block.", {loc}); } } else { if(last_token == yytokentype::TK_NEWLINE && cur[0] != ' ' && cur[0] != '\t') { if (last_indent_type == tok[0]) { if (last_indent_length > cur-tok) { last_indent_length = cur-tok; dedent = 2; if (!indent_length.empty()) { indent_length.pop_back(); } RET(TK_DEDENT); } } else { token_loc(loc); throw parser_local::TokenizerError( "Indentation should be of the same type " "(either tabs or spaces)", {loc}); } } } continue; } // Keywords "as" { KW(AS) } "assert" { KW(ASSERT) } "async" { KW(ASYNC) } "await" { KW(AWAIT) } "break" { KW(BREAK) } "class" { KW(CLASS) } "continue" { KW(CONTINUE) } "def" { KW(DEF) } "del" { KW(DEL) } "elif" { KW(ELIF) } "else" { KW(ELSE) } "except" { KW(EXCEPT) } "finally" { KW(FINALLY) } "for" { KW(FOR) } "from" { KW(FROM) } "global" { KW(GLOBAL) } "if" { KW(IF) } "import" { KW(IMPORT) } "in" { KW(IN) } "is" { KW(IS) } "lambda" { KW(LAMBDA) } "None" { KW(NONE) } "nonlocal" { KW(NONLOCAL) } "pass" { KW(PASS) } "raise" { KW(RAISE) } "return" { KW(RETURN) } "try" { KW(TRY) } "while" { KW(WHILE) } "with" { KW(WITH) } "yield" { KW(YIELD) } "yield" whitespace "from" whitespace { KW(YIELD_FROM) } // Soft Keywords "match" / [^:\n\x00] { if ((last_token == -1 || last_token == yytokentype::TK_DEDENT || last_token == yytokentype::TK_INDENT || last_token == yytokentype::TK_NEWLINE) && parenlevel == 0) { bool is_match_keyword = false; lex_match_or_case(loc, cur, is_match_keyword); if (is_match_keyword) { KW(MATCH); } else { token(yylval.string); RET(TK_NAME); } } else { token(yylval.string); RET(TK_NAME); } } "case" / [^:\n\x00] { if ((last_token == yytokentype::TK_INDENT || last_token == yytokentype::TK_DEDENT) && parenlevel == 0) { bool is_case_keyword = false; lex_match_or_case(loc, cur, is_case_keyword); if (is_case_keyword) { KW(CASE); } else { token(yylval.string); RET(TK_NAME); } } else { token(yylval.string); RET(TK_NAME); } } [rR][bB] | [bB][rR] | [fF][rR] | [rR][fF] | [rR] | [bB] | [fF] | [uU] { if(cur[0] == '\'' || cur[0] == '"'){ KW(STR_PREFIX); } else { token(yylval.string); RET(TK_NAME); } } // Tokens newline { if(parenlevel) { continue; } if(cur[0] == '#') { RET(TK_NEWLINE); } if (last_token == yytokentype::TK_COLON || colon_actual_last_token) { colon_actual_last_token = false; indent = true; } else if (cur[0] != ' ' && cur[0] != '\t' && cur[0] != '\n' && last_indent_length >= cur-tok) { last_indent_length = 0; dedent = 1; } RET(TK_NEWLINE); } "\\" newline { continue; } // Single character symbols "(" { token_loc(loc); record_paren(loc, '('); RET(TK_LPAREN) } "[" { token_loc(loc); record_paren(loc, '['); RET(TK_LBRACKET) } "{" { token_loc(loc); record_paren(loc, '{'); RET(TK_LBRACE) } ")" { token_loc(loc); record_paren(loc, ')'); RET(TK_RPAREN) } "]" { token_loc(loc); record_paren(loc, ']'); RET(TK_RBRACKET) } "}" { token_loc(loc); record_paren(loc, '}'); RET(TK_RBRACE) } "+" { RET(TK_PLUS) } "-" { RET(TK_MINUS) } "=" { RET(TK_EQUAL) } ":" { if(cur[0] == '\n' && !parenlevel){ colon_actual_last_token = true; } RET(TK_COLON); } ";" { RET(TK_SEMICOLON) } "/" { RET(TK_SLASH) } "%" { RET(TK_PERCENT) } "," { RET(TK_COMMA) } "*" { RET(TK_STAR) } "|" { RET(TK_VBAR) } "&" { RET(TK_AMPERSAND) } "." { RET(TK_DOT) } "~" { RET(TK_TILDE) } "^" { RET(TK_CARET) } "@" { RET(TK_AT) } // Multiple character symbols ">>" { RET(TK_RIGHTSHIFT) } ">") T(TK_LEFTSHIFT, "