SlashEq,

Slash,

LeftShiftEq,

LeftShift,

Spaceship,

LessEq,

Less,

RightShiftEq,

RightShift,

GreaterEq,

Greater,

PlusPlus,

PlusEq,

Plus,

MinusMinus,

MinusEq,

Arrow,

Minus,

LogicalOrEq,

LogicalOr,

PipeEq,

Pipe,

LogicalAndEq,

LogicalAnd,

MultiplyEq,

Multiply,

ModuloEq,

Modulo,

AmpersandEq,

Ampersand,

CaretEq,

Caret,

TildeEq,

Tilde,

EqualComparison,

Assignment,

NotEqualComparison,

Not,

LeftBrace,

RightBrace,

LeftParen,

RightParen,

LeftBracket,

RightBracket,

Scope,

Colon,

Semicolon,

Comma,

Dot,

DotDot,

Ellipsis,

EllipsisLess,

EllipsisEqual,

QuestionMark,

At,

Dollar,

FloatLiteral,

BinaryLiteral,

DecimalLiteral,

HexadecimalLiteral,

StringLiteral,

CharacterLiteral,

UserDefinedLiteralSuffix,

Keyword,

Cpp1MultiKeyword,

Cpp2FixedType,

Identifier,

None = 127

switch (l) {

break;case lexeme::FloatLiteral:

case lexeme::BinaryLiteral:

case lexeme::DecimalLiteral:

case lexeme::HexadecimalLiteral:

case lexeme::StringLiteral:

case lexeme::CharacterLiteral: return true;

break;default: return false;

}

-> lexeme

{

switch (l) {

break;case lexeme::LeftBrace: return lexeme::RightBrace;

break;case lexeme::LeftBracket: return lexeme::RightBracket;

break;case lexeme::LeftParen: return lexeme::RightParen;

break;default: return lexeme::None;

}

-> std::string

{

switch (l) {

break;case lexeme::SlashEq: return "SlashEq";

break;case lexeme::Slash: return "Slash";

break;case lexeme::LeftShiftEq: return "LeftShiftEq";

break;case lexeme::LeftShift: return "LeftShift";

break;case lexeme::Spaceship: return "Spaceship";

break;case lexeme::LessEq: return "LessEq";

break;case lexeme::Less: return "Less";

break;case lexeme::RightShiftEq: return "RightShiftEq";

break;case lexeme::RightShift: return "RightShift";

break;case lexeme::GreaterEq: return "GreaterEq";

break;case lexeme::Greater: return "Greater";

break;case lexeme::PlusPlus: return "PlusPlus";

break;case lexeme::PlusEq: return "PlusEq";

break;case lexeme::Plus: return "Plus";

break;case lexeme::MinusMinus: return "MinusMinus";

break;case lexeme::MinusEq: return "MinusEq";

break;case lexeme::Arrow: return "Arrow";

break;case lexeme::Minus: return "Minus";

break;case lexeme::LogicalOrEq: return "LogicalOrEq";

break;case lexeme::LogicalOr: return "LogicalOr";

break;case lexeme::PipeEq: return "PipeEq";

break;case lexeme::Pipe: return "Pipe";

break;case lexeme::LogicalAndEq: return "LogicalAndEq";

break;case lexeme::LogicalAnd: return "LogicalAnd";

break;case lexeme::MultiplyEq: return "MultiplyEq";

break;case lexeme::Multiply: return "Multiply";

break;case lexeme::ModuloEq: return "ModuloEq";

break;case lexeme::Modulo: return "Modulo";

break;case lexeme::AmpersandEq: return "AmpersandEq";

break;case lexeme::Ampersand: return "Ampersand";

break;case lexeme::CaretEq: return "CaretEq";

break;case lexeme::Caret: return "Caret";

break;case lexeme::TildeEq: return "TildeEq";

break;case lexeme::Tilde: return "Tilde";

break;case lexeme::EqualComparison: return "EqualComparison";

break;case lexeme::Assignment: return "Assignment";

break;case lexeme::NotEqualComparison: return "NotEqualComparison";

break;case lexeme::Not: return "Not";

break;case lexeme::LeftBrace: return "LeftBrace";

break;case lexeme::RightBrace: return "RightBrace";

break;case lexeme::LeftParen: return "LeftParen";

break;case lexeme::RightParen: return "RightParen";

break;case lexeme::LeftBracket: return "LeftBracket";

break;case lexeme::RightBracket: return "RightBracket";

break;case lexeme::Scope: return "Scope";

break;case lexeme::Colon: return "Colon";

break;case lexeme::Semicolon: return "Semicolon";

break;case lexeme::Comma: return "Comma";

break;case lexeme::Dot: return "Dot";

break;case lexeme::DotDot: return "DotDot";

break;case lexeme::Ellipsis: return "Ellipsis";

break;case lexeme::EllipsisLess: return "EllipsisLess";

break;case lexeme::EllipsisEqual: return "EllipsisEqual";

break;case lexeme::QuestionMark: return "QuestionMark";

break;case lexeme::At: return "At";

break;case lexeme::Dollar: return "Dollar";

break;case lexeme::FloatLiteral: return "FloatLiteral";

break;case lexeme::BinaryLiteral: return "BinaryLiteral";

break;case lexeme::DecimalLiteral: return "DecimalLiteral";

break;case lexeme::HexadecimalLiteral: return "HexadecimalLiteral";

break;case lexeme::StringLiteral: return "StringLiteral";

break;case lexeme::CharacterLiteral: return "CharacterLiteral";

break;case lexeme::UserDefinedLiteralSuffix: return "UserDefinedLiteralSuffix";

break;case lexeme::Keyword: return "Keyword";

break;case lexeme::Cpp1MultiKeyword: return "Cpp1MultiKeyword";

break;case lexeme::Cpp2FixedType: return "Cpp2FixedType";

break;case lexeme::Identifier: return "Identifier";

break;case lexeme::None: return "(NONE)";

break;default: return "INTERNAL-ERROR";

}

-> bool

{

return l <= lexeme::Not;

{

token(

char const* start,

auto count,

source_position pos,

lexeme type

)

: sv {start, unchecked_narrow<ulong>(count)}

, pos {pos}

, lex_type{type}

{

}

token(

char const* sz,

source_position pos,

lexeme type

)

: sv {sz}

, pos {pos}

, lex_type{type}

{

}

auto as_string_view() const

-> std::string_view

{

assert (sv.data());

return sv;

}

operator std::string_view() const

{

return as_string_view();

}

auto operator== (token const& t) const

-> bool

{

return operator std::string_view() == t.operator std::string_view();

}

auto operator== (std::string_view s) const

-> bool

{

return s == this->operator std::string_view();

}

auto to_string() const

-> std::string

{

return std::string{sv};

}

friend auto operator<< (auto& o, token const& t)

-> auto&

{

return o << std::string_view(t);

}

auto position_col_shift( colno_t offset )

-> void

{

assert (pos.colno + offset > 0);

pos.colno += offset;

}

auto position() const -> source_position { return pos; }

auto length () const -> int { return unchecked_narrow<int>(sv.size()); }

auto type () const -> lexeme { return lex_type; }

auto set_type(lexeme l) -> void { lex_type = l; }

auto visit(auto& v, int depth) const

-> void

{

v.start(*this, depth);

}

auto remove_prefix_if(std::string_view prefix)

-> void

{

if (

sv.size() > prefix.size()

&& sv.starts_with(prefix)

)

{

sv.remove_prefix(prefix.size());

pos.colno += unchecked_narrow<colno_t>(prefix.size());

}

}

auto set_global_token_order(index_t& counter) const

-> void

{

// In a well-formed program we only expect to set this once

if (global_token_order == 0) {

global_token_order = ++counter;

}

}

auto get_global_token_order() const

-> index_t

{

return global_token_order;

}

std::string_view sv;

source_position pos;

lexeme lex_type;

mutable index_t global_token_order = 0;

-> std::string

{

struct label {

std::string text;

label() {

static auto ordinal = 0; // TODO: static

text = std::to_string(++ordinal);

}

};

static auto labels = std::unordered_map<token const*, label const>{}; // TODO: static

assert (t);

return labels[t].text;

-> std::string

{

return "UnnamedTypeParam"

+ std::to_string(ordinal)

+ "_"

+ labelized_position(t);

std::string_view text,

std::vector<error_entry>& errors,

source_position src_pos

)

-> std::string

{

auto const length = std::ssize(text);

assert(length >= 2);

if (text.back() != '"') {

errors.emplace_back(

source_position( src_pos ),

"not a legal string literal",

false,

true // a noisy fallback error message

);

return {};

}

auto pos = 0;

// Skip prefix to first non-" character

while (

pos < length

&& text[pos] != '"'

)

{

++pos;

}

assert(

pos < length

&& text[pos] == '"'

);

++pos;

auto current_start = pos; // the current offset before which the string has been added to ret

auto parts = string_parts{std::string(text.substr(0, current_start)), // begin sequence ", U", u8" depends on the string type

"\"", // end sequence

string_parts::on_both_ends}; // add opening and closing sequence to generated string

bool escape = false;

// Now we're on the first character of the string itself

for (

;

pos < length && !(!escape && text[pos] == '"');

++pos

)

{

escape = (text[pos] == '\\' && !escape);

// Find the next )$

if (

text[pos] == '$'

&& text[pos-1] == ')'

)

{

// Scan back to find the matching (

auto paren_depth = 1;

auto open = pos - 2;

for( ; open > current_start; --open)

{

if (text[open] == ')') {

++paren_depth;

}

else if (text[open] == '(') {

--paren_depth;

if (paren_depth == 0) {

break;

}

}

}

if (text[open] != '(')

{

errors.emplace_back(

source_position( src_pos.lineno, src_pos.colno + pos ),

"no matching ( for string interpolation ending in )$"

);

return {};

}

// 'open' is now at the matching (

// Put the next non-interpolated chunk straight into ret

// unless it's an empty internal "" chunk

if (

open != current_start

|| parts.empty()

)

{

parts.add_string(text.substr(current_start, open - current_start));

}

// Then put interpolated chunk into ret

auto chunk = std::string{text.substr(open, pos - open)};

{ // unescape chunk string

auto last_it = std::remove_if(

std::begin(chunk),

std::end(chunk),

[escape = false, prev = ' '](const auto& e) mutable {

escape = !escape && prev != '\'' && e == '\\';

prev = e;

return escape;

}

);

chunk.erase(last_it, std::end(chunk));

}

// This chunk string is now in the form "(some_capture_text)",

// which might include a :formatter suffix like "(capture_text:formatter)"

if (std::ssize(chunk) < 1)

{

errors.emplace_back(

source_position( src_pos.lineno, src_pos.colno + pos ),

"string interpolation must not be empty"

);

return {};

}

if (chunk.ends_with(':'))

{

errors.emplace_back(

source_position( src_pos.lineno, src_pos.colno + pos ),

"string interpolation ':' must be followed by a std::formatter specifier"

);

return {};

}

// If there's a :formatter suffix, decorate it as: ,"{:formatter}"

if (auto colon = chunk.find_last_of(':');

colon != chunk.npos

&& chunk[colon-1] != ':' // ignore :: scope resolution

)

{

chunk.insert(colon, ",\"{");

chunk.insert(chunk.size()-1, "}\"");

}

parts.add_code("cpp2::to_string" + chunk);

current_start = pos+1;

}

}

// Now we should be on the the final " closing the string

assert(

pos == length-1

&& text[pos] == '"'

);

// Put the final non-interpolated chunk straight into ret

parts.add_string(text.substr(current_start, std::ssize(text)-current_start-1));

return parts.generate();

const std::string& opening_seq,

const std::string& closing_seq,

string_parts::adds_sequences closing_strategy,

std::string_view text,

std::vector<error_entry>& errors,

source_position src_pos

)

-> string_parts

{

auto const length = std::ssize(text);

auto pos = 0;

auto current_start = pos; // the current offset before which the string has been added to ret

string_parts parts{opening_seq, closing_seq, closing_strategy};

// Now we're on the first character of the string itself

for ( ; pos < length; ++pos )

{

// Find the next )$

if (text[pos] == '$' && text[pos-1] == ')')

{

// Scan back to find the matching (

auto paren_depth = 1;

auto open = pos - 2;

for( ; open > current_start; --open)

{

if (text[open] == ')') {

++paren_depth;

}

else if (text[open] == '(') {

--paren_depth;

if (paren_depth == 0) {

break;

}

}

}

if (text[open] != '(')

{

errors.emplace_back(

source_position( src_pos.lineno, src_pos.colno + pos ),

"no matching ( for string interpolation ending in )$"

);

return parts;

}

// 'open' is now at the matching (

// Put the next non-empty non-interpolated chunk straight into ret

if (open != current_start) {

parts.add_string(text.substr(current_start, open - current_start));

}

// Then put interpolated chunk into ret

parts.add_code("cpp2::to_string" + std::string{text.substr(open, pos - open)});

current_start = pos+1;

}

}

// Put the final non-interpolated chunk straight into ret

if (current_start < std::ssize(text)) {

parts.add_string(text.substr(current_start));

}

return parts;

std::string& mutable_line,

int const lineno,

bool& in_comment,

std::string& current_comment,

source_position& current_comment_start,

std::vector<token>& tokens,

std::vector<comment>& comments,

std::vector<error_entry>& errors,

std::optional<raw_string>& raw_string_multiline

)

-> bool

{

auto const& line = mutable_line; // most accesses will be const, so give that the nice name

auto original_size = std::ssize(tokens);

auto i = colno_t{0};

// Token merging helpers

//

auto merge_cpp1_multi_token_fundamental_type_names = [&]

{

// If the last token is a non-Cpp1MultiKeyword, we might be at the end

// of a sequence of Cpp1MultiKeyword tokens that need to be merged

// First, check the last token... only proceed if it is NOT one of those

auto i = std::ssize(tokens)-1;

if (

i < 0

|| tokens[i].type() == lexeme::Cpp1MultiKeyword

)

{

return;

}

// Next, check the two tokens before that... only proceed if they ARE those

--i;

if (

i < 1

|| tokens[i].type() != lexeme::Cpp1MultiKeyword

|| tokens[i-1].type() != lexeme::Cpp1MultiKeyword

)

{

// If this is just one such token, changed its type to regular ::Keyword

if (

i >= 0

&& tokens[i].type() == lexeme::Cpp1MultiKeyword

)

{

tokens[i].set_type( lexeme::Keyword );

}

return;

}

// OK, we have found the end of a sequence of 1 or more Cpp1MultiKeywords, so

// replace them with a single synthesized token that contains all their text

//

// Note: It's intentional that this is a kind of token that can contain whitespace

// Remember the last (non-Cpp1MultiKeyword) token so we can put it back

auto last_token = tokens.back();

tokens.pop_back();

assert(tokens.back().type() == lexeme::Cpp1MultiKeyword);

auto pos = tokens.back().position();

auto num_merged_tokens = 0;

auto is_char = 0;

auto is_short = 0;

auto is_int = 0;

auto is_long = 0;

auto is_double = 0;

auto is_signed = 0;

auto is_unsigned = 0;

generated_text.push_back( "" );

while(

!tokens.empty()

&& tokens.back().type() == lexeme::Cpp1MultiKeyword

)

{

auto text = tokens.back().to_string();

if (text == "char" ) { ++is_char ; }

if (text == "short" ) { ++is_short ; }

if (text == "int" ) { ++is_int ; }

if (text == "long" ) { ++is_long ; }

if (text == "double" ) { ++is_double ; }

if (text == "signed" ) { ++is_signed ; }

if (text == "unsigned") { ++is_unsigned; }

if (num_merged_tokens > 0) {

generated_text.back() = " " + generated_text.back();

}

generated_text.back() = text + generated_text.back();

pos = tokens.back().position();

tokens.pop_back();

++num_merged_tokens;

}

// It's an error to have more than one of these, but we require that

// the number of tokens has not gone down. So just push back as many

// tokens as we merged. This will ensure that the token count remains

// the same.

for (auto i = 0; i < num_merged_tokens; i++) {

tokens.push_back({

&generated_text.back()[0],

std::ssize(generated_text.back()),

pos,

lexeme::Keyword

});

}

if (num_merged_tokens > 1)

{

auto alt = std::string{};

if (is_char && is_signed) { alt = "'i8' (usually best) or 'cpp2::_schar'"; }

else if (is_char && is_unsigned) { alt = "'u8' (usually best) or 'cpp2::_uchar'"; }

else if (is_short && !is_unsigned) { alt = "'short'" ; }

else if (is_short && is_unsigned) { alt = "'ushort'" ; }

else if (is_long == 1 && !is_unsigned) { alt = "'long'" ; }

else if (is_long == 1 && is_unsigned) { alt = "'ulong'" ; }

else if (is_long > 1 && !is_unsigned) { alt = "'longlong'" ; }

else if (is_long > 1 && is_unsigned) { alt = "'ulonglong'" ; }

else if (is_int && !is_unsigned) { alt = "'int'" ; }

else if (is_int && is_unsigned) { alt = "'uint'" ; }

else if (is_double && is_long) { alt = "'longdouble'" ; }

if (std::ssize(alt) > 0) {

errors.emplace_back(

pos,

"'" + tokens.back().to_string() + "' - did you mean " + alt + "?"

);

}

errors.emplace_back(

pos,

"'" + tokens.back().to_string() + "' is an old-style C/C++ multi-word keyword type\n"

" - most such types should be used only for interoperability with older code\n"

" - using those when you need them is fine, but name them with these short names instead:\n"

" short, ushort, int, uint, long, ulong, longlong, ulonglong, longdouble, _schar, _uchar\n"

" - see also cpp2util.h > \"Convenience names for integer types\""

);

}

tokens.push_back(last_token);

};

auto merge_operator_function_names = [&]

{

auto i = std::ssize(tokens)-1;

// If the third-to-last token is "operator", we may need to

// merge an "operator?" name into a single identifier token

if (

i >= 2

&& tokens[i-2] == "operator"

)

{

// If the tokens after "operator" are ">" and without whitespace one of ">=" ">" "="

if (

tokens[i-1].type() == lexeme::Greater

&& (tokens[i-1].position() == source_position{tokens[i].position().lineno, tokens[i].position().colno-1})

&& (tokens[i].type() == lexeme::GreaterEq || tokens[i].type() == lexeme::Greater || tokens[i].type() == lexeme::Assignment))

{

// Merge all three tokens into an identifier

generated_text.push_back( "operator" + tokens[i-1].to_string() + tokens[i].to_string() );

tokens.pop_back();

tokens.pop_back();

auto pos = tokens.back().position();

tokens.pop_back();

tokens.push_back({

&generated_text.back()[0],

std::ssize(generated_text.back()),

pos,

lexeme::Identifier

});

}

// Else if token after "operator" is a single-token operator symbol

else if (is_operator(tokens[i-1].type()))

{

// Merge just "operator" + the symbol into an identifier,

generated_text.push_back( "operator" + tokens[i-1].to_string() );

// and preserve the last token separately

auto last_token = tokens.back();

tokens.pop_back();

tokens.pop_back();

auto pos = tokens.back().position();

tokens.pop_back();

tokens.push_back({

&generated_text.back()[0],

std::ssize(generated_text.back()),

pos,

lexeme::Identifier

});

tokens.push_back(last_token);

}

// Else if token after "operator" is a two-token operator symbol

else if (

(tokens[i-1].type() == lexeme::LeftParen && tokens[i].type() == lexeme::RightParen)

|| (tokens[i-1].type() == lexeme::LeftBracket && tokens[i].type() == lexeme::RightBracket)

)

{

// Merge just "operator" + the symbols into an identifier,

generated_text.push_back( "operator" + tokens[i-1].to_string() + tokens[i].to_string() );

tokens.pop_back();

tokens.pop_back();

auto pos = tokens.back().position();

tokens.pop_back();

tokens.push_back({

&generated_text.back()[0],

std::ssize(generated_text.back()),

pos,

lexeme::Identifier

});

}

}

};

auto qualify_cpp2_special_names = [&]

{

auto i = std::ssize(tokens)-1;

// If the last three tokens are "unique/shared" "." "new", add "cpp2::"

if (

i >= 3

&& (tokens[i-3] != "::" && tokens[i-3].type() != lexeme::Dot && tokens[i - 3].type() != lexeme::DotDot)

&& (tokens[i-2] == "unique" || tokens[i-2] == "shared")

&& tokens[i-1].type() == lexeme::Dot

&& tokens[i] == "new"

)

{

auto pos = tokens[i-2].position();

generated_text.push_back( "cpp2" );

tokens.insert(

tokens.end()-3,

token{

&generated_text.back()[0],

std::ssize(generated_text.back()),

pos,

lexeme::Identifier

}

);

generated_text.push_back( "::" );

tokens.insert(

tokens.end()-3,

token{

&generated_text.back()[0],

std::ssize(generated_text.back()),

pos,

lexeme::Scope

}

);

}

};

// Local helper functions for readability

//

auto peek = [&](int num) {

return

(i+num < std::ssize(line) && i+num >= 0)

? line[i+num]

: '\0';

};

auto store = [&](auto num, lexeme type)

{

tokens.push_back({

&line[i],

num,

source_position(lineno, i + 1),

type

});

i += unchecked_narrow<int>(num-1);

merge_cpp1_multi_token_fundamental_type_names();

merge_operator_function_names();

qualify_cpp2_special_names();

};

//-----------------------------------------------------

// These functions return the length of sequence if

// present at the current location, else 0

//G simple-escape-sequence:

//G '\' { any member of the basic character set except u, U, or x }

//G

auto peek_is_simple_escape_sequence = [&](int offset)

{

auto peek1 = peek(offset);

auto peek2 = peek(1 + offset);

if (

peek1 == '\\'

&& peek2 != 'u'

&& peek2 != 'U'

&& peek2 != 'x'

)

{

return 2;

}

return 0;

};

//G simple-hexadecimal-digit-sequence:

//G hexadecimal-digit

//G simple-hexadecimal-digit-sequence hexadecimal-digit

//G

//G hexadecimal-escape-sequence:

//G '\x' simple-hexadecimal-digit-sequence

//G '\x{' simple-hexadecimal-digit-sequence '}'

//G

auto peek_is_hexadecimal_escape_sequence = [&](int offset)

{

if (

peek(offset) == '\\'

&& peek(1+offset) == 'x'

&& (

is_hexadecimal_digit(peek(2+offset))

|| (peek(2+offset) == '{' && is_hexadecimal_digit(peek(3+offset)))

)

)

{

auto has_bracket = peek(2+offset) == '{';

auto j = 3;

if (has_bracket) { ++j; }

while (

peek(j+offset)

&& is_hexadecimal_digit(peek(j+offset))

)

{

++j;

}

if (has_bracket) {

if (peek(j+offset) == '}') {

++j;

} else {

errors.emplace_back(

source_position(lineno, i + offset),

"invalid hexadecimal escape sequence - \\x{ must"

" be followed by hexadecimal digits and a closing }"

);

return 0;

}

}

return j;

}

return 0;

};

//G universal-character-name:

//G '\u' hex-quad

//G '\U' hex-quad hex-quad

//G '\u{' simple-hexadecimal-digit-sequence '}'

//G

//G hex-quad:

//G hexadecimal-digit hexadecimal-digit hexadecimal-digit hexadecimal-digit

//G

auto peek_is_universal_character_name = [&](colno_t offset)

{

if (

peek(offset) == '\\'

&& peek(1 + offset) == 'u'

&& peek(2 + offset) != '{'

)

{

auto j = 2;

while (

j <= 5

&& is_hexadecimal_digit(peek(j + offset))

)