FazBrowse GitHub Viewer
|
Trending
|
URL:
|
Home
Tools:
[Download Repo ZIP]
[View Raw Code]
[Original HTTPS Page]
codegen/codegen/common/basic_tokenized_file.cpp at master-codegen · AyuGram/codegen · GitHub
AyuGram
/
codegen
Public
forked from
desktop-app/codegen
Notifications
You must be signed in to change notification settings
Fork
19
Star
2
Code
Pull requests
2
Actions
Projects
Security and quality
0
Insights
Additional navigation options
Code
Pull requests
Actions
Projects
Security and quality
Insights
Expand file tree
Breadcrumbs
codegen
/
codegen
/
common
/
basic_tokenized_file.cpp
Copy path
More file actions
More file actions
Latest commit
History
History
History
299 lines (264 loc) · 8.27 KB
Breadcrumbs
codegen
/
codegen
/
common
/
basic_tokenized_file.cpp
Copy path
File metadata and controls
299 lines (264 loc) · 8.27 KB
Raw
Copy raw file
Download raw file
Open symbols panel
Edit and raw actions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
//
This file is part of Desktop App Toolkit,
//
a set of libraries for developing nice desktop applications.
//
//
For license and copyright information please follow this link:
//
https://github.com/desktop-app/legal/blob/master/LEGAL
//
#
include
"
codegen/common/basic_tokenized_file.h
"
#
include
"
codegen/common/logging.h
"
#
include
"
codegen/common/clean_file_reader.h
"
#
include
"
codegen/common/checked_utf8_string.h
"
using
Token = codegen::common::BasicTokenizedFile::Token;
using
Type = Token::Type;
namespace
codegen
{
namespace
common
{
namespace
{
constexpr
int
kErrorUnterminatedStringLiteral
=
201
;
constexpr
int
kErrorIncorrectUtf8String
=
202
;
constexpr
int
kErrorIncorrectToken
=
203
;
constexpr
int
kErrorUnexpectedToken
=
204
;
bool
isDigitChar
(
char
ch) {
return
(ch >=
'
0
'
) && (ch <=
'
9
'
);
}
bool
isNameChar
(
char
ch) {
return
isDigitChar
(ch) || ((ch >=
'
a
'
) && (ch <=
'
z
'
)) || ((ch >=
'
A
'
) && (ch <=
'
Z
'
)) || (ch ==
'
_
'
);
}
bool
isWhitespaceChar
(
char
ch) {
return
(ch ==
'
\n
'
|| ch ==
'
\r
'
|| ch ==
'
'
|| ch ==
'
\t
'
);
}
Token
invalidToken
() {
return
{ Type::Invalid,
QString
(),
ConstUtf8String
(
nullptr
,
0
),
false
};
}
}
//
namespace
BasicTokenizedFile::BasicTokenizedFile
(
const
QString &filepath) : reader_(filepath) {
}
BasicTokenizedFile::BasicTokenizedFile
(
const
QByteArray &content,
const
QString &filepath) : reader_(content, filepath) {
}
bool
BasicTokenizedFile::putBack
() {
if
(currentToken_ >
0
) {
--currentToken_;
return
true
;
}
return
false
;
}
Token
BasicTokenizedFile::getAnyToken
() {
if
(currentToken_ >= tokens_.
size
()) {
if
(
readToken
() == Type::Invalid) {
return
invalidToken
();
}
}
return
tokens_.
at
(currentToken_++);
}
Token
BasicTokenizedFile::getToken
(Type typeCondition) {
if
(
auto
token =
getAnyToken
()) {
if
(token.
type
== typeCondition) {
return
token;
}
putBack
();
}
return
invalidToken
();
}
Type
BasicTokenizedFile::readToken
() {
auto
result =
readOneToken
(StartWithWhitespace::Allow);
//
Try to read double token.
if
(result == Type::Int) {
if
(
readOneToken
(StartWithWhitespace::Deny) == Type::Dot) {
//
We got int and dot, so it is double already.
result =
uniteLastTokens
(Type::Double);
//
Try to read one more int (after dot).
if
(
readOneToken
(StartWithWhitespace::Deny) == Type::Int) {
result =
uniteLastTokens
(Type::Double);
}
}
}
else
if
(result == Type::Dot) {
if
(
readOneToken
(StartWithWhitespace::Deny) == Type::Int) {
//
We got dot and int, so it is double.
result =
uniteLastTokens
(Type::Double);
}
}
return
result;
}
Type
BasicTokenizedFile::readOneToken
(StartWithWhitespace condition) {
skipWhitespaces
();
if
(tokenStartWhitespace_ && condition == StartWithWhitespace::Deny) {
return
Type::Invalid;
}
if
(reader_.
atEnd
()) {
return
Type::Invalid;
}
auto
ch = reader_.
currentChar
();
if
(ch ==
'
"
'
) {
return
readString
();
}
else
if
(
isNameChar
(ch)) {
return
readNameOrNumber
();
}
return
readSingleLetter
();
}
Type
BasicTokenizedFile::saveToken
(Type type,
const
QString &value) {
ConstUtf8String original = { tokenStart_, reader_.
currentPtr
() };
tokens_.
push_back
({ type, value, original, tokenStartWhitespace_ });
return
type;
}
Type
BasicTokenizedFile::uniteLastTokens
(Type type) {
auto
size = tokens_.
size
();
if
(size <
2
) {
return
Type::Invalid;
}
auto
&
token
(tokens_[size -
2
]);
auto
originalFrom = token.
original
.
data
();
auto
originalTill = tokens_.
back
().
original
.
end
();
token.
type
= type;
token.
original
= { originalFrom, originalTill };
token.
value
+= tokens_.
back
().
value
;
tokens_.
pop_back
();
return
type;
}
QString
BasicTokenizedFile::getCurrentLineComment
() {
if
(lineNumber_ > singleLineComments_.
size
()) {
reader_.
logError
(
kErrorInternal
, lineNumber_) <<
"
internal tokenizer error (line number larger than comments list size).
"
;
failed_ =
true
;
return
QString
();
}
auto
commentBytes = singleLineComments_[lineNumber_ -
1
].
mid
(
2
);
//
Skip "//"
CheckedUtf8String
comment
(commentBytes);
if
(!comment.
isValid
()) {
reader_.
logError
(
kErrorIncorrectUtf8String
, lineNumber_) <<
"
incorrect UTF-8 string in the comment.
"
;
failed_ =
true
;
return
QString
();
}
return
comment.
toString
().
trimmed
();
}
Type
BasicTokenizedFile::readNameOrNumber
() {
while
(!reader_.
atEnd
()) {
if
(!
isDigitChar
(reader_.
currentChar
())) {
break
;
}
reader_.
skipChar
();
}
bool
onlyDigits =
true
;
while
(!reader_.
atEnd
()) {
if
(!
isNameChar
(reader_.
currentChar
())) {
break
;
}
onlyDigits =
false
;
reader_.
skipChar
();
}
return
saveToken
(onlyDigits ? Type::Int : Type::Name);
}
Type
BasicTokenizedFile::readString
() {
reader_.
skipChar
();
auto
offset = reader_.
currentPtr
();
QByteArray value;
while
(!reader_.
atEnd
()) {
auto
ch = reader_.
currentChar
();
if
(ch ==
'
"
'
) {
if
(reader_.
currentPtr
() > offset) {
value.
append
(offset, reader_.
currentPtr
() - offset);
}
break
;
}
if
(ch ==
'
\n
'
) {
reader_.
logError
(
kErrorUnterminatedStringLiteral
, lineNumber_) <<
"
unterminated string literal.
"
;
failed_ =
true
;
return
Type::Invalid;
}
if
(ch ==
'
\\
'
) {
if
(reader_.
currentPtr
() > offset) {
value.
append
(offset, reader_.
currentPtr
() - offset);
}
reader_.
skipChar
();
ch = reader_.
currentChar
();
if
(reader_.
atEnd
() || ch ==
'
\n
'
) {
reader_.
logError
(
kErrorUnterminatedStringLiteral
, lineNumber_) <<
"
unterminated string literal.
"
;
failed_ =
true
;
return
Type::Invalid;
}
offset = reader_.
currentPtr
() +
1
;
if
(ch ==
'
n
'
) {
value.
append
(
'
\n
'
);
}
else
if
(ch ==
'
t
'
) {
value.
append
(
'
\t
'
);
}
else
if
(ch ==
'
"
'
) {
value.
append
(
'
"
'
);
}
else
if
(ch ==
'
\\
'
) {
value.
append
(
'
\\
'
);
}
}
reader_.
skipChar
();
}
if
(reader_.
atEnd
()) {
reader_.
logError
(
kErrorUnterminatedStringLiteral
, lineNumber_) <<
"
unterminated string literal.
"
;
failed_ =
true
;
return
Type::Invalid;
}
CheckedUtf8String
checked
(value);
if
(!checked.
isValid
()) {
reader_.
logError
(
kErrorIncorrectUtf8String
, lineNumber_) <<
"
incorrect UTF-8 string literal.
"
;
failed_ =
true
;
return
Type::Invalid;
}
reader_.
skipChar
();
return
saveToken
(Type::String, checked.
toString
());
}
Type
BasicTokenizedFile::readSingleLetter
() {
auto
type = singleLetterTokens_.
value
(reader_.
currentChar
(), Type::Invalid);
if
(type == Type::Invalid) {
reader_.
logError
(
kErrorIncorrectToken
, lineNumber_) <<
"
incorrect token '
"
<< reader_.
currentChar
() <<
"
'
"
;
return
Type::Invalid;
}
reader_.
skipChar
();
return
saveToken
(type);
}
void
BasicTokenizedFile::skipWhitespaces
() {
if
(reader_.
atEnd
())
return
;
auto
ch = reader_.
currentChar
();
tokenStartWhitespace_ =
isWhitespaceChar
(ch);
if
(tokenStartWhitespace_) {
do
{
if
(ch ==
'
\n
'
) {
++lineNumber_;
}
reader_.
skipChar
();
ch = reader_.
currentChar
();
}
while
(!reader_.
atEnd
() &&
isWhitespaceChar
(ch));
}
tokenStart_ = reader_.
currentPtr
();
}
LogStream
operator
<<(LogStream &&stream, BasicTokenizedFile::Token::Type type) {
const
char
*value =
"
'invalid'
"
;
switch
(type) {
case
Type::Invalid:
break
;
case
Type::Int: value =
"
'int'
"
;
break
;
case
Type::Double: value =
"
'double'
"
;
break
;
case
Type::String: value =
"
'string'
"
;
break
;
case
Type::LeftParenthesis: value =
"
'('
"
;
break
;
case
Type::RightParenthesis: value =
"
')'
"
;
break
;
case
Type::LeftBrace: value =
"
'{'
"
;
break
;
case
Type::RightBrace: value =
"
'}'
"
;
break
;
case
Type::LeftBracket: value =
"
'['
"
;
break
;
case
Type::RightBracket: value =
"
']'
"
;
break
;
case
Type::Colon: value =
"
':'
"
;
break
;
case
Type::Semicolon: value =
"
';'
"
;
break
;
case
Type::Comma: value =
"
','
"
;
break
;
case
Type::Dot: value =
"
'.'
"
;
break
;
case
Type::Number: value =
"
'#'
"
;
break
;
case
Type::Plus: value =
"
'+'
"
;
break
;
case
Type::Minus: value =
"
'-'
"
;
break
;
case
Type::Equals: value =
"
'='
"
;
break
;
case
Type::Name: value =
"
'identifier'
"
;
break
;
}
return
std::forward<LogStream>(stream) << value;
}
LogStream
BasicTokenizedFile::logError
(
int
code)
const
{
return
reader_.
logError
(code, lineNumber_);
}
LogStream
BasicTokenizedFile::logErrorUnexpectedToken
()
const
{
if
(currentToken_ < tokens_.
size
()) {
auto
token = tokens_.
at
(currentToken_).
original
.
toStdString
();
return
logError
(
kErrorUnexpectedToken
) <<
"
unexpected token '
"
<< token <<
"
', expected
"
;
}
return
logError
(
kErrorUnexpectedToken
) <<
"
unexpected token, expected
"
;
}
BasicTokenizedFile::~BasicTokenizedFile
() =
default
;
}
//
namespace common
}
//
namespace codegen
Back
|
FazBrowse Home
|
New Git URL