FazBrowse GitHub Viewer
|
Trending
|
URL:
|
Home
Tools:
[Download Repo ZIP]
[View Raw Code]
[Original HTTPS Page]
processing/app/src/processing/app/syntax/TokenMarker.java at master · 2php/processing · GitHub
2php
processing
Repository navigation
Code
Pull requests
Actions
Projects
Wiki
Security and quality
Insights
Expand file tree
Breadcrumbs
processing
/
app
/
src
/
processing
/
app
/
syntax
/
TokenMarker.java
Copy path
More file actions
More file actions
Latest commit
History
History
History
343 lines (308 loc) · 12.1 KB
Breadcrumbs
processing
/
app
/
src
/
processing
/
app
/
syntax
/
TokenMarker.java
Copy path
File metadata and controls
343 lines (308 loc) · 12.1 KB
Raw
Copy raw file
Download raw file
Open symbols panel
Edit and raw actions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
/*
* TokenMarker.java - Generic token marker
* Copyright (C) 1998, 1999 Slava Pestov
*
* You may use and modify this package for any purpose. Redistribution is
* permitted, in both source and binary form, provided that this notice
* remains intact in all source distributions of this package.
*/
package
processing
.
app
.
syntax
;
import
javax
.
swing
.
text
.
Segment
;
/**
* A token marker that splits lines of text into tokens. Each token carries
* a length field and an indentification tag that can be mapped to a color
* for painting that token.<p>
*
* For performance reasons, the linked list of tokens is reused after each
* line is tokenized. Therefore, the return value of <code>markTokens</code>
* should only be used for immediate painting. Notably, it cannot be
* cached.
*
* @author Slava Pestov
*/
public
abstract
class
TokenMarker
{
abstract
public
void
addColoring
(
String
keyword
,
String
coloring
);
/**
* A wrapper for the lower-level <code>markTokensImpl</code> method
* that is called to split a line up into tokens.
* @param line The line
* @param lineIndex The line number
*/
public
Token
markTokens
(
Segment
line
,
int
lineIndex
)
{
if
(
lineIndex
>=
length
)
{
throw
new
IllegalArgumentException
(
"Tokenizing invalid line: "
+
lineIndex
);
}
lastToken
=
null
;
LineInfo
info
=
lineInfo
[
lineIndex
];
LineInfo
prev
;
if
(
lineIndex
==
0
)
prev
=
null
;
else
prev
=
lineInfo
[
lineIndex
-
1
];
byte
oldToken
=
info
.
token
;
byte
token
=
markTokensImpl
(
prev
==
null
?
Token
.
NULL
:
prev
.
token
,
line
,
lineIndex
);
info
.
token
=
token
;
/*
* This is a foul hack. It stops nextLineRequested
* from being cleared if the same line is marked twice.
*
* Why is this necessary? It's all JEditTextArea's fault.
* When something is inserted into the text, firing a
* document event, the insertUpdate() method shifts the
* caret (if necessary) by the amount inserted.
*
* All caret movement is handled by the select() method,
* which eventually pipes the new position to scrollTo()
* and calls repaint().
*
* Note that at this point in time, the new line hasn't
* yet been painted; the caret is moved first.
*
* scrollTo() calls offsetToX(), which tokenizes the line
* unless it is being called on the last line painted
* (in which case it uses the text area's painter cached
* token list). What scrollTo() does next is irrelevant.
*
* After scrollTo() has done it's job, repaint() is
* called, and eventually we end up in paintLine(), whose
* job is to paint the changed line. It, too, calls
* markTokens().
*
* The problem was that if the line started a multiline
* token, the first markTokens() (done in offsetToX())
* would set nextLineRequested (because the line end
* token had changed) but the second would clear it
* (because the line was the same that time) and therefore
* paintLine() would never know that it needed to repaint
* subsequent lines.
*
* This bug took me ages to track down, that's why I wrote
* all the relevant info down so that others wouldn't
* duplicate it.
*/
if
(!(
lastLine
==
lineIndex
&&
nextLineRequested
))
nextLineRequested
= (
oldToken
!=
token
);
lastLine
=
lineIndex
;
addToken
(
0
,
Token
.
END
);
return
firstToken
;
}
/**
* An abstract method that splits a line up into tokens. It
* should parse the line, and call <code>addToken()</code> to
* add syntax tokens to the token list. Then, it should return
* the initial token type for the next line.<p>
*
* For example if the current line contains the start of a
* multiline comment that doesn't end on that line, this method
* should return the comment token type so that it continues on
* the next line.
*
* @param token The initial token type for this line
* @param line The line to be tokenized
* @param lineIndex The index of the line in the document,
* starting at 0
* @return The initial token type for the next line
*/
protected
abstract
byte
markTokensImpl
(
byte
token
,
Segment
line
,
int
lineIndex
);
/**
* Returns if the token marker supports tokens that span multiple
* lines. If this is true, the object using this token marker is
* required to pass all lines in the document to the
* <code>markTokens()</code> method (in turn).<p>
*
* The default implementation returns true; it should be overridden
* to return false on simpler token markers for increased speed.
*/
public
boolean
supportsMultilineTokens
()
{
return
true
;
}
/**
* Informs the token marker that lines have been inserted into
* the document. This inserts a gap in the <code>lineInfo</code>
* array.
* @param index The first line number
* @param lines The number of lines
*/
public
void
insertLines
(
int
index
,
int
lines
)
{
if
(
lines
<=
0
)
return
;
length
+=
lines
;
ensureCapacity
(
length
);
int
len
=
index
+
lines
;
System
.
arraycopy
(
lineInfo
,
index
,
lineInfo
,
len
,
lineInfo
.
length
-
len
);
for
(
int
i
=
index
+
lines
-
1
;
i
>=
index
;
i
--)
{
lineInfo
[
i
] =
new
LineInfo
();
}
}
/**
* Informs the token marker that line have been deleted from
* the document. This removes the lines in question from the
* <code>lineInfo</code> array.
* @param index The first line number
* @param lines The number of lines
*/
public
void
deleteLines
(
int
index
,
int
lines
)
{
if
(
lines
<=
0
)
return
;
int
len
=
index
+
lines
;
length
-=
lines
;
System
.
arraycopy
(
lineInfo
,
len
,
lineInfo
,
index
,
lineInfo
.
length
-
len
);
}
/**
* Returns the number of lines in this token marker.
*/
public
int
getLineCount
()
{
return
length
;
}
/**
* Returns true if the next line should be repainted. This
* will return true after a line has been tokenized that starts
* a multiline token that continues onto the next line.
*/
public
boolean
isNextLineRequested
()
{
return
nextLineRequested
;
}
// protected members
/**
* The first token in the list. This should be used as the return
* value from <code>markTokens()</code>.
*/
protected
Token
firstToken
;
/**
* The last token in the list. New tokens are added here.
* This should be set to null before a new line is to be tokenized.
*/
protected
Token
lastToken
;
/**
* An array for storing information about lines. It is enlarged and
* shrunk automatically by the <code>insertLines()</code> and
* <code>deleteLines()</code> methods.
*/
protected
LineInfo
[]
lineInfo
;
/**
* The number of lines in the model being tokenized. This can be
* less than the length of the <code>lineInfo</code> array.
*/
protected
int
length
;
/**
* The last tokenized line.
*/
protected
int
lastLine
;
/**
* True if the next line should be painted.
*/
protected
boolean
nextLineRequested
;
/**
* Creates a new <code>TokenMarker</code>. This DOES NOT create
* a lineInfo array; an initial call to <code>insertLines()</code>
* does that.
*/
protected
TokenMarker
()
{
lastLine
= -
1
;
}
/**
* Ensures that the <code>lineInfo</code> array can contain the
* specified index. This enlarges it if necessary. No action is
* taken if the array is large enough already.<p>
*
* It should be unnecessary to call this under normal
* circumstances; <code>insertLine()</code> should take care of
* enlarging the line info array automatically.
*
* @param index The array index
*/
protected
void
ensureCapacity
(
int
index
)
{
if
(
lineInfo
==
null
)
lineInfo
=
new
LineInfo
[
index
+
1
];
else
if
(
lineInfo
.
length
<=
index
)
{
LineInfo
[]
lineInfoN
=
new
LineInfo
[(
index
+
1
) *
2
];
System
.
arraycopy
(
lineInfo
,
0
,
lineInfoN
,
0
,
lineInfo
.
length
);
lineInfo
=
lineInfoN
;
}
}
/**
* Adds a token to the token list.
* @param length The length of the token
* @param id The id of the token
*/
protected
void
addToken
(
int
length
,
byte
id
)
{
if
(
id
>=
Token
.
INTERNAL_FIRST
&&
id
<=
Token
.
INTERNAL_LAST
)
throw
new
InternalError
(
"Invalid id: "
+
id
);
if
(
length
==
0
&&
id
!=
Token
.
END
)
return
;
if
(
firstToken
==
null
)
{
firstToken
=
new
Token
(
length
,
id
);
lastToken
=
firstToken
;
}
else
if
(
lastToken
==
null
)
{
lastToken
=
firstToken
;
firstToken
.
length
=
length
;
firstToken
.
id
=
id
;
}
else
if
(
lastToken
.
next
==
null
)
{
lastToken
.
next
=
new
Token
(
length
,
id
);
lastToken
=
lastToken
.
next
;
}
else
{
lastToken
=
lastToken
.
next
;
lastToken
.
length
=
length
;
lastToken
.
id
=
id
;
}
}
/**
* Inner class for storing information about tokenized lines.
*/
private
static
class
LineInfo
{
/**
* Creates a new LineInfo object with token = Token.NULL
* and obj = null.
*/
public
LineInfo
()
{
}
/**
* Creates a new LineInfo object with the specified
* parameters.
*/
// public LineInfo(byte token, Object obj)
// {
// this.token = token;
// this.obj = obj;
// }
/**
* The id of the last token of the line.
*/
public
byte
token
;
/**
* This is for use by the token marker implementations
* themselves. It can be used to store anything that
* is an object and that needs to exist on a per-line
* basis.
*/
// public Object obj;
}
}
Back
|
FazBrowse Home
|
New Git URL