src/vs/editor/common/tokens/lineTokens.ts
527 LOC · 368 covered · 159 uncovered · 98 ranges · 1774 concepts · 30 introducers · 885 tests
File neighbourhood
The centred file is linked to every concept that introduces one of its ranges, every test that runs code from the file, and the gray connector concepts standing between those tests and the file's own introducer concepts. Undirected links join concepts to every file where they introduce source and concepts to the tests they introduce; arrows show specialization between the displayed concepts and bridge only concepts omitted from this view. Concept colors match the source ranges below; connector concepts have no source color and are shown in gray.
Focused file, its introducer and connector concepts, their introduced files, and tests that run code from the file
In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the related-file, concept, and source links on this page.
Graph controls are ready.
Interactive rendering requires JavaScript and WebGL. Use the related-file, concept, and source links on this page while the interactive map is unavailable.
1
>
/*---------------------------------------------------------------------------------------------
lineTokens.ts ×55
2
>
* Copyright (c) Microsoft Corporation. All rights reserved.
3
>
* Licensed under the MIT License. See License.txt in the project root for license information.
4
>
*--------------------------------------------------------------------------------------------*/
5
>
6
>
import { ILanguageIdCodec } from '../languages.js';
7
>
import { FontStyle, ColorId, StandardTokenType, MetadataConsts, ITokenPresentation, TokenMetadata } from '../encodedTokenAttributes.js';
8
>
import { IPosition } from '../core/position.js';
9
>
import { ITextModel } from '../model.js';
10
>
import { OffsetRange } from '../core/ranges/offsetRange.js';
11
>
import { onUnexpectedError } from '../../../base/common/errors.js';
12
>
13
>
14
>
export interface IViewLineTokens {
15
>
languageIdCodec: ILanguageIdCodec;
16
>
equals(other: IViewLineTokens): boolean;
17
>
getCount(): number;
18
>
getStandardTokenType(tokenIndex: number): StandardTokenType;
19
>
getForeground(tokenIndex: number): ColorId;
20
>
getEndOffset(tokenIndex: number): number;
21
>
getClassName(tokenIndex: number): string;
22
>
getInlineStyle(tokenIndex: number, colorMap: string[]): string;
23
>
getPresentation(tokenIndex: number): ITokenPresentation;
24
>
findTokenIndexAtOffset(offset: number): number;
25
>
getLineContent(): string;
26
>
getMetadata(tokenIndex: number): number;
27
>
getLanguageId(tokenIndex: number): string;
28
>
getTokenText(tokenIndex: number): string;
29
>
forEach(callback: (tokenIndex: number) => void): void;
30
>
}
31
>
32
>
export class LineTokens implements IViewLineTokens {
33
>
public static createEmpty(lineContent: string, decoder: ILanguageIdCodec): LineTokens {
34
>
const defaultMetadata = LineTokens.defaultTokenMetadata;
35
>
36
>
const tokens = new Uint32Array(2);
37
>
tokens[0] = lineContent.length;
38
>
tokens[1] = defaultMetadata;
39
>
40
>
return new LineTokens(tokens, lineContent, decoder);
41
>
}
42
>
43
>
public static createFromTextAndMetadata(data: { text: string; metadata: number }[], decoder: ILanguageIdCodec): LineTokens {
45
>
let fullText: string = '';
46
>
const tokens = new Array<number>();
47
>
for (const { text, metadata } of data) {
48
>
tokens.push(offset + text.length, metadata);
49
>
offset += text.length;
50
>
fullText += text;
51
>
}
52
>
return new LineTokens(new Uint32Array(tokens), fullText, decoder);
53
>
}
55
>
public static convertToEndOffset(tokens: Uint32Array, lineTextLength: number): void {
57
>
const lastTokenIndex = tokenCount - 1;
58
>
for (let tokenIndex = 0; tokenIndex < lastTokenIndex; tokenIndex++) {
60
>
}
62
>
}
64
>
public static findIndexInTokensArray(tokens: Uint32Array, desiredIndex: number): number {
67
>
}
69
>
let low = 0;
70
>
let high = (tokens.length >>> 1) - 1;
71
>
72
>
while (low < high) {
73
>
74
>
const mid = low + Math.floor((high - low) / 2);
75
>
const endOffset = tokens[(mid << 1)];
76
>
77
>
if (endOffset === desiredIndex) {
83
>
}
86
>
return low;
89
>
_lineTokensBrand: void = undefined;
90
>
91
>
private readonly _tokens: Uint32Array;
92
>
private readonly _tokensCount: number;
93
>
private readonly _text: string;
94
>
95
>
public readonly languageIdCodec: ILanguageIdCodec;
96
>
97
>
public static defaultTokenMetadata = (
98
>
(FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET)
99
>
| (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET)
100
>
| (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET)
101
>
) >>> 0;
102
>
103
>
constructor(tokens: Uint32Array, text: string, decoder: ILanguageIdCodec) {
105
>
if (tokensLength !== text.length) {
106
onUnexpectedError(new Error('Token length and text length do not match!'));
107
}
109
>
this._tokensCount = (this._tokens.length >>> 1);
110
>
this._text = text;
111
>
this.languageIdCodec = decoder;
112
>
}
114
>
public getTextLength(): number {
116
>
}
118
>
public equals(other: IViewLineTokens): boolean {
119
if (other instanceof LineTokens) {
120
return this.slicedEquals(other, 0, this._tokensCount);
121
}
122
return false;
123
}
125
>
public slicedEquals(other: LineTokens, sliceFromTokenIndex: number, sliceTokenCount: number): boolean {
126
if (this._text !== other._text) {
127
return false;
128
}
129
if (this._tokensCount !== other._tokensCount) {
130
return false;
131
}
132
const from = (sliceFromTokenIndex << 1);
133
const to = from + (sliceTokenCount << 1);
134
for (let i = from; i < to; i++) {
135
if (this._tokens[i] !== other._tokens[i]) {
136
return false;
137
}
138
}
139
return true;
140
}
142
>
public getLineContent(): string {
144
>
}
146
>
public getCount(): number {
148
>
}
150
>
public getStartOffset(tokenIndex: number): number {
153
>
}
157
>
public getMetadata(tokenIndex: number): number {
159
>
return metadata;
160
>
}
162
>
public getLanguageId(tokenIndex: number): string {
164
>
const languageId = TokenMetadata.getLanguageId(metadata);
165
>
return this.languageIdCodec.decodeLanguageId(languageId);
166
>
}
168
>
public getStandardTokenType(tokenIndex: number): StandardTokenType {
170
>
return TokenMetadata.getTokenType(metadata);
171
>
}
173
>
public getForeground(tokenIndex: number): ColorId {
175
>
return TokenMetadata.getForeground(metadata);
176
>
}
178
>
public getClassName(tokenIndex: number): string {
180
>
return TokenMetadata.getClassNameFromMetadata(metadata);
181
>
}
183
>
public getInlineStyle(tokenIndex: number, colorMap: string[]): string {
184
const metadata = this._tokens[(tokenIndex << 1) + 1];
185
return TokenMetadata.getInlineStyleFromMetadata(metadata, colorMap);
186
}
188
>
public getPresentation(tokenIndex: number): ITokenPresentation {
189
const metadata = this._tokens[(tokenIndex << 1) + 1];
190
return TokenMetadata.getPresentationFromMetadata(metadata);
191
}
193
>
public getEndOffset(tokenIndex: number): number {
195
>
}
197
>
/**
198
>
* Find the token containing offset `offset`.
199
>
* @param offset The search offset
200
>
* @return The index of the token containing the offset.
201
>
*/
202
>
public findTokenIndexAtOffset(offset: number): number {
204
>
}
206
>
public inflate(): IViewLineTokens {
208
>
}
210
>
public sliceAndInflate(startOffset: number, endOffset: number, deltaOffset: number): IViewLineTokens {
212
>
}
214
>
public sliceZeroCopy(range: OffsetRange): IViewLineTokens {
215
return this.sliceAndInflate(range.start, range.endExclusive, 0);
216
}
218
>
/**
219
>
* @pure
220
>
* @param insertTokens Must be sorted by offset.
221
>
*/
222
>
public withInserted(insertTokens: { offset: number; text: string; tokenMetadata: number }[]): LineTokens {
224
return this;
225
}
227
>
let nextOriginalTokenIdx = 0;
228
>
let nextInsertTokenIdx = 0;
229
>
let text = '';
230
>
const newTokens = new Array<number>();
231
>
232
>
let originalEndOffset = 0;
233
>
while (true) {
234
>
const nextOriginalTokenEndOffset = nextOriginalTokenIdx < this._tokensCount ? this._tokens[nextOriginalTokenIdx << 1] : -1;
235
>
const nextInsertToken = nextInsertTokenIdx < insertTokens.length ? insertTokens[nextInsertTokenIdx] : null;
236
>
237
>
if (nextOriginalTokenEndOffset !== -1 && (nextInsertToken === null || nextOriginalTokenEndOffset <= nextInsertToken.offset)) {
238
>
// original token ends before next insert token
239
>
text += this._text.substring(originalEndOffset, nextOriginalTokenEndOffset);
240
>
const metadata = this._tokens[(nextOriginalTokenIdx << 1) + 1];
241
>
newTokens.push(text.length, metadata);
242
>
nextOriginalTokenIdx++;
243
>
originalEndOffset = nextOriginalTokenEndOffset;
244
>
245
>
} else if (nextInsertToken) {
246
>
if (nextInsertToken.offset > originalEndOffset) {
248
>
text += this._text.substring(originalEndOffset, nextInsertToken.offset);
249
>
const metadata = this._tokens[(nextOriginalTokenIdx << 1) + 1];
250
>
newTokens.push(text.length, metadata);
251
>
originalEndOffset = nextInsertToken.offset;
252
>
}
254
>
text += nextInsertToken.text;
255
>
newTokens.push(text.length, nextInsertToken.tokenMetadata);
256
>
nextInsertTokenIdx++;
257
>
} else {
258
>
break;
259
>
}
260
>
}
261
>
262
>
return new LineTokens(new Uint32Array(newTokens), text, this.languageIdCodec);
263
>
}
265
>
public getTokensInRange(range: OffsetRange): TokenArray {
266
const builder = new TokenArrayBuilder();
267
268
const startTokenIndex = this.findTokenIndexAtOffset(range.start);
269
const endTokenIndex = this.findTokenIndexAtOffset(range.endExclusive);
270
271
for (let tokenIndex = startTokenIndex; tokenIndex <= endTokenIndex; tokenIndex++) {
272
const tokenRange = new OffsetRange(this.getStartOffset(tokenIndex), this.getEndOffset(tokenIndex));
273
const length = tokenRange.intersectionLength(range);
274
if (length > 0) {
275
builder.add(length, this.getMetadata(tokenIndex));
276
}
277
}
278
279
return builder.build();
280
}
282
>
public getTokenText(tokenIndex: number): string {
283
>
const startOffset = this.getStartOffset(tokenIndex);
languageConfigurationExtensionPoint.ts ×38
284
>
const endOffset = this.getEndOffset(tokenIndex);
285
>
const text = this._text.substring(startOffset, endOffset);
286
>
return text;
287
>
}
289
>
public forEach(callback: (tokenIndex: number) => void): void {
291
>
for (let tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) {
292
>
callback(tokenIndex);
293
>
}
294
>
}
296
>
toString(): string {
297
let result = '';
298
this.forEach((i) => {
299
result += `[${this.getTokenText(i)}]{${this.getClassName(i)}}`;
300
});
301
return result;
302
}
304
>
305
>
class SliceLineTokens implements IViewLineTokens {
306
>
307
>
private readonly _source: LineTokens;
308
>
private readonly _startOffset: number;
309
>
private readonly _endOffset: number;
310
>
private readonly _deltaOffset: number;
311
>
312
>
private readonly _firstTokenIndex: number;
313
>
private readonly _tokensCount: number;
314
>
315
>
public readonly languageIdCodec: ILanguageIdCodec;
316
>
317
>
constructor(source: LineTokens, startOffset: number, endOffset: number, deltaOffset: number) {
319
>
this._startOffset = startOffset;
320
>
this._endOffset = endOffset;
321
>
this._deltaOffset = deltaOffset;
322
>
this._firstTokenIndex = source.findTokenIndexAtOffset(startOffset);
323
>
this.languageIdCodec = source.languageIdCodec;
324
>
325
>
this._tokensCount = 0;
326
>
for (let i = this._firstTokenIndex, len = source.getCount(); i < len; i++) {
327
>
const tokenStartOffset = source.getStartOffset(i);
328
>
if (tokenStartOffset >= endOffset) {
329
>
break;
330
>
}
331
>
this._tokensCount++;
332
>
}
333
>
}
335
>
public getMetadata(tokenIndex: number): number {
336
return this._source.getMetadata(this._firstTokenIndex + tokenIndex);
337
}
339
>
public getLanguageId(tokenIndex: number): string {
340
return this._source.getLanguageId(this._firstTokenIndex + tokenIndex);
341
}
343
>
public getLineContent(): string {
344
return this._source.getLineContent().substring(this._startOffset, this._endOffset);
345
}
347
>
public equals(other: IViewLineTokens): boolean {
348
if (other instanceof SliceLineTokens) {
349
return (
350
this._startOffset === other._startOffset
351
&& this._endOffset === other._endOffset
352
&& this._deltaOffset === other._deltaOffset
353
&& this._source.slicedEquals(other._source, this._firstTokenIndex, this._tokensCount)
354
);
355
}
356
return false;
357
}
359
>
public getCount(): number {
361
>
}
363
>
public getStandardTokenType(tokenIndex: number): StandardTokenType {
364
return this._source.getStandardTokenType(this._firstTokenIndex + tokenIndex);
365
}
367
>
public getForeground(tokenIndex: number): ColorId {
369
>
}
371
>
public getEndOffset(tokenIndex: number): number {
372
>
const tokenEndOffset = this._source.getEndOffset(this._firstTokenIndex + tokenIndex);
lineTokens.ts ×5
373
>
return Math.min(this._endOffset, tokenEndOffset) - this._startOffset + this._deltaOffset;
374
>
}
376
>
public getClassName(tokenIndex: number): string {
377
return this._source.getClassName(this._firstTokenIndex + tokenIndex);
378
}
380
>
public getInlineStyle(tokenIndex: number, colorMap: string[]): string {
381
return this._source.getInlineStyle(this._firstTokenIndex + tokenIndex, colorMap);
382
}
384
>
public getPresentation(tokenIndex: number): ITokenPresentation {
385
return this._source.getPresentation(this._firstTokenIndex + tokenIndex);
386
}
388
>
public findTokenIndexAtOffset(offset: number): number {
389
return this._source.findTokenIndexAtOffset(offset + this._startOffset - this._deltaOffset) - this._firstTokenIndex;
390
}
392
>
public getTokenText(tokenIndex: number): string {
393
const adjustedTokenIndex = this._firstTokenIndex + tokenIndex;
394
const tokenStartOffset = this._source.getStartOffset(adjustedTokenIndex);
395
const tokenEndOffset = this._source.getEndOffset(adjustedTokenIndex);
396
let text = this._source.getTokenText(adjustedTokenIndex);
397
if (tokenStartOffset < this._startOffset) {
398
text = text.substring(this._startOffset - tokenStartOffset);
399
}
400
if (tokenEndOffset > this._endOffset) {
401
text = text.substring(0, text.length - (tokenEndOffset - this._endOffset));
402
}
403
return text;
404
}
406
>
public forEach(callback: (tokenIndex: number) => void): void {
407
for (let tokenIndex = 0; tokenIndex < this.getCount(); tokenIndex++) {
408
callback(tokenIndex);
409
}
410
}
412
>
413
>
export function getStandardTokenTypeAtPosition(model: ITextModel, position: IPosition): StandardTokenType | undefined {
414
const lineNumber = position.lineNumber;
415
if (!model.tokenization.isCheapToTokenize(lineNumber)) {
416
return undefined;
417
}
418
model.tokenization.forceTokenization(lineNumber);
419
const lineTokens = model.tokenization.getLineTokens(lineNumber);
420
const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
421
const tokenType = lineTokens.getStandardTokenType(tokenIndex);
422
return tokenType;
423
}
425
>
426
>
427
>
/**
428
>
* This class represents a sequence of tokens.
429
>
* Conceptually, each token has a length and a metadata number.
430
>
* A token array might be used to annotate a string with metadata.
431
>
* Use {@link TokenArrayBuilder} to efficiently create a token array.
432
>
*
433
>
* TODO: Make this class more efficient (e.g. by using a Int32Array).
434
>
*/
435
>
export class TokenArray {
436
>
public static fromLineTokens(lineTokens: LineTokens): TokenArray {
437
>
const tokenInfo: TokenInfo[] = [];
438
>
for (let i = 0; i < lineTokens.getCount(); i++) {
439
>
tokenInfo.push(new TokenInfo(lineTokens.getEndOffset(i) - lineTokens.getStartOffset(i), lineTokens.getMetadata(i)));
440
>
}
441
>
return TokenArray.create(tokenInfo);
442
>
}
443
>
444
>
public static create(tokenInfo: TokenInfo[]): TokenArray {
445
return new TokenArray(tokenInfo);
446
}
448
>
private constructor(
449
private readonly _tokenInfo: TokenInfo[]
450
) { }
452
>
public toLineTokens(lineContent: string, decoder: ILanguageIdCodec): LineTokens {
453
return LineTokens.createFromTextAndMetadata(this.map((r, t) => ({ text: r.substring(lineContent), metadata: t.metadata })), decoder);
454
}
456
>
public forEach(cb: (range: OffsetRange, tokenInfo: TokenInfo) => void): void {
457
let lengthSum = 0;
458
for (const tokenInfo of this._tokenInfo) {
459
const range = new OffsetRange(lengthSum, lengthSum + tokenInfo.length);
460
cb(range, tokenInfo);
461
lengthSum += tokenInfo.length;
462
}
463
}
465
>
public map<T>(cb: (range: OffsetRange, tokenInfo: TokenInfo) => T): T[] {
466
const result: T[] = [];
467
let lengthSum = 0;
468
for (const tokenInfo of this._tokenInfo) {
469
const range = new OffsetRange(lengthSum, lengthSum + tokenInfo.length);
470
result.push(cb(range, tokenInfo));
471
lengthSum += tokenInfo.length;
472
}
473
return result;
474
}
476
>
public slice(range: OffsetRange): TokenArray {
477
const result: TokenInfo[] = [];
478
let lengthSum = 0;
479
for (const tokenInfo of this._tokenInfo) {
480
const tokenStart = lengthSum;
481
const tokenEndEx = tokenStart + tokenInfo.length;
482
if (tokenEndEx > range.start) {
483
if (tokenStart >= range.endExclusive) {
484
break;
485
}
486
487
const deltaBefore = Math.max(0, range.start - tokenStart);
488
const deltaAfter = Math.max(0, tokenEndEx - range.endExclusive);
489
490
result.push(new TokenInfo(tokenInfo.length - deltaBefore - deltaAfter, tokenInfo.metadata));
491
}
492
493
lengthSum += tokenInfo.length;
494
}
495
return TokenArray.create(result);
496
}
498
>
public append(other: TokenArray): TokenArray {
499
const result: TokenInfo[] = this._tokenInfo.concat(other._tokenInfo);
500
return TokenArray.create(result);
501
}
503
>
504
>
export type ITokenMetadata = number;
505
>
506
>
export class TokenInfo {
507
>
constructor(
508
public readonly length: number,
509
public readonly metadata: ITokenMetadata
510
) { }
512
>
/**
513
>
* TODO: Make this class more efficient (e.g. by using a Int32Array).
514
>
*/
515
>
516
>
export class TokenArrayBuilder {
517
private readonly _tokens: TokenInfo[] = [];
519
>
public add(length: number, metadata: ITokenMetadata): void {
520
this._tokens.push(new TokenInfo(length, metadata));
521
}
523
>
public build(): TokenArray {
524
return TokenArray.create(this._tokens);
525
}
527