src/vs/editor/common/services/semanticTokensProviderStyling.ts
328 LOC · 279 covered · 49 uncovered · 27 ranges · 4 concepts · 4 introducers · 3 tests
File neighbourhood
The centred file is linked to every concept that introduces one of its ranges, every test that runs code from the file, and the gray connector concepts standing between those tests and the file's own introducer concepts. Undirected links join concepts to every file where they introduce source and concepts to the tests they introduce; arrows show specialization between the displayed concepts and bridge only concepts omitted from this view. Concept colors match the source ranges below; connector concepts have no source color and are shown in gray.
Focused file, its introducer and connector concepts, their introduced files, and tests that run code from the file
In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the related-file, concept, and source links on this page.
Graph controls are ready.
Interactive rendering requires JavaScript and WebGL. Use the related-file, concept, and source links on this page while the interactive map is unavailable.
1
>
/*---------------------------------------------------------------------------------------------
semanticTokensProviderStyling.ts ×18
2
>
* Copyright (c) Microsoft Corporation. All rights reserved.
3
>
* Licensed under the MIT License. See License.txt in the project root for license information.
4
>
*--------------------------------------------------------------------------------------------*/
5
>
6
>
import { SemanticTokensLegend, SemanticTokens } from '../languages.js';
7
>
import { FontStyle, MetadataConsts, TokenMetadata } from '../encodedTokenAttributes.js';
8
>
import { IThemeService } from '../../../platform/theme/common/themeService.js';
9
>
import { ILogService, LogLevel } from '../../../platform/log/common/log.js';
10
>
import { SparseMultilineTokens } from '../tokens/sparseMultilineTokens.js';
11
>
import { ILanguageService } from '../languages/language.js';
12
>
13
>
const enum SemanticTokensProviderStylingConstants {
14
>
NO_STYLING = 0b01111111111111111111111111111111
15
>
}
16
>
17
>
const ENABLE_TRACE = false;
18
>
19
>
export class SemanticTokensProviderStyling {
20
>
21
>
private readonly _hashTable: HashTable;
22
>
private _hasWarnedOverlappingTokens = false;
23
>
private _hasWarnedInvalidLengthTokens = false;
24
>
private _hasWarnedInvalidEditStart = false;
25
>
26
>
constructor(
27
>
private readonly _legend: SemanticTokensLegend,
28
>
@IThemeService private readonly _themeService: IThemeService,
29
>
@ILanguageService private readonly _languageService: ILanguageService,
30
>
@ILogService private readonly _logService: ILogService
31
>
) {
32
>
this._hashTable = new HashTable();
33
>
}
34
>
35
>
public getMetadata(tokenTypeIndex: number, tokenModifierSet: number, languageId: string): number {
36
>
const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
37
>
const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet, encodedLanguageId);
38
>
let metadata: number;
39
>
if (entry) {
41
>
if (ENABLE_TRACE && this._logService.getLevel() === LogLevel.Trace) {
42
this._logService.trace(`SemanticTokensProviderStyling [CACHED] ${tokenTypeIndex} / ${tokenModifierSet}: foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`);
43
}
45
>
let tokenType = this._legend.tokenTypes[tokenTypeIndex];
46
>
const tokenModifiers: string[] = [];
47
>
if (tokenType) {
48
>
let modifierSet = tokenModifierSet;
49
>
for (let modifierIndex = 0; modifierSet > 0 && modifierIndex < this._legend.tokenModifiers.length; modifierIndex++) {
50
if (modifierSet & 1) {
51
tokenModifiers.push(this._legend.tokenModifiers[modifierIndex]);
52
}
53
modifierSet = modifierSet >> 1;
54
}
55
>
if (ENABLE_TRACE && modifierSet > 0 && this._logService.getLevel() === LogLevel.Trace) {
semanticTokensProviderStyling.ts ×18
56
this._logService.trace(`SemanticTokensProviderStyling: unknown token modifier index: ${tokenModifierSet.toString(2)} for legend: ${JSON.stringify(this._legend.tokenModifiers)}`);
57
tokenModifiers.push('not-in-legend');
58
}
60
>
const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers, languageId);
61
>
if (typeof tokenStyle === 'undefined') {
62
metadata = SemanticTokensProviderStylingConstants.NO_STYLING;
64
>
metadata = 0;
65
>
if (typeof tokenStyle.italic !== 'undefined') {
66
const italicBit = (tokenStyle.italic ? FontStyle.Italic : 0) << MetadataConsts.FONT_STYLE_OFFSET;
67
metadata |= italicBit | MetadataConsts.SEMANTIC_USE_ITALIC;
68
}
70
const boldBit = (tokenStyle.bold ? FontStyle.Bold : 0) << MetadataConsts.FONT_STYLE_OFFSET;
71
metadata |= boldBit | MetadataConsts.SEMANTIC_USE_BOLD;
72
}
74
const underlineBit = (tokenStyle.underline ? FontStyle.Underline : 0) << MetadataConsts.FONT_STYLE_OFFSET;
75
metadata |= underlineBit | MetadataConsts.SEMANTIC_USE_UNDERLINE;
76
}
78
const strikethroughBit = (tokenStyle.strikethrough ? FontStyle.Strikethrough : 0) << MetadataConsts.FONT_STYLE_OFFSET;
79
metadata |= strikethroughBit | MetadataConsts.SEMANTIC_USE_STRIKETHROUGH;
80
}
82
>
const foregroundBits = (tokenStyle.foreground) << MetadataConsts.FOREGROUND_OFFSET;
83
>
metadata |= foregroundBits | MetadataConsts.SEMANTIC_USE_FOREGROUND;
84
>
}
85
>
if (metadata === 0) {
87
>
metadata = SemanticTokensProviderStylingConstants.NO_STYLING;
88
>
}
90
>
} else {
91
>
if (ENABLE_TRACE && this._logService.getLevel() === LogLevel.Trace) {
semanticTokensProviderStyling.ts ×6
92
this._logService.trace(`SemanticTokensProviderStyling: unknown token type index: ${tokenTypeIndex} for legend: ${JSON.stringify(this._legend.tokenTypes)}`);
93
}
94
>
metadata = SemanticTokensProviderStylingConstants.NO_STYLING;
semanticTokensProviderStyling.ts ×6
95
>
tokenType = 'not-in-legend';
96
>
}
97
>
this._hashTable.add(tokenTypeIndex, tokenModifierSet, encodedLanguageId, metadata);
semanticTokensProviderStyling.ts ×18
98
>
99
>
if (ENABLE_TRACE && this._logService.getLevel() === LogLevel.Trace) {
100
this._logService.trace(`SemanticTokensProviderStyling ${tokenTypeIndex} (${tokenType}) / ${tokenModifierSet} (${tokenModifiers.join(' ')}): foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`);
101
}
103
>
104
>
return metadata;
105
>
}
106
>
107
>
public warnOverlappingSemanticTokens(lineNumber: number, startColumn: number): void {
108
>
if (!this._hasWarnedOverlappingTokens) {
109
>
this._hasWarnedOverlappingTokens = true;
110
>
this._logService.warn(`Overlapping semantic tokens detected at lineNumber ${lineNumber}, column ${startColumn}`);
111
>
}
112
>
}
113
>
114
>
public warnInvalidLengthSemanticTokens(lineNumber: number, startColumn: number): void {
116
>
this._hasWarnedInvalidLengthTokens = true;
117
>
this._logService.warn(`Semantic token with invalid length detected at lineNumber ${lineNumber}, column ${startColumn}`);
118
>
}
119
>
}
121
>
public warnInvalidEditStart(previousResultId: string | undefined, resultId: string | undefined, editIndex: number, editStart: number, maxExpectedStart: number): void {
122
if (!this._hasWarnedInvalidEditStart) {
123
this._hasWarnedInvalidEditStart = true;
124
this._logService.warn(`Invalid semantic tokens edit detected (previousResultId: ${previousResultId}, resultId: ${resultId}) at edit #${editIndex}: The provided start offset ${editStart} is outside the previous data (length ${maxExpectedStart}).`);
125
}
126
}
128
>
}
129
>
130
>
const enum SemanticColoringConstants {
131
>
/**
132
>
* Let's aim at having 8KB buffers if possible...
133
>
* So that would be 8192 / (5 * 4) = 409.6 tokens per area
134
>
*/
135
>
DesiredTokensPerArea = 400,
136
>
137
>
/**
138
>
* Try to keep the total number of areas under 1024 if possible,
139
>
* simply compensate by having more tokens per area...
140
>
*/
141
>
DesiredMaxAreas = 1024,
142
>
}
143
>
144
>
export function toMultilineTokens2(tokens: SemanticTokens, styling: SemanticTokensProviderStyling, languageId: string): SparseMultilineTokens[] {
145
>
const srcData = tokens.data;
146
>
const tokenCount = (tokens.data.length / 5) | 0;
147
>
const tokensPerArea = Math.max(Math.ceil(tokenCount / SemanticColoringConstants.DesiredMaxAreas), SemanticColoringConstants.DesiredTokensPerArea);
148
>
const result: SparseMultilineTokens[] = [];
149
>
150
>
let tokenIndex = 0;
151
>
let lastLineNumber = 1;
152
>
let lastStartCharacter = 0;
153
>
while (tokenIndex < tokenCount) {
154
>
const tokenStartIndex = tokenIndex;
155
>
let tokenEndIndex = Math.min(tokenStartIndex + tokensPerArea, tokenCount);
156
>
157
>
// Keep tokens on the same line in the same area...
158
>
if (tokenEndIndex < tokenCount) {
159
160
let smallTokenEndIndex = tokenEndIndex;
161
while (smallTokenEndIndex - 1 > tokenStartIndex && srcData[5 * smallTokenEndIndex] === 0) {
162
smallTokenEndIndex--;
163
}
164
165
if (smallTokenEndIndex - 1 === tokenStartIndex) {
166
// there are so many tokens on this line that our area would be empty, we must now go right
167
let bigTokenEndIndex = tokenEndIndex;
168
while (bigTokenEndIndex + 1 < tokenCount && srcData[5 * bigTokenEndIndex] === 0) {
169
bigTokenEndIndex++;
170
}
171
tokenEndIndex = bigTokenEndIndex;
172
} else {
173
tokenEndIndex = smallTokenEndIndex;
174
}
175
}
177
>
let destData = new Uint32Array((tokenEndIndex - tokenStartIndex) * 4);
178
>
let destOffset = 0;
179
>
let areaLine = 0;
180
>
let prevLineNumber = 0;
181
>
let prevEndCharacter = 0;
182
>
while (tokenIndex < tokenEndIndex) {
183
>
const srcOffset = 5 * tokenIndex;
184
>
const deltaLine = srcData[srcOffset];
185
>
const deltaCharacter = srcData[srcOffset + 1];
186
>
// Casting both `lineNumber`, `startCharacter` and `endCharacter` here to uint32 using `|0`
187
>
// to validate below with the actual values that will be inserted in the Uint32Array result
188
>
const lineNumber = (lastLineNumber + deltaLine) | 0;
189
>
const startCharacter = (deltaLine === 0 ? (lastStartCharacter + deltaCharacter) | 0 : deltaCharacter);
190
>
const length = srcData[srcOffset + 2];
191
>
const endCharacter = (startCharacter + length) | 0;
192
>
const tokenTypeIndex = srcData[srcOffset + 3];
193
>
const tokenModifierSet = srcData[srcOffset + 4];
194
>
195
>
if (endCharacter <= startCharacter) {
196
>
// this token is invalid (most likely a negative length casted to uint32)
semanticTokensProviderStyling.ts ×6
197
>
styling.warnInvalidLengthSemanticTokens(lineNumber, startCharacter + 1);
198
>
} else if (prevLineNumber === lineNumber && prevEndCharacter > startCharacter) {
semanticTokensProviderStyling.ts ×18
199
>
// this token overlaps with the previous token
200
>
styling.warnOverlappingSemanticTokens(lineNumber, startCharacter + 1);
201
>
} else {
202
>
const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet, languageId);
203
>
204
>
if (metadata !== SemanticTokensProviderStylingConstants.NO_STYLING) {
205
>
if (areaLine === 0) {
206
>
areaLine = lineNumber;
207
>
}
208
>
destData[destOffset] = lineNumber - areaLine;
209
>
destData[destOffset + 1] = startCharacter;
210
>
destData[destOffset + 2] = endCharacter;
211
>
destData[destOffset + 3] = metadata;
212
>
destOffset += 4;
213
>
214
>
prevLineNumber = lineNumber;
215
>
prevEndCharacter = endCharacter;
216
>
}
217
>
}
218
>
219
>
lastLineNumber = lineNumber;
220
>
lastStartCharacter = startCharacter;
221
>
tokenIndex++;
222
>
}
223
>
224
>
if (destOffset !== destData.length) {
225
>
destData = destData.subarray(0, destOffset);
226
>
}
227
>
228
>
const tokens = SparseMultilineTokens.create(areaLine, destData);
229
>
result.push(tokens);
230
>
}
231
>
232
>
return result;
233
>
}
234
>
235
>
class HashTableEntry {
236
>
public readonly tokenTypeIndex: number;
237
>
public readonly tokenModifierSet: number;
238
>
public readonly languageId: number;
239
>
public readonly metadata: number;
240
>
public next: HashTableEntry | null;
241
>
242
>
constructor(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number) {
243
>
this.tokenTypeIndex = tokenTypeIndex;
244
>
this.tokenModifierSet = tokenModifierSet;
245
>
this.languageId = languageId;
246
>
this.metadata = metadata;
247
>
this.next = null;
248
>
}
249
>
}
250
>
251
>
class HashTable {
252
>
253
>
private static _SIZES = [3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381, 32749, 65521, 131071, 262139, 524287, 1048573, 2097143];
254
>
255
>
private _elementsCount: number;
256
>
private _currentLengthIndex: number;
257
>
private _currentLength: number;
258
>
private _growCount: number;
259
>
private _elements: (HashTableEntry | null)[];
260
>
261
>
constructor() {
262
>
this._elementsCount = 0;
263
>
this._currentLengthIndex = 0;
264
>
this._currentLength = HashTable._SIZES[this._currentLengthIndex];
265
>
this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0);
266
>
this._elements = [];
267
>
HashTable._nullOutEntries(this._elements, this._currentLength);
268
>
}
269
>
270
>
private static _nullOutEntries(entries: (HashTableEntry | null)[], length: number): void {
271
>
for (let i = 0; i < length; i++) {
272
>
entries[i] = null;
273
>
}
274
>
}
275
>
276
>
private _hash2(n1: number, n2: number): number {
277
>
return (((n1 << 5) - n1) + n2) | 0; // n1 * 31 + n2, keep as int32
278
>
}
279
>
280
>
private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): number {
281
>
return this._hash2(this._hash2(tokenTypeIndex, tokenModifierSet), languageId) % this._currentLength;
282
>
}
283
>
284
>
public get(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): HashTableEntry | null {
285
>
const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet, languageId);
286
>
287
>
let p = this._elements[hash];
288
>
while (p) {
289
>
if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet && p.languageId === languageId) {
semanticTokensProviderStyling.ts ×2
290
>
return p;
291
>
}
293
>
}
295
>
return null;
296
>
}
297
>
298
>
public add(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number): void {
299
>
this._elementsCount++;
300
>
if (this._growCount !== 0 && this._elementsCount >= this._growCount) {
302
>
const oldElements = this._elements;
303
>
304
>
this._currentLengthIndex++;
305
>
this._currentLength = HashTable._SIZES[this._currentLengthIndex];
306
>
this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0);
307
>
this._elements = [];
308
>
HashTable._nullOutEntries(this._elements, this._currentLength);
309
>
310
>
for (const first of oldElements) {
311
>
let p = first;
312
>
while (p) {
313
>
const oldNext = p.next;
314
>
p.next = null;
315
>
this._add(p);
316
>
p = oldNext;
317
>
}
318
>
}
319
>
}
320
>
this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, languageId, metadata));
semanticTokensProviderStyling.ts ×18
321
>
}
322
>
323
>
private _add(element: HashTableEntry): void {
324
>
const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet, element.languageId);
325
>
element.next = this._elements[hash];
326
>
this._elements[hash] = element;
327
>
}
328
>
}