semanticTokensProviderStyling.ts ×18

Frontier kind: Code frontier

unlabeled · c_7b4099bbe9fd

3 tests · 39901 LOC · 241 files · introduces 0 tests · 247 LOC · 2 files

Introduces — evidence that enters the hierarchy at this concept

Code
20 ranges247 lines · 2 files
Tests
0 tests

Contains — complete concept membership

All code (extent)
4716 ranges39901 lines · 241 files · Browse complete extent
All tests (intent)
3 testsBrowse complete intent

Neighbourhood graph

The orange circle is the focus. Violet and green circles are every ancestor and descendant, broader and narrower, at any distance; blue squares and pink diamonds are the introduced files and exact introduced tests of every visible concept, not only the focus's. Arrows point from broader to narrower concepts and bridge only concepts omitted from this view. Undirected links show source or test introduction. Concept and file size follows LOC; exact test nodes use test-count units.

Introduced files, introduced tests, and structurally relevant concept specialization

In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the native relationship evidence on this page.

Graph controls are ready.

Interactive rendering requires JavaScript and WebGL. Use the native relationship evidence on this page while the interactive map is unavailable.

Native relationship evidence

Every exact file and test below is linked only from the concept that introduces it.

Introduced tests

Every collected test enters the hierarchy at exactly one concept.

No tests are introduced at this concept. Its intent tests are introduced by other concepts.

Introduced code

Every collected source range enters the hierarchy at exactly one concept.

2 files ranked by introduced lines: 247 introduced LOC across 20 ranges. Expand a file to inspect source; the > gutter marks introduced lines.

src/vs/editor/common/services/semanticTokensProviderStyling.ts 239 introduced LOC · 18 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- semanticTokensProviderStyling.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { SemanticTokensLegend, SemanticTokens } from '../languages.js';
7 > import { FontStyle, MetadataConsts, TokenMetadata } from '../encodedTokenAttributes.js';
8 > import { IThemeService } from '../../../platform/theme/common/themeService.js';
9 > import { ILogService, LogLevel } from '../../../platform/log/common/log.js';
10 > import { SparseMultilineTokens } from '../tokens/sparseMultilineTokens.js';
11 > import { ILanguageService } from '../languages/language.js';
12 >
13 > const enum SemanticTokensProviderStylingConstants {
14 > NO_STYLING = 0b01111111111111111111111111111111
15 > }
16 >
17 > const ENABLE_TRACE = false;
18 >
19 > export class SemanticTokensProviderStyling {
20 >
21 > private readonly _hashTable: HashTable;
22 > private _hasWarnedOverlappingTokens = false;
23 > private _hasWarnedInvalidLengthTokens = false;
24 > private _hasWarnedInvalidEditStart = false;
25 >
26 > constructor(
27 > private readonly _legend: SemanticTokensLegend,
28 > @IThemeService private readonly _themeService: IThemeService,
29 > @ILanguageService private readonly _languageService: ILanguageService,
30 > @ILogService private readonly _logService: ILogService
31 > ) {
32 > this._hashTable = new HashTable();
33 > }
34 >
35 > public getMetadata(tokenTypeIndex: number, tokenModifierSet: number, languageId: string): number {
36 > const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
37 > const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet, encodedLanguageId);
38 > let metadata: number;
39 > if (entry) {
40 metadata = entry.metadata;
41 if (ENABLE_TRACE && this._logService.getLevel() === LogLevel.Trace) {
42 this._logService.trace(`SemanticTokensProviderStyling [CACHED] ${tokenTypeIndex} / ${tokenModifierSet}: foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`);
43 }
45 > let tokenType = this._legend.tokenTypes[tokenTypeIndex];
46 > const tokenModifiers: string[] = [];
47 > if (tokenType) {
48 > let modifierSet = tokenModifierSet;
49 > for (let modifierIndex = 0; modifierSet > 0 && modifierIndex < this._legend.tokenModifiers.length; modifierIndex++) {
50 if (modifierSet & 1) {
51 tokenModifiers.push(this._legend.tokenModifiers[modifierIndex]);
53 modifierSet = modifierSet >> 1;
54 }
55 > if (ENABLE_TRACE && modifierSet > 0 && this._logService.getLevel() === LogLevel.Trace) { semanticTokensProviderStyling.ts
56 this._logService.trace(`SemanticTokensProviderStyling: unknown token modifier index: ${tokenModifierSet.toString(2)} for legend: ${JSON.stringify(this._legend.tokenModifiers)}`);
57 tokenModifiers.push('not-in-legend');
58 }
60 > const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers, languageId);
61 > if (typeof tokenStyle === 'undefined') {
62 metadata = SemanticTokensProviderStylingConstants.NO_STYLING;
64 > metadata = 0;
65 > if (typeof tokenStyle.italic !== 'undefined') {
66 const italicBit = (tokenStyle.italic ? FontStyle.Italic : 0) << MetadataConsts.FONT_STYLE_OFFSET;
67 metadata |= italicBit | MetadataConsts.SEMANTIC_USE_ITALIC;
68 }
69 > if (typeof tokenStyle.bold !== 'undefined') { semanticTokensProviderStyling.ts
70 const boldBit = (tokenStyle.bold ? FontStyle.Bold : 0) << MetadataConsts.FONT_STYLE_OFFSET;
71 metadata |= boldBit | MetadataConsts.SEMANTIC_USE_BOLD;
72 }
73 > if (typeof tokenStyle.underline !== 'undefined') { semanticTokensProviderStyling.ts
74 const underlineBit = (tokenStyle.underline ? FontStyle.Underline : 0) << MetadataConsts.FONT_STYLE_OFFSET;
75 metadata |= underlineBit | MetadataConsts.SEMANTIC_USE_UNDERLINE;
76 }
77 > if (typeof tokenStyle.strikethrough !== 'undefined') { semanticTokensProviderStyling.ts
78 const strikethroughBit = (tokenStyle.strikethrough ? FontStyle.Strikethrough : 0) << MetadataConsts.FONT_STYLE_OFFSET;
79 metadata |= strikethroughBit | MetadataConsts.SEMANTIC_USE_STRIKETHROUGH;
80 }
81 > if (tokenStyle.foreground) { semanticTokensProviderStyling.ts
82 > const foregroundBits = (tokenStyle.foreground) << MetadataConsts.FOREGROUND_OFFSET;
83 > metadata |= foregroundBits | MetadataConsts.SEMANTIC_USE_FOREGROUND;
84 > }
85 > if (metadata === 0) {
86 // Nothing!
87 metadata = SemanticTokensProviderStylingConstants.NO_STYLING;
88 }
90 > } else {
91 if (ENABLE_TRACE && this._logService.getLevel() === LogLevel.Trace) {
92 this._logService.trace(`SemanticTokensProviderStyling: unknown token type index: ${tokenTypeIndex} for legend: ${JSON.stringify(this._legend.tokenTypes)}`);
95 tokenType = 'not-in-legend';
96 }
97 > this._hashTable.add(tokenTypeIndex, tokenModifierSet, encodedLanguageId, metadata); semanticTokensProviderStyling.ts
98 >
99 > if (ENABLE_TRACE && this._logService.getLevel() === LogLevel.Trace) {
100 this._logService.trace(`SemanticTokensProviderStyling ${tokenTypeIndex} (${tokenType}) / ${tokenModifierSet} (${tokenModifiers.join(' ')}): foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`);
101 }
103 >
104 > return metadata;
105 > }
106 >
107 > public warnOverlappingSemanticTokens(lineNumber: number, startColumn: number): void {
108 > if (!this._hasWarnedOverlappingTokens) {
109 > this._hasWarnedOverlappingTokens = true;
110 > this._logService.warn(`Overlapping semantic tokens detected at lineNumber ${lineNumber}, column ${startColumn}`);
111 > }
112 > }
113 >
114 > public warnInvalidLengthSemanticTokens(lineNumber: number, startColumn: number): void {
115 if (!this._hasWarnedInvalidLengthTokens) {
116 this._hasWarnedInvalidLengthTokens = true;
118 }
119 }
121 > public warnInvalidEditStart(previousResultId: string | undefined, resultId: string | undefined, editIndex: number, editStart: number, maxExpectedStart: number): void {
122 if (!this._hasWarnedInvalidEditStart) {
123 this._hasWarnedInvalidEditStart = true;
125 }
126 }
128 > }
129 >
130 > const enum SemanticColoringConstants {
131 > /**
132 > * Let's aim at having 8KB buffers if possible...
133 > * So that would be 8192 / (5 * 4) = 409.6 tokens per area
134 > */
135 > DesiredTokensPerArea = 400,
136 >
137 > /**
138 > * Try to keep the total number of areas under 1024 if possible,
139 > * simply compensate by having more tokens per area...
140 > */
141 > DesiredMaxAreas = 1024,
142 > }
143 >
144 > export function toMultilineTokens2(tokens: SemanticTokens, styling: SemanticTokensProviderStyling, languageId: string): SparseMultilineTokens[] {
145 > const srcData = tokens.data;
146 > const tokenCount = (tokens.data.length / 5) | 0;
147 > const tokensPerArea = Math.max(Math.ceil(tokenCount / SemanticColoringConstants.DesiredMaxAreas), SemanticColoringConstants.DesiredTokensPerArea);
148 > const result: SparseMultilineTokens[] = [];
149 >
150 > let tokenIndex = 0;
151 > let lastLineNumber = 1;
152 > let lastStartCharacter = 0;
153 > while (tokenIndex < tokenCount) {
154 > const tokenStartIndex = tokenIndex;
155 > let tokenEndIndex = Math.min(tokenStartIndex + tokensPerArea, tokenCount);
156 >
157 > // Keep tokens on the same line in the same area...
158 > if (tokenEndIndex < tokenCount) {
159
160 let smallTokenEndIndex = tokenEndIndex;
174 }
175 }
177 > let destData = new Uint32Array((tokenEndIndex - tokenStartIndex) * 4);
178 > let destOffset = 0;
179 > let areaLine = 0;
180 > let prevLineNumber = 0;
181 > let prevEndCharacter = 0;
182 > while (tokenIndex < tokenEndIndex) {
183 > const srcOffset = 5 * tokenIndex;
184 > const deltaLine = srcData[srcOffset];
185 > const deltaCharacter = srcData[srcOffset + 1];
186 > // Casting both `lineNumber`, `startCharacter` and `endCharacter` here to uint32 using `|0`
187 > // to validate below with the actual values that will be inserted in the Uint32Array result
188 > const lineNumber = (lastLineNumber + deltaLine) | 0;
189 > const startCharacter = (deltaLine === 0 ? (lastStartCharacter + deltaCharacter) | 0 : deltaCharacter);
190 > const length = srcData[srcOffset + 2];
191 > const endCharacter = (startCharacter + length) | 0;
192 > const tokenTypeIndex = srcData[srcOffset + 3];
193 > const tokenModifierSet = srcData[srcOffset + 4];
194 >
195 > if (endCharacter <= startCharacter) {
196 // this token is invalid (most likely a negative length casted to uint32)
197 styling.warnInvalidLengthSemanticTokens(lineNumber, startCharacter + 1);
198 > } else if (prevLineNumber === lineNumber && prevEndCharacter > startCharacter) { semanticTokensProviderStyling.ts
199 > // this token overlaps with the previous token
200 > styling.warnOverlappingSemanticTokens(lineNumber, startCharacter + 1);
201 > } else {
202 > const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet, languageId);
203 >
204 > if (metadata !== SemanticTokensProviderStylingConstants.NO_STYLING) {
205 > if (areaLine === 0) {
206 > areaLine = lineNumber;
207 > }
208 > destData[destOffset] = lineNumber - areaLine;
209 > destData[destOffset + 1] = startCharacter;
210 > destData[destOffset + 2] = endCharacter;
211 > destData[destOffset + 3] = metadata;
212 > destOffset += 4;
213 >
214 > prevLineNumber = lineNumber;
215 > prevEndCharacter = endCharacter;
216 > }
217 > }
218 >
219 > lastLineNumber = lineNumber;
220 > lastStartCharacter = startCharacter;
221 > tokenIndex++;
222 > }
223 >
224 > if (destOffset !== destData.length) {
225 > destData = destData.subarray(0, destOffset);
226 > }
227 >
228 > const tokens = SparseMultilineTokens.create(areaLine, destData);
229 > result.push(tokens);
230 > }
231 >
232 > return result;
233 > }
234 >
235 > class HashTableEntry {
236 > public readonly tokenTypeIndex: number;
237 > public readonly tokenModifierSet: number;
238 > public readonly languageId: number;
239 > public readonly metadata: number;
240 > public next: HashTableEntry | null;
241 >
242 > constructor(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number) {
243 > this.tokenTypeIndex = tokenTypeIndex;
244 > this.tokenModifierSet = tokenModifierSet;
245 > this.languageId = languageId;
246 > this.metadata = metadata;
247 > this.next = null;
248 > }
249 > }
250 >
251 > class HashTable {
252 >
253 > private static _SIZES = [3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381, 32749, 65521, 131071, 262139, 524287, 1048573, 2097143];
254 >
255 > private _elementsCount: number;
256 > private _currentLengthIndex: number;
257 > private _currentLength: number;
258 > private _growCount: number;
259 > private _elements: (HashTableEntry | null)[];
260 >
261 > constructor() {
262 > this._elementsCount = 0;
263 > this._currentLengthIndex = 0;
264 > this._currentLength = HashTable._SIZES[this._currentLengthIndex];
265 > this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0);
266 > this._elements = [];
267 > HashTable._nullOutEntries(this._elements, this._currentLength);
268 > }
269 >
270 > private static _nullOutEntries(entries: (HashTableEntry | null)[], length: number): void {
271 > for (let i = 0; i < length; i++) {
272 > entries[i] = null;
273 > }
274 > }
275 >
276 > private _hash2(n1: number, n2: number): number {
277 > return (((n1 << 5) - n1) + n2) | 0; // n1 * 31 + n2, keep as int32
278 > }
279 >
280 > private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): number {
281 > return this._hash2(this._hash2(tokenTypeIndex, tokenModifierSet), languageId) % this._currentLength;
282 > }
283 >
284 > public get(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): HashTableEntry | null {
285 > const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet, languageId);
286 >
287 > let p = this._elements[hash];
288 > while (p) {
289 if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet && p.languageId === languageId) {
290 return p;
292 p = p.next;
293 }
295 > return null;
296 > }
297 >
298 > public add(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number): void {
299 > this._elementsCount++;
300 > if (this._growCount !== 0 && this._elementsCount >= this._growCount) {
301 // expand!
302 const oldElements = this._elements;
318 }
319 }
320 > this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, languageId, metadata)); semanticTokensProviderStyling.ts
321 > }
322 >
323 > private _add(element: HashTableEntry): void {
324 > const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet, element.languageId);
325 > element.next = this._elements[hash];
326 > this._elements[hash] = element;
327 > }
328 > }
src/vs/editor/common/tokens/sparseMultilineTokens.ts 8 introduced LOC · 2 ranges

Open complete file

45
46 public toString(): string {
47 > return this._tokens.toString(this._startLineNumber); sparseMultilineTokens.ts
48 > }
49
50 private _updateEndLineNumber(): void {
188
189 public toString(startLineNumber: number): string {
190 > const pieces: string[] = []; sparseMultilineTokens.ts
191 > for (let i = 0; i < this._tokenCount; i++) {
192 > pieces.push(`(${this._getDeltaLine(i) + startLineNumber},${this._getStartCharacter(i)}-${this._getEndCharacter(i)})`);
193 > }
194 > return `[${pieces.join(',')}]`;
195 > }
196
197 public getMaxDeltaLine(): number {