174
}
175
}
177
>
let destData = new Uint32Array((tokenEndIndex - tokenStartIndex) * 4);
178
>
let destOffset = 0;
179
>
let areaLine = 0;
180
>
let prevLineNumber = 0;
181
>
let prevEndCharacter = 0;
182
>
while (tokenIndex < tokenEndIndex) {
183
>
const srcOffset = 5 * tokenIndex;
184
>
const deltaLine = srcData[srcOffset];
185
>
const deltaCharacter = srcData[srcOffset + 1];
186
>
// Casting both `lineNumber`, `startCharacter` and `endCharacter` here to uint32 using `|0`
187
>
// to validate below with the actual values that will be inserted in the Uint32Array result
188
>
const lineNumber = (lastLineNumber + deltaLine) | 0;
189
>
const startCharacter = (deltaLine === 0 ? (lastStartCharacter + deltaCharacter) | 0 : deltaCharacter);
190
>
const length = srcData[srcOffset + 2];
191
>
const endCharacter = (startCharacter + length) | 0;
192
>
const tokenTypeIndex = srcData[srcOffset + 3];
193
>
const tokenModifierSet = srcData[srcOffset + 4];
194
>
195
>
if (endCharacter <= startCharacter) {
196
// this token is invalid (most likely a negative length casted to uint32)
197
styling.warnInvalidLengthSemanticTokens(lineNumber, startCharacter + 1);
199
>
// this token overlaps with the previous token
200
>
styling.warnOverlappingSemanticTokens(lineNumber, startCharacter + 1);
201
>
} else {
202
>
const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet, languageId);
203
>
204
>
if (metadata !== SemanticTokensProviderStylingConstants.NO_STYLING) {
205
>
if (areaLine === 0) {
206
>
areaLine = lineNumber;
207
>
}
208
>
destData[destOffset] = lineNumber - areaLine;
209
>
destData[destOffset + 1] = startCharacter;
210
>
destData[destOffset + 2] = endCharacter;
211
>
destData[destOffset + 3] = metadata;
212
>
destOffset += 4;
213
>
214
>
prevLineNumber = lineNumber;
215
>
prevEndCharacter = endCharacter;
216
>
}
217
>
}
218
>
219
>
lastLineNumber = lineNumber;
220
>
lastStartCharacter = startCharacter;
221
>
tokenIndex++;
222
>
}
223
>
224
>
if (destOffset !== destData.length) {
225
>
destData = destData.subarray(0, destOffset);
226
>
}
227
>
228
>
const tokens = SparseMultilineTokens.create(areaLine, destData);
229
>
result.push(tokens);
230
>
}
231
>
232
>
return result;
233
>
}
234
>
235
>
class HashTableEntry {
236
>
public readonly tokenTypeIndex: number;
237
>
public readonly tokenModifierSet: number;
238
>
public readonly languageId: number;
239
>
public readonly metadata: number;
240
>
public next: HashTableEntry | null;
241
>
242
>
constructor(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number) {
243
>
this.tokenTypeIndex = tokenTypeIndex;
244
>
this.tokenModifierSet = tokenModifierSet;
245
>
this.languageId = languageId;
246
>
this.metadata = metadata;
247
>
this.next = null;
248
>
}
249
>
}
250
>
251
>
class HashTable {
252
>
253
>
private static _SIZES = [3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381, 32749, 65521, 131071, 262139, 524287, 1048573, 2097143];
254
>
255
>
private _elementsCount: number;
256
>
private _currentLengthIndex: number;
257
>
private _currentLength: number;
258
>
private _growCount: number;
259
>
private _elements: (HashTableEntry | null)[];
260
>
261
>
constructor() {
262
>
this._elementsCount = 0;
263
>
this._currentLengthIndex = 0;
264
>
this._currentLength = HashTable._SIZES[this._currentLengthIndex];
265
>
this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0);
266
>
this._elements = [];
267
>
HashTable._nullOutEntries(this._elements, this._currentLength);
268
>
}
269
>
270
>
private static _nullOutEntries(entries: (HashTableEntry | null)[], length: number): void {
271
>
for (let i = 0; i < length; i++) {
272
>
entries[i] = null;
273
>
}
274
>
}
275
>
276
>
private _hash2(n1: number, n2: number): number {
277
>
return (((n1 << 5) - n1) + n2) | 0; // n1 * 31 + n2, keep as int32
278
>
}
279
>
280
>
private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): number {
281
>
return this._hash2(this._hash2(tokenTypeIndex, tokenModifierSet), languageId) % this._currentLength;
282
>
}
283
>
284
>
public get(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): HashTableEntry | null {
285
>
const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet, languageId);
286
>
287
>
let p = this._elements[hash];
288
>
while (p) {
289
if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet && p.languageId === languageId) {
290
return p;