src/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.ts
323 LOC · 222 covered · 101 uncovered · 70 ranges · 1708 concepts · 22 introducers · 861 tests
File neighbourhood
The centred file is linked to every concept that introduces one of its ranges, every test that runs code from the file, and the gray connector concepts standing between those tests and the file's own introducer concepts. Undirected links join concepts to every file where they introduce source and concepts to the tests they introduce; arrows show specialization between the displayed concepts and bridge only concepts omitted from this view. Concept colors match the source ranges below; connector concepts have no source color and are shown in gray.
Focused file, its introducer and connector concepts, their introduced files, and tests that run code from the file
In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the related-file, concept, and source links on this page.
Graph controls are ready.
Interactive rendering requires JavaScript and WebGL. Use the related-file, concept, and source links on this page while the interactive map is unavailable.
1
>
/*---------------------------------------------------------------------------------------------
textModel.ts ×190
2
>
* Copyright (c) Microsoft Corporation. All rights reserved.
3
>
* Licensed under the MIT License. See License.txt in the project root for license information.
4
>
*--------------------------------------------------------------------------------------------*/
5
>
6
>
import { onUnexpectedError } from '../../../../base/common/errors.js';
7
>
import { Emitter, Event } from '../../../../base/common/event.js';
8
>
import { MutableDisposable, DisposableMap } from '../../../../base/common/lifecycle.js';
9
>
import { countEOL } from '../../core/misc/eolCounter.js';
10
>
import { Position } from '../../core/position.js';
11
>
import { LineRange } from '../../core/ranges/lineRange.js';
12
>
import { StandardTokenType } from '../../encodedTokenAttributes.js';
13
>
import { IBackgroundTokenizer, IState, ILanguageIdCodec, TokenizationRegistry, ITokenizationSupport, IBackgroundTokenizationStore } from '../../languages.js';
14
>
import { IAttachedView } from '../../model.js';
15
>
import { FontTokensUpdate, IModelContentChangedEvent } from '../../textModelEvents.js';
16
>
import { BackgroundTokenizationState } from '../../tokenizationTextModelPart.js';
17
>
import { ContiguousMultilineTokens } from '../../tokens/contiguousMultilineTokens.js';
18
>
import { ContiguousMultilineTokensBuilder } from '../../tokens/contiguousMultilineTokensBuilder.js';
19
>
import { ContiguousTokensStore } from '../../tokens/contiguousTokensStore.js';
20
>
import { LineTokens } from '../../tokens/lineTokens.js';
21
>
import { TextModel } from '../textModel.js';
22
>
import { TokenizerWithStateStoreAndTextModel, DefaultBackgroundTokenizer, TrackingTokenizationStateStore } from '../textModelTokens.js';
23
>
import { AbstractSyntaxTokenBackend, AttachedViewHandler, AttachedViews } from './abstractSyntaxTokenBackend.js';
24
>
25
>
/** For TextMate */
26
>
export class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {
27
>
private _tokenizer: TokenizerWithStateStoreAndTextModel | null = null;
28
>
protected _backgroundTokenizationState: BackgroundTokenizationState = BackgroundTokenizationState.InProgress;
29
>
protected readonly _onDidChangeBackgroundTokenizationState: Emitter<void> = this._register(new Emitter<void>());
30
>
public readonly onDidChangeBackgroundTokenizationState: Event<void> = this._onDidChangeBackgroundTokenizationState.event;
31
>
32
>
private _defaultBackgroundTokenizer: DefaultBackgroundTokenizer | null = null;
33
>
private readonly _backgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());
34
>
35
>
private readonly _tokens = new ContiguousTokensStore(this._languageIdCodec);
36
>
private _debugBackgroundTokens: ContiguousTokensStore | undefined;
37
>
private _debugBackgroundStates: TrackingTokenizationStateStore<IState> | undefined;
38
>
39
>
private readonly _debugBackgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());
40
>
41
>
private readonly _attachedViewStates = this._register(new DisposableMap<IAttachedView, AttachedViewHandler>());
42
>
43
>
constructor(
45
>
textModel: TextModel,
46
>
private readonly getLanguageId: () => string,
47
>
attachedViews: AttachedViews,
48
>
) {
49
>
super(languageIdCodec, textModel);
50
>
51
>
this._register(TokenizationRegistry.onDidChange((e) => {
53
>
if (e.changedLanguages.indexOf(languageId) === -1) {
54
return;
55
}
58
>
59
>
this.todo_resetTokenization();
60
>
61
>
this._register(attachedViews.onDidChangeVisibleRanges(({ view, state }) => {
62
if (state) {
63
let existing = this._attachedViewStates.get(view);
64
if (!existing) {
65
existing = new AttachedViewHandler(() => this.refreshRanges(existing!.lineRanges));
66
this._attachedViewStates.set(view, existing);
67
}
68
existing.handleStateChange(state);
69
} else {
70
this._attachedViewStates.deleteAndDispose(view);
71
}
73
>
}
75
>
public todo_resetTokenization(fireTokenChangeEvent: boolean = true): void {
77
>
this._debugBackgroundTokens?.flush();
78
>
if (this._debugBackgroundStates) {
79
this._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());
80
}
82
>
this._onDidChangeTokens.fire({
83
>
semanticTokensApplied: false,
84
>
ranges: [
85
>
{
86
>
fromLineNumber: 1,
87
>
toLineNumber: this._textModel.getLineCount(),
88
>
},
89
>
],
90
>
});
91
>
}
92
>
93
>
const initializeTokenization = (): [ITokenizationSupport, IState] | [null, null] => {
94
>
if (this._textModel.isTooLargeForTokenization()) {
95
return [null, null];
96
}
98
>
if (!tokenizationSupport) {
100
>
}
102
>
try {
103
>
initialState = tokenizationSupport.getInitialState();
104
>
} catch (e) {
105
onUnexpectedError(e);
106
return [null, null];
107
}
110
>
111
>
const [tokenizationSupport, initialState] = initializeTokenization();
112
>
if (tokenizationSupport && initialState) {
113
>
this._tokenizer = new TokenizerWithStateStoreAndTextModel(this._textModel.getLineCount(), tokenizationSupport, this._textModel, this._languageIdCodec);
tokenizerSyntaxTokenBackend.ts ×11
116
>
}
118
>
this._backgroundTokenizer.clear();
119
>
120
>
this._defaultBackgroundTokenizer = null;
121
>
if (this._tokenizer) {
123
>
setTokens: (tokens) => {
125
>
},
127
this.setFontInfo(changes);
128
},
130
>
if (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) {
textModelTokens.ts ×4
131
>
// We already did a full tokenization and don't go back to progressing.
tokenizerSyntaxTokenBackend.ts ×1
132
>
return;
133
>
}
135
>
this._backgroundTokenizationState = newState;
136
>
this._onDidChangeBackgroundTokenizationState.fire();
137
>
},
139
if (!this._tokenizer) { return; }
140
const firstInvalidEndStateLineNumber = this._tokenizer.store.getFirstInvalidEndStateLineNumber();
141
// Don't accept states for definitely valid states, the renderer is ahead of the worker!
142
if (firstInvalidEndStateLineNumber !== null && lineNumber >= firstInvalidEndStateLineNumber) {
143
this._tokenizer?.store.setEndState(lineNumber, state);
144
}
145
},
147
>
148
>
if (tokenizationSupport && tokenizationSupport.createBackgroundTokenizer && !tokenizationSupport.backgroundTokenizerShouldOnlyVerifyTokens) {
149
>
this._backgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, b);
tokenizerSyntaxTokenBackend.ts ×2
150
>
}
151
>
if (!this._backgroundTokenizer.value && !this._textModel.isTooLargeForTokenization()) {
tokenizerSyntaxTokenBackend.ts ×11
153
>
new DefaultBackgroundTokenizer(this._tokenizer, b);
154
>
this._defaultBackgroundTokenizer.handleChanges();
155
>
}
157
>
if (tokenizationSupport?.backgroundTokenizerShouldOnlyVerifyTokens && tokenizationSupport.createBackgroundTokenizer) {
158
this._debugBackgroundTokens = new ContiguousTokensStore(this._languageIdCodec);
159
this._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());
160
this._debugBackgroundTokenizer.clear();
161
this._debugBackgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, {
162
setTokens: (tokens) => {
163
this._debugBackgroundTokens?.setMultilineTokens(tokens, this._textModel);
164
},
165
setFontInfo: (changes: FontTokensUpdate) => {
166
this.setFontInfo(changes);
167
},
168
backgroundTokenizationFinished() {
169
// NO OP
170
},
171
setEndState: (lineNumber, state) => {
172
this._debugBackgroundStates?.setEndState(lineNumber, state);
173
},
174
});
176
>
this._debugBackgroundTokens = undefined;
177
>
this._debugBackgroundStates = undefined;
178
>
this._debugBackgroundTokenizer.value = undefined;
179
>
}
180
>
}
182
>
this.refreshAllVisibleLineTokens();
183
>
}
185
>
public handleDidChangeAttached() {
187
>
}
189
>
public handleDidChangeContent(e: IModelContentChangedEvent): void {
191
>
// Don't fire the event, as the view might not have got the text change event yet
textModel.ts ×3
192
>
this.todo_resetTokenization(false);
195
>
const [eolCount, firstLineLength] = countEOL(c.text);
196
>
197
>
this._tokens.acceptEdit(c.range, eolCount, firstLineLength);
198
>
this._debugBackgroundTokens?.acceptEdit(c.range, eolCount, firstLineLength);
199
>
}
200
>
this._debugBackgroundStates?.acceptChanges(e.changes);
201
>
202
>
if (this._tokenizer) {
204
>
}
206
>
}
209
>
private setTokens(tokens: ContiguousMultilineTokens[]): { changes: { fromLineNumber: number; toLineNumber: number }[] } {
210
>
const { changes } = this._tokens.setMultilineTokens(tokens, this._textModel);
tokenizerSyntaxTokenBackend.ts ×2
211
>
212
>
if (changes.length > 0) {
213
>
this._onDidChangeTokens.fire({ semanticTokensApplied: false, ranges: changes, });
contiguousTokensStore.ts ×11
214
>
}
216
>
return { changes: changes };
217
>
}
219
>
private setFontInfo(changes: FontTokensUpdate): void {
220
this._onDidChangeFontTokens.fire({ changes });
221
}
223
>
private refreshAllVisibleLineTokens(): void {
224
>
const ranges = LineRange.joinMany([...this._attachedViewStates].map(([_, s]) => s.lineRanges));
textModel.ts ×20
225
>
this.refreshRanges(ranges);
226
>
}
228
>
private refreshRanges(ranges: readonly LineRange[]): void {
230
this.refreshRange(range.startLineNumber, range.endLineNumberExclusive - 1);
231
}
234
>
private refreshRange(startLineNumber: number, endLineNumber: number): void {
235
if (!this._tokenizer) {
236
return;
237
}
238
239
startLineNumber = Math.max(1, Math.min(this._textModel.getLineCount(), startLineNumber));
240
endLineNumber = Math.min(this._textModel.getLineCount(), endLineNumber);
241
242
const builder = new ContiguousMultilineTokensBuilder();
243
const { heuristicTokens } = this._tokenizer.tokenizeHeuristically(builder, startLineNumber, endLineNumber);
244
const changedTokens = this.setTokens(builder.finalize());
245
246
if (heuristicTokens) {
247
// We overrode tokens with heuristically computed ones.
248
// Because old states might get reused (thus stopping invalidation),
249
// we have to explicitly request the tokens for the changed ranges again.
250
for (const c of changedTokens.changes) {
251
this._backgroundTokenizer.value?.requestTokens(c.fromLineNumber, c.toLineNumber + 1);
252
}
253
}
254
255
this._defaultBackgroundTokenizer?.checkFinished();
256
}
258
>
public forceTokenization(lineNumber: number): void {
260
>
this._tokenizer?.updateTokensUntilLine(builder, lineNumber);
261
>
this.setTokens(builder.finalize());
262
>
this._defaultBackgroundTokenizer?.checkFinished();
263
>
}
265
>
public hasAccurateTokensForLine(lineNumber: number): boolean {
266
if (!this._tokenizer) {
267
return true;
268
}
269
return this._tokenizer.hasAccurateTokensForLine(lineNumber);
270
}
272
>
public isCheapToTokenize(lineNumber: number): boolean {
275
>
}
279
>
public getLineTokens(lineNumber: number): LineTokens {
281
>
const result = this._tokens.getTokens(
282
>
this._textModel.getLanguageId(),
283
>
lineNumber - 1,
284
>
lineText
285
>
);
286
>
if (this._debugBackgroundTokens && this._debugBackgroundStates && this._tokenizer) {
287
if (this._debugBackgroundStates.getFirstInvalidEndStateLineNumberOrMax() > lineNumber && this._tokenizer.store.getFirstInvalidEndStateLineNumberOrMax() > lineNumber) {
288
const backgroundResult = this._debugBackgroundTokens.getTokens(
289
this._textModel.getLanguageId(),
290
lineNumber - 1,
291
lineText
292
);
293
if (!result.equals(backgroundResult) && this._debugBackgroundTokenizer.value?.reportMismatchingTokens) {
294
this._debugBackgroundTokenizer.value.reportMismatchingTokens(lineNumber);
295
}
296
}
297
}
299
>
}
301
>
public getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {
302
if (!this._tokenizer) {
303
return StandardTokenType.Other;
304
}
305
306
const position = this._textModel.validatePosition(new Position(lineNumber, column));
307
this.forceTokenization(position.lineNumber);
308
return this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);
309
}
311
>
312
>
public tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {
313
if (!this._tokenizer) {
314
return null;
315
}
316
this.forceTokenization(lineNumber);
317
return this._tokenizer.tokenizeLinesAt(lineNumber, lines);
318
}
320
>
public get hasTokens(): boolean {
322
>
}