Vihren Atlas

tokenizerSyntaxTokenBackend.ts ×11

Code frontier

unlabeled · c_4d7e385517d3

82 tests · 40564 LOC · 238 files · introduces 0 tests · 44 LOC · 5 files

Introduces — evidence that enters the hierarchy at this concept

Code
18 ranges44 lines · 5 files
Tests
0 tests

Contains — complete concept membership

All code (extent)
4921 ranges40564 lines · 238 files · Browse complete extent
All tests (intent)
82 testsBrowse complete intent

Neighbourhood graph

The orange circle is the focus. Violet and green circles are every ancestor and descendant, broader and narrower, at any distance; blue squares and pink diamonds are the introduced files and exact introduced tests of every visible concept, not only the focus's. Arrows point from broader to narrower concepts and bridge only concepts omitted from this view. Undirected links show source or test introduction. Concept and file size follows LOC; exact test nodes use test-count units.

Introduced files, introduced tests, and structurally relevant concept specialization

In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the native relationship evidence on this page.

Graph controls are ready.

Interactive rendering requires JavaScript and WebGL. Use the native relationship evidence on this page while the interactive map is unavailable.

Native relationship evidence

Every exact file and test below is linked only from the concept that introduces it.

Introduced tests

Every collected test enters the hierarchy at exactly one concept.

No tests are introduced at this concept. Its intent tests are introduced by other concepts.

Introduced code

Every collected source range enters the hierarchy at exactly one concept.

5 files ranked by introduced lines: 44 introduced LOC across 18 ranges. Expand a file to inspect source; the > gutter marks introduced lines.

src/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.ts 23 introduced LOC · 11 ranges

Open complete file

99 return [null, null];
100 }
101 > let initialState: IState; tokenizerSyntaxTokenBackend.ts
102 > try {
103 > initialState = tokenizationSupport.getInitialState();
104 > } catch (e) {
105 onUnexpectedError(e);
106 return [null, null];
107 }
108 > return [tokenizationSupport, initialState]; tokenizerSyntaxTokenBackend.ts
109 };
110
111 const [tokenizationSupport, initialState] = initializeTokenization();
112 if (tokenizationSupport && initialState) {
113 > this._tokenizer = new TokenizerWithStateStoreAndTextModel(this._textModel.getLineCount(), tokenizationSupport, this._textModel, this._languageIdCodec); tokenizerSyntaxTokenBackend.ts
114 } else {
115 this._tokenizer = null;
120 this._defaultBackgroundTokenizer = null;
121 if (this._tokenizer) {
122 > const b: IBackgroundTokenizationStore = { tokenizerSyntaxTokenBackend.ts
123 > setTokens: (tokens) => {
124 this.setTokens(tokens);
125 },
126 > setFontInfo: (changes: FontTokensUpdate) => { tokenizerSyntaxTokenBackend.ts
127 this.setFontInfo(changes);
128 },
129 > backgroundTokenizationFinished: () => { tokenizerSyntaxTokenBackend.ts
130 if (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) {
131 // We already did a full tokenization and don't go back to progressing.
136 this._onDidChangeBackgroundTokenizationState.fire();
137 },
138 > setEndState: (lineNumber, state) => { tokenizerSyntaxTokenBackend.ts
139 if (!this._tokenizer) { return; }
140 const firstInvalidEndStateLineNumber = this._tokenizer.store.getFirstInvalidEndStateLineNumber();
144 }
145 },
146 > }; tokenizerSyntaxTokenBackend.ts
147 >
148 > if (tokenizationSupport && tokenizationSupport.createBackgroundTokenizer && !tokenizationSupport.backgroundTokenizerShouldOnlyVerifyTokens) {
149 this._backgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, b);
150 }
151 > if (!this._backgroundTokenizer.value && !this._textModel.isTooLargeForTokenization()) { tokenizerSyntaxTokenBackend.ts
152 this._backgroundTokenizer.value = this._defaultBackgroundTokenizer =
153 new DefaultBackgroundTokenizer(this._tokenizer, b);
154 this._defaultBackgroundTokenizer.handleChanges();
155 }
156 > tokenizerSyntaxTokenBackend.ts
157 > if (tokenizationSupport?.backgroundTokenizerShouldOnlyVerifyTokens && tokenizationSupport.createBackgroundTokenizer) {
158 this._debugBackgroundTokens = new ContiguousTokensStore(this._languageIdCodec);
159 this._debugBackgroundStates = new TrackingTokenizationStateStore(this._textModel.getLineCount());
173 },
174 });
175 > } else { tokenizerSyntaxTokenBackend.ts
176 > this._debugBackgroundTokens = undefined;
177 > this._debugBackgroundStates = undefined;
178 > this._debugBackgroundTokenizer.value = undefined;
179 > }
180 > }
181
182 this.refreshAllVisibleLineTokens();
src/vs/editor/common/model/textModelTokens.ts 16 introduced LOC · 4 ranges

Open complete file

31
32 constructor(
33 > lineCount: number, textModelTokens.ts
34 > public readonly tokenizationSupport: ITokenizationSupport
35 > ) {
36 > this.initialState = this.tokenizationSupport.getInitialState() as TState;
37 > this.store = new TrackingTokenizationStateStore<TState>(lineCount);
38 > }
39
40 public getStartState(lineNumber: number): TState | null {
49 export class TokenizerWithStateStoreAndTextModel<TState extends IState = IState> extends TokenizerWithStateStore<TState> {
50 constructor(
51 > lineCount: number, textModelTokens.ts
52 > tokenizationSupport: ITokenizationSupport,
53 > public readonly _textModel: ITextModel,
54 > public readonly _languageIdCodec: ILanguageIdCodec
55 > ) {
56 > super(lineCount, tokenizationSupport);
57 > }
58
59 public updateTokensUntilLine(builder: ContiguousMultilineTokensBuilder, lineNumber: number): void {
219
220 constructor(private lineCount: number) {
221 > this._invalidEndStatesLineNumbers.addRange(new OffsetRange(1, lineCount + 1)); textModelTokens.ts
222 > }
223
224 public getEndState(lineNumber: number): TState | null {
289
290 export class TokenizationStateStore<TState extends IState> {
291 > private readonly _lineEndStates = new FixedArray<TState | null>(null); textModelTokens.ts
292
293 public getEndState(lineNumber: number): TState | null {
src/vs/editor/common/model/fixedArray.ts 2 introduced LOC · 1 range

Open complete file

14
15 constructor(
16 > private readonly _default: T fixedArray.ts
17 > ) { }
18
19 public get(index: number): T {
src/vs/editor/common/tokenizationRegistry.ts 2 introduced LOC · 1 range

Open complete file

65 const tokenizationSupport = this.get(languageId);
66 if (tokenizationSupport) {
67 > return tokenizationSupport; tokenizationRegistry.ts
68 > }
69
70 const factory = this._factories.get(languageId);
src/vs/editor/common/model/tokens/tokenizationTextModelPart.ts 1 introduced LOC · 1 range

Open complete file

82
83 reader.store.add(tokens.onDidChangeTokens(e => {
84 > this._emitModelTokensChangedEvent(e); tokenizationTextModelPart.ts
85 }));
86 reader.store.add(tokens.onDidChangeFontTokens(e => {

Direct hierarchy

Parents

Children