Vihren Atlas

textModelTokens.ts ×24

Code frontier

unlabeled · c_bfa43e6d14bf

22 tests · 40882 LOC · 238 files · introduces 0 tests · 79 LOC · 3 files

Introduces — evidence that enters the hierarchy at this concept

Code
28 ranges79 lines · 3 files
Tests
0 tests

Contains — complete concept membership

All code (extent)
5032 ranges40882 lines · 238 files · Browse complete extent
All tests (intent)
22 testsBrowse complete intent

Neighbourhood graph

The orange circle is the focus. Violet and green circles are every ancestor and descendant, broader and narrower, at any distance; blue squares and pink diamonds are the introduced files and exact introduced tests of every visible concept, not only the focus's. Arrows point from broader to narrower concepts and bridge only concepts omitted from this view. Undirected links show source or test introduction. Concept and file size follows LOC; exact test nodes use test-count units.

Introduced files, introduced tests, and structurally relevant concept specialization

In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the native relationship evidence on this page.

Graph controls are ready.

Interactive rendering requires JavaScript and WebGL. Use the native relationship evidence on this page while the interactive map is unavailable.

Native relationship evidence

Every exact file and test below is linked only from the concept that introduces it.

Introduced tests

Every collected test enters the hierarchy at exactly one concept.

No tests are introduced at this concept. Its intent tests are introduced by other concepts.

Introduced code

Every collected source range enters the hierarchy at exactly one concept.

3 files ranked by introduced lines: 79 introduced LOC across 28 ranges. Expand a file to inspect source; the > gutter marks introduced lines.

src/vs/editor/common/model/textModelTokens.ts 69 introduced LOC · 24 ranges

Open complete file

43
44 public getFirstInvalidLine(): { lineNumber: number; startState: TState } | null {
45 > return this.store.getFirstInvalidLine(this.initialState); textModelTokens.ts
46 > }
47 }
48
58
59 public updateTokensUntilLine(builder: ContiguousMultilineTokensBuilder, lineNumber: number): void {
60 > const languageId = this._textModel.getLanguageId(); textModelTokens.ts
61 >
62 > while (true) {
63 > const lineToTokenize = this.getFirstInvalidLine();
64 > if (!lineToTokenize || lineToTokenize.lineNumber > lineNumber) {
65 > break;
66 > }
67 >
68 > const text = this._textModel.getLineContent(lineToTokenize.lineNumber);
69 >
70 > const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, text, true, lineToTokenize.startState);
71 > builder.add(lineToTokenize.lineNumber, r.tokens);
72 > this.store.setEndState(lineToTokenize.lineNumber, r.endState as TState);
73 > }
74 > }
75
76 /** assumes state is up to date */
230 */
231 public setEndState(lineNumber: number, state: TState): boolean {
232 > if (!state) { textModelTokens.ts
233 throw new BugIndicatingError('Cannot set null/undefined state');
234 }
235 > textModelTokens.ts
236 > this._invalidEndStatesLineNumbers.delete(lineNumber);
237 > const r = this._tokenizationStateStore.setEndState(lineNumber, state);
238 > if (r && lineNumber < this.lineCount) {
239 // because the state changed, we cannot trust the next state anymore and have to invalidate it.
240 this._invalidEndStatesLineNumbers.addRange(new OffsetRange(lineNumber + 1, lineNumber + 2));
241 }
242 > textModelTokens.ts
243 > return r;
244 > }
245
246 public acceptChange(range: LineRange, newLineCount: number): void {
270
271 public getStartState(lineNumber: number, initialState: TState): TState | null {
272 > if (lineNumber === 1) { return initialState; } textModelTokens.ts
273 return this.getEndState(lineNumber - 1);
274 > } textModelTokens.ts
275
276 public getFirstInvalidLine(initialState: TState): { lineNumber: number; startState: TState } | null {
277 > const lineNumber = this.getFirstInvalidEndStateLineNumber(); textModelTokens.ts
278 > if (lineNumber === null) {
279 return null;
280 }
281 > const startState = this.getStartState(lineNumber, initialState); textModelTokens.ts
282 > if (!startState) {
283 throw new BugIndicatingError('Start state must be defined');
284 }
285 > textModelTokens.ts
286 > return { lineNumber, startState };
287 > }
288 }
289
296
297 public setEndState(lineNumber: number, state: TState): boolean {
298 > const oldState = this._lineEndStates.get(lineNumber); textModelTokens.ts
299 > if (oldState && oldState.equals(state)) {
300 return false;
301 }
302 > textModelTokens.ts
303 > this._lineEndStates.set(lineNumber, state);
304 > return true;
305 > }
306
307 public acceptChange(range: LineRange, newLineCount: number): void {
342
343 public get min(): number | null {
344 > if (this._ranges.length === 0) { textModelTokens.ts
345 return null;
346 }
347 > return this._ranges[0].start; textModelTokens.ts
348 > }
349
350 public removeMin(): number | null {
362
363 public delete(value: number): void {
364 > const idx = this._ranges.findIndex(r => r.contains(value)); textModelTokens.ts
365 > if (idx !== -1) {
366 > const range = this._ranges[idx];
367 > if (range.start === value) {
368 > if (range.endExclusive === value + 1) {
369 this._ranges.splice(idx, 1);
370 > } else { textModelTokens.ts
371 this._ranges[idx] = new OffsetRange(value + 1, range.endExclusive);
372 }
373 > } else { textModelTokens.ts
374 if (range.endExclusive === value + 1) {
375 this._ranges[idx] = new OffsetRange(range.start, value);
378 }
379 }
380 > } textModelTokens.ts
381 > }
382
383 public addRange(range: OffsetRange): void {
424
425
426 > function safeTokenize(languageIdCodec: ILanguageIdCodec, languageId: string, tokenizationSupport: ITokenizationSupport | null, text: string, hasEOL: boolean, state: IState): EncodedTokenizationResult { textModelTokens.ts
427 > let r: EncodedTokenizationResult | null = null;
428 >
429 > if (tokenizationSupport) {
430 > try {
431 > r = tokenizationSupport.tokenizeEncoded(text, hasEOL, state.clone());
432 > } catch (e) {
433 onUnexpectedError(e);
434 }
435 > } textModelTokens.ts
436 >
437 > if (!r) {
438 r = nullTokenizeEncoded(languageIdCodec.encodeLanguageId(languageId), state);
439 }
440 > textModelTokens.ts
441 > LineTokens.convertToEndOffset(r.tokens, text.length);
442 > return r;
443 > }
444
445 export class DefaultBackgroundTokenizer implements IBackgroundTokenizer {
546
547 public checkFinished(): void {
548 > if (this._isDisposed) { textModelTokens.ts
549 return;
550 }
551 > if (this._tokenizerWithStateStore.store.allStatesValid()) { textModelTokens.ts
552 this._backgroundTokenStore.backgroundTokenizationFinished();
553 }
554 > } textModelTokens.ts
555
556 public requestTokens(startLineNumber: number, endLineNumberExclusive: number): void {
src/vs/editor/common/model/fixedArray.ts 8 introduced LOC · 3 ranges

Open complete file

18
19 public get(index: number): T {
20 > if (index < this._store.length) { fixedArray.ts
21 return this._store[index];
22 }
23 > return this._default; fixedArray.ts
24 > }
25
26 public set(index: number, value: T): void {
27 > while (index >= this._store.length) { fixedArray.ts
28 > this._store[this._store.length] = this._default;
29 > }
30 > this._store[index] = value;
31 > }
32
33 public replace(index: number, oldLength: number, newLength: number): void {
src/vs/editor/common/core/ranges/offsetRange.ts 2 introduced LOC · 1 range

Open complete file

99
100 public contains(offset: number): boolean {
101 > return this.start <= offset && offset < this.endExclusive; offsetRange.ts
102 > }
103
104 /**

Direct hierarchy

Parents

Children