Vihren Atlas

src/vs/editor/common/tokens/contiguousTokensStore.ts

257 LOC · 246 covered · 11 uncovered · 75 ranges · 1708 concepts · 33 introducers · 861 tests

File neighbourhood

The centred file is linked to every concept that introduces one of its ranges, every test that runs code from the file, and the gray connector concepts standing between those tests and the file's own introducer concepts. Undirected links join concepts to every file where they introduce source and concepts to the tests they introduce; arrows show specialization between the displayed concepts and bridge only concepts omitted from this view. Concept colors match the source ranges below; connector concepts have no source color and are shown in gray.

Focused file, its introducer and connector concepts, their introduced files, and tests that run code from the file

In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the related-file, concept, and source links on this page.

Graph controls are ready.

Interactive rendering requires JavaScript and WebGL. Use the related-file, concept, and source links on this page while the interactive map is unavailable.

Files sharing a concept 48

A concept that introduces code in this file also introduces code in these files.

Files with a broader (ancestor) concept 46

These files introduce code for a direct ancestor — a broader, larger-extent concept — of a concept that introduces code here.

Files with a narrower (descendant) concept 55

These files introduce code for a direct descendant — a narrower, larger-intent concept — of a concept that introduces code here.

showing 50 of 55

1 > /*--------------------------------------------------------------------------------------------- textModel.ts ×190
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import * as arrays from '../../../base/common/arrays.js';
7 > import { Position } from '../core/position.js';
8 > import { IRange } from '../core/range.js';
9 > import { ContiguousTokensEditing, EMPTY_LINE_TOKENS, toUint32Array } from './contiguousTokensEditing.js';
10 > import { LineTokens } from './lineTokens.js';
11 > import { ILanguageIdCodec } from '../languages.js';
12 > import { LanguageId, FontStyle, ColorId, StandardTokenType, MetadataConsts, TokenMetadata } from '../encodedTokenAttributes.js';
13 > import { ITextModel } from '../model.js';
14 > import { ContiguousMultilineTokens } from './contiguousMultilineTokens.js';
15 >
16 > /**
17 > * Represents contiguous tokens in a text model.
18 > */
19 > export class ContiguousTokensStore {
20 > private _lineTokens: (Uint32Array | ArrayBuffer | null)[];
21 > private _len: number;
22 > private readonly _languageIdCodec: ILanguageIdCodec;
23 >
24 > constructor(languageIdCodec: ILanguageIdCodec) {
25 > this._lineTokens = []; textModel.ts ×20
26 > this._len = 0;
27 > this._languageIdCodec = languageIdCodec;
28 > }
29 > textModel.ts ×190
30 > public flush(): void {
31 > this._lineTokens = []; textModel.ts ×20
32 > this._len = 0;
33 > }
34 > textModel.ts ×190
35 > get hasTokens(): boolean {
36 > return this._lineTokens.length > 0; parser.ts ×14
37 > }
38 > textModel.ts ×190
39 > public getTokens(topLevelLanguageId: string, lineIndex: number, lineText: string): LineTokens {
40 > let rawLineTokens: Uint32Array | ArrayBuffer | null = null; contiguousTokensStore.ts ×3
41 > if (lineIndex < this._len) {
42 > rawLineTokens = this._lineTokens[lineIndex]; contiguousTokensStore.ts ×1
43 > }
44 > contiguousTokensStore.ts ×3
45 > if (rawLineTokens !== null && rawLineTokens !== EMPTY_LINE_TOKENS) {
46 > return new LineTokens(toUint32Array(rawLineTokens), lineText, this._languageIdCodec); contiguousTokensStore.ts ×1
47 > }
48 > contiguousTokensStore.ts ×1
49 > const lineTokens = new Uint32Array(2);
50 > lineTokens[0] = lineText.length;
51 > lineTokens[1] = getDefaultMetadata(this._languageIdCodec.encodeLanguageId(topLevelLanguageId));
52 > return new LineTokens(lineTokens, lineText, this._languageIdCodec);
53 > } contiguousTokensStore.ts ×3
54 > textModel.ts ×190
55 > private static _massageTokens(topLevelLanguageId: LanguageId, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null): Uint32Array | ArrayBuffer {
56 > contiguousTokensStore.ts ×11
57 > const tokens = _tokens ? toUint32Array(_tokens) : null;
58 >
59 > if (lineTextLength === 0) {
60 > let hasDifferentLanguageId = false; contiguousTokensStore.ts ×2
61 > if (tokens && tokens.length > 1) {
62 hasDifferentLanguageId = (TokenMetadata.getLanguageId(tokens[1]) !== topLevelLanguageId);
63 }
64 > contiguousTokensStore.ts ×2
65 > if (!hasDifferentLanguageId) {
66 > return EMPTY_LINE_TOKENS;
67 > }
68 > }
69 > contiguousTokensStore.ts ×1
70 > if (!tokens || tokens.length === 0) { contiguousTokensStore.ts ×11
71 > const tokens = new Uint32Array(2); contiguousTokensStore.ts ×1
72 > tokens[0] = lineTextLength;
73 > tokens[1] = getDefaultMetadata(topLevelLanguageId);
74 > return tokens.buffer;
75 > }
76 > contiguousTokensStore.ts ×2
77 > // Ensure the last token covers the end of the text
78 > tokens[tokens.length - 2] = lineTextLength;
79 >
80 > if (tokens.byteOffset === 0 && tokens.byteLength === tokens.buffer.byteLength) { contiguousTokensStore.ts ×11
81 > // Store directly the ArrayBuffer pointer to save an object contiguousTokensStore.ts ×2
82 > return tokens.buffer;
83 > }
84 return tokens;
85 > } contiguousTokensStore.ts ×11
86 > textModel.ts ×190
87 > private _ensureLine(lineIndex: number): void {
88 > while (lineIndex >= this._len) { contiguousTokensStore.ts ×11
89 > this._lineTokens[this._len] = null;
90 > this._len++;
91 > }
92 > }
93 > textModel.ts ×190
94 > private _deleteLines(start: number, deleteCount: number): void {
95 > if (deleteCount === 0) { contiguousTokensEditing.ts ×4
96 return;
97 }
98 > if (start + deleteCount > this._len) { contiguousTokensEditing.ts ×4
99 deleteCount = this._len - start;
100 }
101 > this._lineTokens.splice(start, deleteCount); contiguousTokensEditing.ts ×4
102 > this._len -= deleteCount;
103 > }
104 > textModel.ts ×190
105 > private _insertLines(insertIndex: number, insertCount: number): void {
106 > if (insertCount === 0) { contiguousTokensStore.ts ×3
107 return;
108 }
109 > const lineTokens: (Uint32Array | ArrayBuffer | null)[] = []; contiguousTokensStore.ts ×3
110 > for (let i = 0; i < insertCount; i++) {
111 > lineTokens[i] = null;
112 > }
113 > this._lineTokens = arrays.arrayInsert(this._lineTokens, insertIndex, lineTokens);
114 > this._len += insertCount;
115 > }
116 > textModel.ts ×190
117 > public setTokens(topLevelLanguageId: string, lineIndex: number, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null, checkEquality: boolean): boolean {
118 > const tokens = ContiguousTokensStore._massageTokens(this._languageIdCodec.encodeLanguageId(topLevelLanguageId), lineTextLength, _tokens); contiguousTokensStore.ts ×11
119 > this._ensureLine(lineIndex);
120 > const oldTokens = this._lineTokens[lineIndex];
121 > this._lineTokens[lineIndex] = tokens;
122 >
123 > if (checkEquality) {
124 > return !ContiguousTokensStore._equals(oldTokens, tokens);
125 > }
126 > return false; contiguousTokensStore.ts ×2
127 > } contiguousTokensStore.ts ×11
128 > textModel.ts ×190
129 > private static _equals(_a: Uint32Array | ArrayBuffer | null, _b: Uint32Array | ArrayBuffer | null) {
130 > if (!_a || !_b) { contiguousTokensStore.ts ×11
131 > return !_a && !_b;
132 > }
133 > contiguousTokensStore.ts ×1
134 > const a = toUint32Array(_a);
135 > const b = toUint32Array(_b);
136 >
137 > if (a.length !== b.length) {
138 > return false; contiguousTokensStore.ts ×1
139 > }
140 > for (let i = 0, len = a.length; i < len; i++) { contiguousTokensStore.ts ×2
141 > if (a[i] !== b[i]) { contiguousTokensStore.ts ×2
142 return false;
143 }
144 > } contiguousTokensStore.ts ×2
145 > return true; contiguousTokensStore.ts ×2
146 > } contiguousTokensStore.ts ×11
147 > textModel.ts ×190
148 > //#region Editing
149 >
150 > public acceptEdit(range: IRange, eolCount: number, firstLineLength: number): void {
151 > this._acceptDeleteRange(range); intervalTree.ts ×7
152 > this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength);
153 > }
154 > textModel.ts ×190
155 > private _acceptDeleteRange(range: IRange): void {
156 > intervalTree.ts ×7
157 > const firstLineIndex = range.startLineNumber - 1;
158 > if (firstLineIndex >= this._len) {
159 > return; contiguousTokensStore.ts ×1
160 > }
161 > textModelTokens.ts ×3
162 > if (range.startLineNumber === range.endLineNumber) {
163 > if (range.startColumn === range.endColumn) { contiguousTokensStore.ts ×1
164 > // Nothing to delete contiguousTokensStore.ts ×1
165 > return;
166 > }
167 > contiguousTokensStore.ts ×1
168 > this._lineTokens[firstLineIndex] = ContiguousTokensEditing.delete(this._lineTokens[firstLineIndex], range.startColumn - 1, range.endColumn - 1);
169 > return;
170 > }
171 > contiguousTokensEditing.ts ×4
172 > this._lineTokens[firstLineIndex] = ContiguousTokensEditing.deleteEnding(this._lineTokens[firstLineIndex], range.startColumn - 1);
173 >
174 > const lastLineIndex = range.endLineNumber - 1;
175 > let lastLineTokens: Uint32Array | ArrayBuffer | null = null;
176 > if (lastLineIndex < this._len) {
177 > lastLineTokens = ContiguousTokensEditing.deleteBeginning(this._lineTokens[lastLineIndex], range.endColumn - 1);
178 > }
179 >
180 > // Take remaining text on last line and append it to remaining text on first line
181 > this._lineTokens[firstLineIndex] = ContiguousTokensEditing.append(this._lineTokens[firstLineIndex], lastLineTokens);
182 >
183 > // Delete middle lines
184 > this._deleteLines(range.startLineNumber, range.endLineNumber - range.startLineNumber);
185 > } intervalTree.ts ×7
186 > textModel.ts ×190
187 > private _acceptInsertText(position: Position, eolCount: number, firstLineLength: number): void {
188 > intervalTree.ts ×7
189 > if (eolCount === 0 && firstLineLength === 0) {
190 > // Nothing to insert pieceTreeTextBuffer.ts ×1
191 > return;
192 > }
193 > pieceTreeTextBuffer.ts ×1
194 > const lineIndex = position.lineNumber - 1;
195 > if (lineIndex >= this._len) {
196 > return; contiguousTokensStore.ts ×1
197 > }
198 > contiguousTokensEditing.ts ×2
199 > if (eolCount === 0) {
200 > // Inserting text on one line contiguousTokensStore.ts ×1
201 > this._lineTokens[lineIndex] = ContiguousTokensEditing.insert(this._lineTokens[lineIndex], position.column - 1, firstLineLength);
202 > return;
203 > }
204 > contiguousTokensStore.ts ×3
205 > this._lineTokens[lineIndex] = ContiguousTokensEditing.deleteEnding(this._lineTokens[lineIndex], position.column - 1);
206 > this._lineTokens[lineIndex] = ContiguousTokensEditing.insert(this._lineTokens[lineIndex], position.column - 1, firstLineLength);
207 >
208 > this._insertLines(position.lineNumber, eolCount);
209 > } intervalTree.ts ×7
210 > textModel.ts ×190
211 > //#endregion
212 >
213 > public setMultilineTokens(tokens: ContiguousMultilineTokens[], textModel: ITextModel): { changes: { fromLineNumber: number; toLineNumber: number }[] } {
214 > if (tokens.length === 0) { tokenizerSyntaxTokenBackend.ts ×2
215 > return { changes: [] }; contiguousTokensStore.ts ×1
216 > }
217 > contiguousTokensStore.ts ×11
218 > const ranges: { fromLineNumber: number; toLineNumber: number }[] = [];
219 >
220 > for (let i = 0, len = tokens.length; i < len; i++) {
221 > const element = tokens[i];
222 > let minChangedLineNumber = 0;
223 > let maxChangedLineNumber = 0;
224 > let hasChange = false;
225 > for (let lineNumber = element.startLineNumber; lineNumber <= element.endLineNumber; lineNumber++) {
226 > if (hasChange) {
227 > this.setTokens(textModel.getLanguageId(), lineNumber - 1, textModel.getLineLength(lineNumber), element.getLineTokens(lineNumber), false); contiguousTokensStore.ts ×2
228 > maxChangedLineNumber = lineNumber;
229 > } else { contiguousTokensStore.ts ×11
230 > const lineHasChange = this.setTokens(textModel.getLanguageId(), lineNumber - 1, textModel.getLineLength(lineNumber), element.getLineTokens(lineNumber), true);
231 > if (lineHasChange) {
232 > hasChange = true;
233 > minChangedLineNumber = lineNumber;
234 > maxChangedLineNumber = lineNumber;
235 > }
236 > }
237 > }
238 > if (hasChange) {
239 > ranges.push({ fromLineNumber: minChangedLineNumber, toLineNumber: maxChangedLineNumber, });
240 > }
241 > }
242 >
243 > return { changes: ranges };
244 > } tokenizerSyntaxTokenBackend.ts ×2
245 > } textModel.ts ×190
246 >
247 > function getDefaultMetadata(topLevelLanguageId: LanguageId): number { contiguousTokensStore.ts ×1
248 > return (
249 > (topLevelLanguageId << MetadataConsts.LANGUAGEID_OFFSET)
250 > | (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET)
251 > | (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET)
252 > | (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET)
253 > | (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET)
254 > // If there is no grammar, we just take a guess and try to match brackets.
255 > | (MetadataConsts.BALANCED_BRACKETS_MASK)
256 > ) >>> 0;
257 > }