Vihren Atlas

tokenizer.ts ×13

Code frontier

unlabeled · c_e5d862c81674

17 tests · 40922 LOC · 238 files · introduces 0 tests · 93 LOC · 3 files

Introduces — evidence that enters the hierarchy at this concept

Code
16 ranges93 lines · 3 files
Tests
0 tests

Contains — complete concept membership

All code (extent)
5042 ranges40922 lines · 238 files · Browse complete extent
All tests (intent)
17 testsBrowse complete intent

Neighbourhood graph

The orange circle is the focus. Violet and green circles are every ancestor and descendant, broader and narrower, at any distance; blue squares and pink diamonds are the introduced files and exact introduced tests of every visible concept, not only the focus's. Arrows point from broader to narrower concepts and bridge only concepts omitted from this view. Undirected links show source or test introduction. Concept and file size follows LOC; exact test nodes use test-count units.

Introduced files, introduced tests, and structurally relevant concept specialization

In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the native relationship evidence on this page.

Graph controls are ready.

Interactive rendering requires JavaScript and WebGL. Use the native relationship evidence on this page while the interactive map is unavailable.

Native relationship evidence

Every exact file and test below is linked only from the concept that introduces it.

Introduced tests

Every collected test enters the hierarchy at exactly one concept.

No tests are introduced at this concept. Its intent tests are introduced by other concepts.

Introduced code

Every collected source range enters the hierarchy at exactly one concept.

3 files ranked by introduced lines: 93 introduced LOC across 16 ranges. Expand a file to inspect source; the > gutter marks introduced lines.

src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts 88 introduced LOC · 13 ranges

Open complete file

68
69 constructor(
70 > private readonly textModel: ITokenizerSource, tokenizer.ts
71 > private readonly bracketTokens: LanguageAgnosticBracketTokens
72 > ) {
73 > this.reader = new NonPeekableTextBufferTokenizer(this.textModel, this.bracketTokens);
74 > this._offset = lengthZero;
75 > this.didPeek = false;
76 > this.peeked = null;
77 > this.textBufferLineCount = textModel.getLineCount();
78 > this.textBufferLastLineLength = textModel.getLineLength(this.textBufferLineCount);
79 > }
80
81 private _offset: Length;
104
105 read(): Token | null {
106 > let token: Token | null; tokenizer.ts
107 > if (this.peeked) {
108 this.didPeek = false;
109 token = this.peeked;
110 > } else { tokenizer.ts
111 token = this.reader.read();
112 }
113 > if (token) { tokenizer.ts
114 > this._offset = lengthAdd(this._offset, token.length);
115 > }
116 > return token;
117 > }
118
119 peek(): Token | null {
134
135 constructor(private readonly textModel: ITokenizerSource, private readonly bracketTokens: LanguageAgnosticBracketTokens) {
136 > this.textBufferLineCount = textModel.getLineCount(); tokenizer.ts
137 > this.textBufferLastLineLength = textModel.getLineLength(this.textBufferLineCount);
138 > }
139
140 private lineIdx = 0;
163
164 public read(): Token | null {
165 > if (this.peekedToken) { tokenizer.ts
166 const token = this.peekedToken;
167 this.peekedToken = null;
169 return token;
170 }
171 > tokenizer.ts
172 > if (this.lineIdx > this.textBufferLineCount - 1 || (this.lineIdx === this.textBufferLineCount - 1 && this.lineCharOffset >= this.textBufferLastLineLength)) {
173 > // We are after the end
174 > return null;
175 > }
176 >
177 > if (this.line === null) {
178 > this.lineTokens = this.textModel.tokenization.getLineTokens(this.lineIdx + 1);
179 > this.line = this.lineTokens.getLineContent();
180 > this.lineTokenOffset = this.lineCharOffset === 0 ? 0 : this.lineTokens.findTokenIndexAtOffset(this.lineCharOffset);
181 > }
182 >
183 > const startLineIdx = this.lineIdx;
184 > const startLineCharOffset = this.lineCharOffset;
185 >
186 > // limits the length of text tokens.
187 > // If text tokens get too long, incremental updates will be slow
188 > let lengthHeuristic = 0;
189 > while (true) {
190 > const lineTokens = this.lineTokens!;
191 > const tokenCount = lineTokens.getCount();
192 >
193 > let peekedBracketToken: Token | null = null;
194 >
195 > if (this.lineTokenOffset < tokenCount) {
196 > const tokenMetadata = lineTokens.getMetadata(this.lineTokenOffset);
197 > while (this.lineTokenOffset + 1 < tokenCount && tokenMetadata === lineTokens.getMetadata(this.lineTokenOffset + 1)) {
198 // Skip tokens that are identical.
199 // Sometimes, (bracket) identifiers are split up into multiple tokens.
200 this.lineTokenOffset++;
201 }
202 > tokenizer.ts
203 > const isOther = TokenMetadata.getTokenType(tokenMetadata) === StandardTokenType.Other;
204 > const containsBracketType = TokenMetadata.containsBalancedBrackets(tokenMetadata);
205 >
206 > const endOffset = lineTokens.getEndOffset(this.lineTokenOffset);
207 > // Is there a bracket token next? Only consume text.
208 > if (containsBracketType && isOther && this.lineCharOffset < endOffset) {
209 const languageId = lineTokens.getLanguageId(this.lineTokenOffset);
210 const text = this.line.substring(this.lineCharOffset, endOffset);
224 }
225 }
226 > tokenizer.ts
227 > lengthHeuristic += endOffset - this.lineCharOffset;
228 >
229 > if (peekedBracketToken) {
230 // Don't skip the entire token, as a single token could contain multiple brackets.
231
239 return peekedBracketToken;
240 }
241 > } else { tokenizer.ts
242 > // Skip the entire token, as the token contains no brackets at all.
243 > this.lineTokenOffset++;
244 > this.lineCharOffset = endOffset;
245 > }
246 > } else {
247 > if (this.lineIdx === this.textBufferLineCount - 1) {
248 break;
249 }
250 > this.lineIdx++; tokenizer.ts
251 > this.lineTokens = this.textModel.tokenization.getLineTokens(this.lineIdx + 1);
252 > this.lineTokenOffset = 0;
253 > this.line = this.lineTokens.getLineContent();
254 > this.lineCharOffset = 0;
255 >
256 > lengthHeuristic += 33; // max 1000/33 = 30 lines
257 > // This limits the amount of work to recompute min-indentation
258 >
259 > if (lengthHeuristic > 1000) {
260 // only break (automatically) at the end of line.
261 break;
262 }
263 > } tokenizer.ts
264 >
265 > if (lengthHeuristic > 1500) {
266 // Eventually break regardless of the line length so that
267 // very long lines do not cause bad performance.
270 break;
271 }
272 > } tokenizer.ts
273 >
274 > // If a token contains some proper indentation, it also contains \n{INDENTATION+}(?!{INDENTATION}),
275 > // unless the line is too long.
276 > // Thus, the min indentation of the document is the minimum min indentation of every text node.
277 > const length = lengthDiff(startLineIdx, startLineCharOffset, this.lineIdx, this.lineCharOffset);
278 > return new Token(length, TokenKind.Text, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length));
279 > }
280 }
281
src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/length.ts 3 introduced LOC · 2 ranges

Open complete file

13 */
14 export function lengthDiff(startLineCount: number, startColumnCount: number, endLineCount: number, endColumnCount: number): Length {
15 > return (startLineCount !== endLineCount) length.ts
16 > ? toLength(endLineCount - startLineCount, endColumnCount)
17 : toLength(0, endColumnCount - startColumnCount);
18 > } length.ts
19
20 /**
src/vs/editor/common/encodedTokenAttributes.ts 2 introduced LOC · 1 range

Open complete file

108
109 public static containsBalancedBrackets(metadata: number): boolean {
110 > return (metadata & MetadataConsts.BALANCED_BRACKETS_MASK) !== 0; encodedTokenAttributes.ts
111 > }
112
113 public static getFontStyle(metadata: number): FontStyle {

Direct hierarchy

Parents

Children