Vihren Atlas

tokenizer.ts ×18

Code frontier

unlabeled · c_51111ec9073a

862 tests · 17324 LOC · 72 files · introduces 0 tests · 196 LOC · 2 files

Introduces — evidence that enters the hierarchy at this concept

Code
30 ranges196 lines · 2 files
Tests
0 tests

Contains — complete concept membership

All code (extent)
1851 ranges17324 lines · 72 files · Browse complete extent
All tests (intent)
862 testsBrowse complete intent

Neighbourhood graph

The orange circle is the focus. Violet and green circles are every ancestor and descendant, broader and narrower, at any distance; blue squares and pink diamonds are the introduced files and exact introduced tests of every visible concept, not only the focus's. Arrows point from broader to narrower concepts and bridge only concepts omitted from this view. Undirected links show source or test introduction. Concept and file size follows LOC; exact test nodes use test-count units.

Introduced files, introduced tests, and structurally relevant concept specialization

In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the native relationship evidence on this page.

Graph controls are ready.

Interactive rendering requires JavaScript and WebGL. Use the native relationship evidence on this page while the interactive map is unavailable.

Native relationship evidence

Every exact file and test below is linked only from the concept that introduces it.

Introduced tests

Every collected test enters the hierarchy at exactly one concept.

No tests are introduced at this concept. Its intent tests are introduced by other concepts.

Introduced code

Every collected source range enters the hierarchy at exactly one concept.

2 files ranked by introduced lines: 196 introduced LOC across 30 ranges. Expand a file to inspect source; the > gutter marks introduced lines.

src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts 114 introduced LOC · 18 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- tokenizer.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { NotSupportedError } from '../../../../../base/common/errors.js';
7 > import { StandardTokenType, TokenMetadata } from '../../../encodedTokenAttributes.js';
8 > import { IViewLineTokens } from '../../../tokens/lineTokens.js';
9 > import { BracketAstNode, TextAstNode } from './ast.js';
10 > import { BracketTokens, LanguageAgnosticBracketTokens } from './brackets.js';
11 > import { Length, lengthAdd, lengthDiff, lengthGetColumnCountIfZeroLineCount, lengthToObj, lengthZero, toLength } from './length.js';
12 > import { SmallImmutableSet } from './smallImmutableSet.js';
13 >
14 > export interface Tokenizer {
15 > readonly offset: Length;
16 > readonly length: Length;
17 >
18 > read(): Token | null;
19 > peek(): Token | null;
20 > skip(length: Length): void;
21 >
22 > getText(): string;
23 > }
24 >
25 > export const enum TokenKind {
26 > Text = 0,
27 > OpeningBracket = 1,
28 > ClosingBracket = 2,
29 > }
30 >
31 > export type OpeningBracketId = number;
32 >
33 > export class Token {
34 > constructor(
35 readonly length: Length,
36 readonly kind: TokenKind,
49 readonly astNode: BracketAstNode | TextAstNode | undefined,
50 ) { }
51 > } tokenizer.ts
52 >
53 > export interface ITokenizerSource {
54 > getValue(): string;
55 > getLineCount(): number;
56 > getLineLength(lineNumber: number): number;
57 >
58 > tokenization: {
59 > getLineTokens(lineNumber: number): IViewLineTokens;
60 > };
61 > }
62 >
63 > export class TextBufferTokenizer implements Tokenizer {
64 > private readonly textBufferLineCount: number;
65 > private readonly textBufferLastLineLength: number;
66 >
67 > private readonly reader;
68 >
69 > constructor(
70 private readonly textModel: ITokenizerSource,
71 private readonly bracketTokens: LanguageAgnosticBracketTokens
78 this.textBufferLastLineLength = textModel.getLineLength(this.textBufferLineCount);
79 }
80 > tokenizer.ts
81 > private _offset: Length;
82 >
83 > get offset() {
84 return this._offset;
85 }
86 > tokenizer.ts
87 > get length() {
88 return toLength(this.textBufferLineCount - 1, this.textBufferLastLineLength);
89 }
90 > tokenizer.ts
91 > getText() {
92 return this.textModel.getValue();
93 }
94 > tokenizer.ts
95 > skip(length: Length): void {
96 this.didPeek = false;
97 this._offset = lengthAdd(this._offset, length);
99 this.reader.setPosition(obj.lineCount, obj.columnCount);
100 }
101 > tokenizer.ts
102 > private didPeek;
103 > private peeked: Token | null;
104 >
105 > read(): Token | null {
106 let token: Token | null;
107 if (this.peeked) {
116 return token;
117 }
118 > tokenizer.ts
119 > peek(): Token | null {
120 if (!this.didPeek) {
121 this.peeked = this.reader.read();
124 return this.peeked;
125 }
126 > } tokenizer.ts
127 >
128 > /**
129 > * Does not support peek.
130 > */
131 > class NonPeekableTextBufferTokenizer {
132 > private readonly textBufferLineCount: number;
133 > private readonly textBufferLastLineLength: number;
134 >
135 > constructor(private readonly textModel: ITokenizerSource, private readonly bracketTokens: LanguageAgnosticBracketTokens) {
136 this.textBufferLineCount = textModel.getLineCount();
137 this.textBufferLastLineLength = textModel.getLineLength(this.textBufferLineCount);
138 }
139 > tokenizer.ts
140 > private lineIdx = 0;
141 > private line: string | null = null;
142 > private lineCharOffset = 0;
143 > private lineTokens: IViewLineTokens | null = null;
144 > private lineTokenOffset = 0;
145 >
146 > public setPosition(lineIdx: number, column: number): void {
147 // We must not jump into a token!
148 if (lineIdx === this.lineIdx) {
158 this.peekedToken = null;
159 }
160 > tokenizer.ts
161 > /** Must be a zero line token. The end of the document cannot be peeked. */
162 > private peekedToken: Token | null = null;
163 >
164 > public read(): Token | null {
165 if (this.peekedToken) {
166 const token = this.peekedToken;
278 return new Token(length, TokenKind.Text, -1, SmallImmutableSet.getEmpty(), new TextAstNode(length));
279 }
280 > } tokenizer.ts
281 >
282 > export class FastTokenizer implements Tokenizer {
283 > private _offset: Length = lengthZero;
284 > private readonly tokens: readonly Token[];
285 > private idx = 0;
286 >
287 > constructor(private readonly text: string, brackets: BracketTokens) {
288 const regExpStr = brackets.getRegExpStr();
289 const regexp = regExpStr ? new RegExp(regExpStr + '|\n', 'gi') : null;
372 this.tokens = tokens;
373 }
374 > tokenizer.ts
375 > get offset(): Length {
376 return this._offset;
377 }
378 > tokenizer.ts
379 > readonly length: Length;
380 >
381 > read(): Token | null {
382 return this.tokens[this.idx++] || null;
383 }
384 > tokenizer.ts
385 > peek(): Token | null {
386 return this.tokens[this.idx] || null;
387 }
388 > tokenizer.ts
389 > skip(length: Length): void {
390 throw new NotSupportedError();
391 }
392 > tokenizer.ts
393 > getText(): string {
394 return this.text;
395 }
396 > } tokenizer.ts
src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/brackets.ts 82 introduced LOC · 12 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- brackets.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 > import { escapeRegExpCharacters } from '../../../../../base/common/strings.js';
6 > import { ResolvedLanguageConfiguration } from '../../../languages/languageConfigurationRegistry.js';
7 > import { BracketKind } from '../../../languages/supports/languageBracketsConfiguration.js';
8 > import { BracketAstNode } from './ast.js';
9 > import { toLength } from './length.js';
10 > import { DenseKeyProvider, identityKeyProvider, SmallImmutableSet } from './smallImmutableSet.js';
11 > import { OpeningBracketId, Token, TokenKind } from './tokenizer.js';
12 >
13 > export class BracketTokens {
14 > static createFromLanguage(configuration: ResolvedLanguageConfiguration, denseKeyProvider: DenseKeyProvider<string>): BracketTokens {
15 > function getId(bracketInfo: BracketKind): OpeningBracketId {
16 > return denseKeyProvider.getKey(`${bracketInfo.languageId}:::${bracketInfo.bracketText}`);
17 > }
18 >
19 > const map = new Map<string, Token>();
20 > for (const openingBracket of configuration.bracketsNew.openingBrackets) {
21 > const length = toLength(0, openingBracket.bracketText.length);
22 > const openingTextId = getId(openingBracket);
23 > const bracketIds = SmallImmutableSet.getEmpty().add(openingTextId, identityKeyProvider);
24 > map.set(openingBracket.bracketText, new Token(
25 > length,
26 > TokenKind.OpeningBracket,
27 > openingTextId,
28 > bracketIds,
29 > BracketAstNode.create(length, openingBracket, bracketIds)
30 > ));
31 > }
32 >
33 > for (const closingBracket of configuration.bracketsNew.closingBrackets) {
34 > const length = toLength(0, closingBracket.bracketText.length);
35 > let bracketIds = SmallImmutableSet.getEmpty();
36 > const closingBrackets = closingBracket.getOpeningBrackets();
37 > for (const bracket of closingBrackets) {
38 > bracketIds = bracketIds.add(getId(bracket), identityKeyProvider);
39 > }
40 > map.set(closingBracket.bracketText, new Token(
41 > length,
42 > TokenKind.ClosingBracket,
43 > getId(closingBrackets[0]),
44 > bracketIds,
45 > BracketAstNode.create(length, closingBracket, bracketIds)
46 > ));
47 > }
48 >
49 > return new BracketTokens(map);
50 > }
51 >
52 > private hasRegExp = false;
53 > private _regExpGlobal: RegExp | null = null;
54 >
55 > constructor(
56 private readonly map: Map<string, Token>
57 ) { }
58 > brackets.ts
59 > getRegExpStr(): string | null {
60 if (this.isEmpty) {
61 return null;
67 }
68 }
69 > brackets.ts
70 > /**
71 > * Returns null if there is no such regexp (because there are no brackets).
72 > */
73 > get regExpGlobal(): RegExp | null {
74 if (!this.hasRegExp) {
75 const regExpStr = this.getRegExpStr();
79 return this._regExpGlobal;
80 }
81 > brackets.ts
82 > getToken(value: string): Token | undefined {
83 return this.map.get(value.toLowerCase());
84 }
85 > brackets.ts
86 > findClosingTokenText(openingBracketIds: SmallImmutableSet<OpeningBracketId>): string | undefined {
87 for (const [closingText, info] of this.map) {
88 if (info.kind === TokenKind.ClosingBracket && info.bracketIds.intersects(openingBracketIds)) {
92 return undefined;
93 }
94 > brackets.ts
95 > get isEmpty(): boolean {
96 return this.map.size === 0;
97 }
98 > } brackets.ts
99 >
100 function prepareBracketForRegExp(str: string): string {
101 let escaped = escapeRegExpCharacters(str);
110 return escaped;
111 }
112 > brackets.ts
113 > export class LanguageAgnosticBracketTokens {
114 > private readonly languageIdToBracketTokens = new Map<string, BracketTokens>();
115 >
116 > constructor(
117 private readonly denseKeyProvider: DenseKeyProvider<string>,
118 private readonly getLanguageConfiguration: (languageId: string) => ResolvedLanguageConfiguration,
119 ) {
120 }
121 > brackets.ts
122 > public didLanguageChange(languageId: string): boolean {
123 // Report a change whenever the language configuration updates.
124 return this.languageIdToBracketTokens.has(languageId);
125 }
126 > brackets.ts
127 > getSingleLanguageBracketTokens(languageId: string): BracketTokens {
128 let singleLanguageBracketTokens = this.languageIdToBracketTokens.get(languageId);
129 if (!singleLanguageBracketTokens) {
133 return singleLanguageBracketTokens;
134 }
135 > brackets.ts
136 > getToken(value: string, languageId: string): Token | undefined {
137 const singleLanguageBracketTokens = this.getSingleLanguageBracketTokens(languageId);
138 return singleLanguageBracketTokens.getToken(value);
139 }
140 > } brackets.ts

Direct hierarchy

Parents

Children