Vihren Atlas

src/vs/platform/contextkey/common/scanner.ts

384 LOC · 378 covered · 6 uncovered · 132 ranges · 11454 concepts · 41 introducers · 5884 tests

File neighbourhood

The centred file is linked to every concept that introduces one of its ranges, every test that runs code from the file, and the gray connector concepts standing between those tests and the file's own introducer concepts. Undirected links join concepts to every file where they introduce source and concepts to the tests they introduce; arrows show specialization between the displayed concepts and bridge only concepts omitted from this view. Concept colors match the source ranges below; connector concepts have no source color and are shown in gray.

Focused file, its introducer and connector concepts, their introduced files, and tests that run code from the file

In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the related-file, concept, and source links on this page.

Graph controls are ready.

Interactive rendering requires JavaScript and WebGL. Use the related-file, concept, and source links on this page while the interactive map is unavailable.

Files sharing a concept 1

A concept that introduces code in this file also introduces code in these files.

Files with a broader (ancestor) concept 2

These files introduce code for a direct ancestor — a broader, larger-extent concept — of a concept that introduces code here.

Files with a narrower (descendant) concept 3

These files introduce code for a direct descendant — a narrower, larger-intent concept — of a concept that introduces code here.

1 > /*--------------------------------------------------------------------------------------------- scanner.ts ×36
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { CharCode } from '../../../base/common/charCode.js';
7 > import { illegalState } from '../../../base/common/errors.js';
8 > import { localize } from '../../../nls.js';
9 >
10 > export const enum TokenType {
11 > LParen,
12 > RParen,
13 > Neg,
14 > Eq,
15 > NotEq,
16 > Lt,
17 > LtEq,
18 > Gt,
19 > GtEq,
20 > RegexOp,
21 > RegexStr,
22 > True,
23 > False,
24 > In,
25 > Not,
26 > And,
27 > Or,
28 > Str,
29 > QuotedStr,
30 > Error,
31 > EOF,
32 > }
33 >
34 > export type Token =
35 > | { type: TokenType.LParen; offset: number }
36 > | { type: TokenType.RParen; offset: number }
37 > | { type: TokenType.Neg; offset: number }
38 > | { type: TokenType.Eq; offset: number; isTripleEq: boolean }
39 > | { type: TokenType.NotEq; offset: number; isTripleEq: boolean }
40 > | { type: TokenType.Lt; offset: number }
41 > | { type: TokenType.LtEq; offset: number }
42 > | { type: TokenType.Gt; offset: number }
43 > | { type: TokenType.GtEq; offset: number }
44 > | { type: TokenType.RegexOp; offset: number }
45 > | { type: TokenType.RegexStr; offset: number; lexeme: string }
46 > | { type: TokenType.True; offset: number }
47 > | { type: TokenType.False; offset: number }
48 > | { type: TokenType.In; offset: number }
49 > | { type: TokenType.Not; offset: number }
50 > | { type: TokenType.And; offset: number }
51 > | { type: TokenType.Or; offset: number }
52 > | { type: TokenType.Str; offset: number; lexeme: string }
53 > | { type: TokenType.QuotedStr; offset: number; lexeme: string }
54 > | { type: TokenType.Error; offset: number; lexeme: string }
55 > | { type: TokenType.EOF; offset: number };
56 >
57 > type KeywordTokenType = TokenType.Not | TokenType.In | TokenType.False | TokenType.True;
58 > type TokenTypeWithoutLexeme =
59 > TokenType.LParen |
60 > TokenType.RParen |
61 > TokenType.Neg |
62 > TokenType.Lt |
63 > TokenType.LtEq |
64 > TokenType.Gt |
65 > TokenType.GtEq |
66 > TokenType.RegexOp |
67 > TokenType.True |
68 > TokenType.False |
69 > TokenType.In |
70 > TokenType.Not |
71 > TokenType.And |
72 > TokenType.Or |
73 > TokenType.EOF;
74 >
75 > /**
76 > * Example:
77 > * `foo == bar'` - note how single quote doesn't have a corresponding closing quote,
78 > * so it's reported as unexpected
79 > */
80 > export type LexingError = {
81 > offset: number; /** note that this doesn't take into account escape characters from the original encoding of the string, e.g., within an extension manifest file's JSON encoding */
82 > lexeme: string;
83 > additionalInfo?: string;
84 > };
85 >
86 > function hintDidYouMean(...meant: string[]) { scanner.ts ×5
87 > switch (meant.length) {
88 > case 1:
89 > return localize('contextkey.scanner.hint.didYouMean1', "Did you mean {0}?", meant[0]); scanner.ts ×2
90 > case 2: scanner.ts ×5
91 > return localize('contextkey.scanner.hint.didYouMean2', "Did you mean {0} or {1}?", meant[0], meant[1]); scanner.ts ×2
92 > case 3: scanner.ts ×5
93 return localize('contextkey.scanner.hint.didYouMean3', "Did you mean {0}, {1} or {2}?", meant[0], meant[1], meant[2]);
94 > default: // we just don't expect that many scanner.ts ×5
95 return undefined;
96 > } scanner.ts ×5
97 > }
98 > scanner.ts ×36
99 > const hintDidYouForgetToOpenOrCloseQuote = localize('contextkey.scanner.hint.didYouForgetToOpenOrCloseQuote', "Did you forget to open or close the quote?");
100 > const hintDidYouForgetToEscapeSlash = localize('contextkey.scanner.hint.didYouForgetToEscapeSlash', "Did you forget to escape the '/' (slash) character? Put two backslashes before it to escape, e.g., '\\\\/\'.");
101 >
102 > /**
103 > * A simple scanner for context keys.
104 > *
105 > * Example:
106 > *
107 > * ```ts
108 > * const scanner = new Scanner().reset('resourceFileName =~ /docker/ && !config.docker.enabled');
109 > * const tokens = [...scanner];
110 > * if (scanner.errorTokens.length > 0) {
111 > * scanner.errorTokens.forEach(err => console.error(`Unexpected token at ${err.offset}: ${err.lexeme}\nHint: ${err.additional}`));
112 > * } else {
113 > * // process tokens
114 > * }
115 > * ```
116 > */
117 > export class Scanner {
118 >
119 > static getLexeme(token: Token): string {
120 > switch (token.type) {
121 > case TokenType.LParen:
122 > return '('; scanner.ts ×3
123 > case TokenType.RParen: scanner.ts ×36
124 > return ')'; scanner.ts ×3
125 > case TokenType.Neg: scanner.ts ×36
126 > return '!'; scanner.ts ×1
127 > case TokenType.Eq: scanner.ts ×36
128 > return token.isTripleEq ? '===' : '=='; scanner.ts ×1
129 > case TokenType.NotEq: scanner.ts ×36
130 > return token.isTripleEq ? '!==' : '!='; scanner.ts ×13
131 > case TokenType.Lt: scanner.ts ×36
132 > return '<'; scanner.ts ×13
133 > case TokenType.LtEq: scanner.ts ×36
134 > return '<='; scanner.ts ×13
135 > case TokenType.Gt: scanner.ts ×36
136 > return '>'; scanner.ts ×13
137 > case TokenType.GtEq: scanner.ts ×36
138 > return '>='; scanner.ts ×13
139 > case TokenType.RegexOp: scanner.ts ×36
140 > return '=~'; scanner.ts ×13
141 > case TokenType.RegexStr: scanner.ts ×36
142 > return token.lexeme; scanner.ts ×3
143 > case TokenType.True: scanner.ts ×36
144 > return 'true'; scanner.ts ×13
145 > case TokenType.False: scanner.ts ×36
146 > return 'false'; scanner.ts ×13
147 > case TokenType.In: scanner.ts ×36
148 > return 'in'; scanner.ts ×1
149 > case TokenType.Not: scanner.ts ×36
150 > return 'not'; scanner.ts ×13
151 > case TokenType.And: scanner.ts ×36
152 > return '&&'; scanner.ts ×1
153 > case TokenType.Or: scanner.ts ×36
154 > return '||'; scanner.ts ×13
155 > case TokenType.Str: scanner.ts ×36
156 > return token.lexeme; scanner.ts ×1
157 > case TokenType.QuotedStr: scanner.ts ×36
158 > return token.lexeme; scanner.ts ×13
159 > case TokenType.Error: scanner.ts ×36
160 > return token.lexeme; scanner.ts ×1
161 > case TokenType.EOF: scanner.ts ×36
162 > return 'EOF'; scanner.ts ×13
163 > default: scanner.ts ×36
164 > throw illegalState(`unhandled token type: ${JSON.stringify(token)}; have you forgotten to add a case?`); scanner.ts ×13
165 > } scanner.ts ×36
166 > }
167 >
168 > private static _regexFlags = new Set(['i', 'g', 's', 'm', 'y', 'u'].map(ch => ch.charCodeAt(0)));
169 >
170 > private static _keywords = new Map<string, KeywordTokenType>([
171 > ['not', TokenType.Not],
172 > ['in', TokenType.In],
173 > ['false', TokenType.False],
174 > ['true', TokenType.True],
175 > ]);
176 >
177 > private _input: string = '';
178 > private _start: number = 0;
179 > private _current: number = 0;
180 > private _tokens: Token[] = [];
181 > private _errors: LexingError[] = [];
182 >
183 > get errors(): Readonly<LexingError[]> {
184 > return this._errors; contextkey.ts ×4
185 > }
186 > scanner.ts ×36
187 > reset(value: string) {
188 > this._input = value; scanner.ts ×11
189 >
190 > this._start = 0;
191 > this._current = 0;
192 > this._tokens = [];
193 > this._errors = [];
194 >
195 > return this;
196 > }
197 > scanner.ts ×36
198 > scan() {
199 > while (!this._isAtEnd()) { scanner.ts ×11
200 >
201 > this._start = this._current;
202 >
203 > const ch = this._advance();
204 > switch (ch) {
205 > case CharCode.OpenParen: this._addToken(TokenType.LParen); break;
206 > case CharCode.CloseParen: this._addToken(TokenType.RParen); break;
207 >
208 > case CharCode.ExclamationMark:
209 > if (this._match(CharCode.Equals)) { scanner.ts ×3
210 > const isTripleEq = this._match(CharCode.Equals); // eat last `=` if `!==` scanner.ts ×1
211 > this._tokens.push({ type: TokenType.NotEq, offset: this._start, isTripleEq });
212 > } else { scanner.ts ×3
213 > this._addToken(TokenType.Neg); scanner.ts ×1
214 > }
215 > break; scanner.ts ×3
216 > scanner.ts ×11
217 > case CharCode.SingleQuote: this._quotedString(); break;
218 > case CharCode.Slash: this._regex(); break;
219 >
220 > case CharCode.Equals:
221 > if (this._match(CharCode.Equals)) { // support `==` scanner.ts ×3
222 > const isTripleEq = this._match(CharCode.Equals); // eat last `=` if `===` scanner.ts ×1
223 > this._tokens.push({ type: TokenType.Eq, offset: this._start, isTripleEq });
224 > } else if (this._match(CharCode.Tilde)) { scanner.ts ×3
225 > this._addToken(TokenType.RegexOp); scanner.ts ×1
226 > } else { scanner.ts ×1
227 > this._error(hintDidYouMean('==', '=~')); scanner.ts ×2
228 > }
229 > break; scanner.ts ×3
230 > scanner.ts ×11
231 > case CharCode.LessThan: this._addToken(this._match(CharCode.Equals) ? TokenType.LtEq : TokenType.Lt); break;
232 >
233 > case CharCode.GreaterThan: this._addToken(this._match(CharCode.Equals) ? TokenType.GtEq : TokenType.Gt); break;
234 >
235 > case CharCode.Ampersand:
236 > if (this._match(CharCode.Ampersand)) { scanner.ts ×2
237 > this._addToken(TokenType.And);
238 > } else {
239 this._error(hintDidYouMean('&&'));
240 }
241 > break; scanner.ts ×2
242 > scanner.ts ×11
243 > case CharCode.Pipe:
244 > if (this._match(CharCode.Pipe)) { scanner.ts ×3
245 > this._addToken(TokenType.Or); scanner.ts ×1
246 > } else { scanner.ts ×3
247 > this._error(hintDidYouMean('||')); scanner.ts ×2
248 > }
249 > break; scanner.ts ×3
250 > scanner.ts ×11
251 > // TODO@ulugbekna: 1) rewrite using a regex 2) reconsider what characters are considered whitespace, including unicode, nbsp, etc.
252 > case CharCode.Space:
253 > case CharCode.CarriageReturn:
254 > case CharCode.Tab:
255 > case CharCode.LineFeed:
256 > case CharCode.NoBreakSpace: // &nbsp
257 > break; scanner.ts ×1
258 > scanner.ts ×11
259 > default:
260 > this._string(); scanner.ts ×3
261 > } scanner.ts ×11
262 > }
263 >
264 > this._start = this._current;
265 > this._addToken(TokenType.EOF);
266 >
267 > return Array.from(this._tokens);
268 > }
269 > scanner.ts ×36
270 > private _match(expected: number): boolean {
271 > if (this._isAtEnd()) { scanner.ts ×3
272 return false;
273 }
274 > if (this._input.charCodeAt(this._current) !== expected) { scanner.ts ×3
275 > return false; scanner.ts ×1
276 > }
277 > this._current++; scanner.ts ×1
278 > return true;
279 > } scanner.ts ×3
280 > scanner.ts ×36
281 > private _advance(): number {
282 > return this._input.charCodeAt(this._current++); scanner.ts ×11
283 > }
284 > scanner.ts ×36
285 > private _peek(): number {
286 > return this._isAtEnd() ? CharCode.Null : this._input.charCodeAt(this._current); scanner.ts ×4
287 > }
288 > scanner.ts ×36
289 > private _addToken(type: TokenTypeWithoutLexeme) {
290 > this._tokens.push({ type, offset: this._start }); scanner.ts ×11
291 > }
292 > scanner.ts ×36
293 > private _error(additional?: string) {
294 > const offset = this._start; scanner.ts ×1
295 > const lexeme = this._input.substring(this._start, this._current);
296 > const errToken: Token = { type: TokenType.Error, offset: this._start, lexeme };
297 > this._errors.push({ offset, lexeme, additionalInfo: additional });
298 > this._tokens.push(errToken);
299 > }
300 > scanner.ts ×36
301 > // u - unicode, y - sticky // TODO@ulugbekna: we accept double quotes as part of the string rather than as a delimiter (to preserve old parser's behavior)
302 > private stringRe = /[a-zA-Z0-9_<>\-\./\\:\*\?\+\[\]\^,#@;"%\$\p{L}-]+/uy;
303 > private _string() {
304 > this.stringRe.lastIndex = this._start; scanner.ts ×3
305 > const match = this.stringRe.exec(this._input);
306 > if (match) {
307 > this._current = this._start + match[0].length;
308 > const lexeme = this._input.substring(this._start, this._current);
309 > const keyword = Scanner._keywords.get(lexeme);
310 > if (keyword) {
311 > this._addToken(keyword); scanner.ts ×1
312 > } else { scanner.ts ×3
313 > this._tokens.push({ type: TokenType.Str, lexeme, offset: this._start });
314 > }
315 > }
316 > }
317 > scanner.ts ×36
318 > // captures the lexeme without the leading and trailing '
319 > private _quotedString() {
320 > while (this._peek() !== CharCode.SingleQuote && !this._isAtEnd()) { // TODO@ulugbekna: add support for escaping ' ? scanner.ts ×4
321 > this._advance(); scanner.ts ×1
322 > }
323 > scanner.ts ×4
324 > if (this._isAtEnd()) {
325 > this._error(hintDidYouForgetToOpenOrCloseQuote); scanner.ts ×1
326 > return;
327 > }
328 > scanner.ts ×1
329 > // consume the closing '
330 > this._advance();
331 >
332 > this._tokens.push({ type: TokenType.QuotedStr, lexeme: this._input.substring(this._start + 1, this._current - 1), offset: this._start + 1 });
333 > } scanner.ts ×4
334 > scanner.ts ×36
335 > /*
336 > * Lexing a regex expression: /.../[igsmyu]*
337 > * Based on https://github.com/microsoft/TypeScript/blob/9247ef115e617805983740ba795d7a8164babf89/src/compiler/scanner.ts#L2129-L2181
338 > *
339 > * Note that we want slashes within a regex to be escaped, e.g., /file:\\/\\/\\// should match `file:///`
340 > */
341 > private _regex() {
342 > let p = this._current; scanner.ts ×9
343 >
344 > let inEscape = false;
345 > let inCharacterClass = false;
346 > while (true) {
347 > if (p >= this._input.length) {
348 > this._current = p; scanner.ts ×1
349 > this._error(hintDidYouForgetToEscapeSlash);
350 > return;
351 > }
352 > scanner.ts ×9
353 > const ch = this._input.charCodeAt(p);
354 >
355 > if (inEscape) { // parsing an escape character
356 > inEscape = false; scanner.ts ×2
357 > } else if (ch === CharCode.Slash && !inCharacterClass) { // end of regex scanner.ts ×9
358 > p++; scanner.ts ×3
359 > break;
360 > } else if (ch === CharCode.OpenSquareBracket) { scanner.ts ×9
361 > inCharacterClass = true; scanner.ts ×2
362 > } else if (ch === CharCode.Backslash) { scanner.ts ×9
363 > inEscape = true; scanner.ts ×2
364 > } else if (ch === CharCode.CloseSquareBracket) { scanner.ts ×9
365 > inCharacterClass = false; scanner.ts ×2
366 > }
367 > p++; scanner.ts ×9
368 > }
369 > scanner.ts ×3
370 > // Consume flags // TODO@ulugbekna: use regex instead
371 > while (p < this._input.length && Scanner._regexFlags.has(this._input.charCodeAt(p))) { scanner.ts ×9
372 > p++; scanner.ts ×1
373 > }
374 > scanner.ts ×3
375 > this._current = p;
376 >
377 > const lexeme = this._input.substring(this._start, this._current);
378 > this._tokens.push({ type: TokenType.RegexStr, lexeme, offset: this._start });
379 > } scanner.ts ×9
380 > scanner.ts ×36
381 > private _isAtEnd() {
382 > return this._current >= this._input.length; scanner.ts ×11
383 > }
384 > } scanner.ts ×36