Vihren Atlas

tokenization.ts ×28

Code frontier

unlabeled · c_915d89ceb7f5

884 tests · 3948 LOC · 21 files · introduces 0 tests · 134 LOC · 1 file

Introduces — evidence that enters the hierarchy at this concept

Code
28 ranges134 lines · 1 files
Tests
0 tests

Contains — complete concept membership

All code (extent)
573 ranges3948 lines · 21 files · Browse complete extent
All tests (intent)
884 testsBrowse complete intent

Neighbourhood graph

The orange circle is the focus. Violet and green circles are every ancestor and descendant, broader and narrower, at any distance; blue squares and pink diamonds are the introduced files and exact introduced tests of every visible concept, not only the focus's. Arrows point from broader to narrower concepts and bridge only concepts omitted from this view. Undirected links show source or test introduction. Concept and file size follows LOC; exact test nodes use test-count units.

Introduced files, introduced tests, and structurally relevant concept specialization

In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the native relationship evidence on this page.

Graph controls are ready.

Interactive rendering requires JavaScript and WebGL. Use the native relationship evidence on this page while the interactive map is unavailable.

Native relationship evidence

Every exact file and test below is linked only from the concept that introduces it.

Introduced tests

Every collected test enters the hierarchy at exactly one concept.

No tests are introduced at this concept. Its intent tests are introduced by other concepts.

Introduced code

Every collected source range enters the hierarchy at exactly one concept.

1 file ranked by introduced lines: 134 introduced LOC across 28 ranges. Expand a file to inspect source; the > gutter marks introduced lines.

src/vs/editor/common/languages/supports/tokenization.ts 134 introduced LOC · 28 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- tokenization.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { Color } from '../../../../base/common/color.js';
7 > import { IFontTokenOptions } from '../../../../platform/theme/common/themeService.js';
8 > import { LanguageId, FontStyle, ColorId, StandardTokenType, MetadataConsts } from '../../encodedTokenAttributes.js';
9 >
10 > export interface ITokenThemeRule {
11 > token: string;
12 > foreground?: string;
13 > background?: string;
14 > fontStyle?: string;
15 > }
16 >
17 > export class ParsedTokenThemeRule {
18 > _parsedThemeRuleBrand: void = undefined;
19 >
20 > readonly token: string;
21 > readonly index: number;
22 >
23 > /**
24 > * -1 if not set. An or mask of `FontStyle` otherwise.
25 > */
26 > readonly fontStyle: FontStyle;
27 > readonly foreground: string | null;
28 > readonly background: string | null;
29 >
30 > constructor(
31 token: string,
32 index: number,
41 this.background = background;
42 }
43 > } tokenization.ts
44 >
45 > /**
46 > * Parse a raw theme into rules.
47 > */
48 > export function parseTokenTheme(source: ITokenThemeRule[]): ParsedTokenThemeRule[] {
49 if (!source || !Array.isArray(source)) {
50 return [];
100 return result;
101 }
102 > tokenization.ts
103 > /**
104 > * Resolve rules (i.e. inheritance).
105 > */
106 function resolveParsedTokenThemeRules(parsedThemeRules: ParsedTokenThemeRule[], customTokenColors: string[]): TokenTheme {
107
151 return new TokenTheme(colorMap, root);
152 }
153 > tokenization.ts
154 > const colorRegExp = /^#?([0-9A-Fa-f]{6})([0-9A-Fa-f]{2})?$/;
155 >
156 > export class ColorMap {
157 >
158 > private _lastColorId: number;
159 > private readonly _id2color: Color[];
160 > private readonly _color2id: Map<string, ColorId>;
161 >
162 > constructor() {
163 this._lastColorId = 0;
164 this._id2color = [];
165 this._color2id = new Map<string, ColorId>();
166 }
167 > tokenization.ts
168 > public getId(color: string | null): ColorId {
169 if (color === null) {
170 return 0;
184 return value;
185 }
186 > tokenization.ts
187 > public getColorMap(): Color[] {
188 return this._id2color.slice(0);
189 }
190 > tokenization.ts
191 > }
192 >
193 > export class TokenTheme {
194 >
195 > public static createFromRawTokenTheme(source: ITokenThemeRule[], customTokenColors: string[]): TokenTheme {
196 return this.createFromParsedTokenTheme(parseTokenTheme(source), customTokenColors);
197 }
198 > tokenization.ts
199 > public static createFromParsedTokenTheme(source: ParsedTokenThemeRule[], customTokenColors: string[]): TokenTheme {
200 return resolveParsedTokenThemeRules(source, customTokenColors);
201 }
202 > tokenization.ts
203 > private readonly _colorMap: ColorMap;
204 > private readonly _root: ThemeTrieElement;
205 > private readonly _cache: Map<string, number>;
206 >
207 > constructor(colorMap: ColorMap, root: ThemeTrieElement) {
208 this._colorMap = colorMap;
209 this._root = root;
210 this._cache = new Map<string, number>();
211 }
212 > tokenization.ts
213 > public getColorMap(): Color[] {
214 return this._colorMap.getColorMap();
215 }
216 > tokenization.ts
217 > /**
218 > * used for testing purposes
219 > */
220 > public getThemeTrieElement(): ExternalThemeTrieElement {
221 return this._root.toExternalThemeTrieElement();
222 }
223 > tokenization.ts
224 > public _match(token: string): ThemeTrieElementRule {
225 return this._root.match(token);
226 }
227 > tokenization.ts
228 > public match(languageId: LanguageId, token: string): number {
229 // The cache contains the metadata without the language bits set.
230 let result = this._cache.get(token);
244 ) >>> 0;
245 }
246 > } tokenization.ts
247 >
248 > const STANDARD_TOKEN_TYPE_REGEXP = /\b(comment|string|regex|regexp)\b/;
249 > export function toStandardTokenType(tokenType: string): StandardTokenType {
250 const m = tokenType.match(STANDARD_TOKEN_TYPE_REGEXP);
251 if (!m) {
264 throw new Error('Unexpected match for standard token type!');
265 }
266 > tokenization.ts
267 > export function strcmp(a: string, b: string): number {
268 if (a < b) {
269 return -1;
274 return 0;
275 }
276 > tokenization.ts
277 > export class ThemeTrieElementRule {
278 > _themeTrieElementRuleBrand: void = undefined;
279 >
280 > private _fontStyle: FontStyle;
281 > private _foreground: ColorId;
282 > private _background: ColorId;
283 > public metadata: number;
284 >
285 > constructor(fontStyle: FontStyle, foreground: ColorId, background: ColorId) {
286 this._fontStyle = fontStyle;
287 this._foreground = foreground;
293 ) >>> 0;
294 }
295 > tokenization.ts
296 > public clone(): ThemeTrieElementRule {
297 return new ThemeTrieElementRule(this._fontStyle, this._foreground, this._background);
298 }
299 > tokenization.ts
300 > public acceptOverwrite(fontStyle: FontStyle, foreground: ColorId, background: ColorId): void {
301 if (fontStyle !== FontStyle.NotSet) {
302 this._fontStyle = fontStyle;
314 ) >>> 0;
315 }
316 > } tokenization.ts
317 >
318 > export class ExternalThemeTrieElement {
319 >
320 > public readonly mainRule: ThemeTrieElementRule;
321 > public readonly children: Map<string, ExternalThemeTrieElement>;
322 >
323 > constructor(
324 mainRule: ThemeTrieElementRule,
325 children: Map<string, ExternalThemeTrieElement> | { [key: string]: ExternalThemeTrieElement } = new Map<string, ExternalThemeTrieElement>()
335 }
336 }
337 > } tokenization.ts
338 >
339 > export class ThemeTrieElement {
340 > _themeTrieElementBrand: void = undefined;
341 >
342 > private readonly _mainRule: ThemeTrieElementRule;
343 > private readonly _children: Map<string, ThemeTrieElement>;
344 >
345 > constructor(mainRule: ThemeTrieElementRule) {
346 this._mainRule = mainRule;
347 this._children = new Map<string, ThemeTrieElement>();
348 }
349 > tokenization.ts
350 > /**
351 > * used for testing purposes
352 > */
353 > public toExternalThemeTrieElement(): ExternalThemeTrieElement {
354 const children = new Map<string, ExternalThemeTrieElement>();
355 this._children.forEach((element, index) => {
358 return new ExternalThemeTrieElement(this._mainRule, children);
359 }
360 > tokenization.ts
361 > public match(token: string): ThemeTrieElementRule {
362 if (token === '') {
363 return this._mainRule;
382 return this._mainRule;
383 }
384 > tokenization.ts
385 > public insert(token: string, fontStyle: FontStyle, foreground: ColorId, background: ColorId): void {
386 if (token === '') {
387 // Merge into the main rule
409 child.insert(tail, fontStyle, foreground, background);
410 }
411 > } tokenization.ts
412 >
413 > export function generateTokensCSSForColorMap(colorMap: readonly Color[]): string {
414 const rules: string[] = [];
415 for (let i = 1, len = colorMap.length; i < len; i++) {
424 return rules.join('\n');
425 }
426 > tokenization.ts
427 > export function generateTokensCSSForFontMap(fontMap: readonly IFontTokenOptions[]): string {
428 const rules: string[] = [];
429 const fonts = new Set<string>();
450 return rules.join('\n');
451 }
452 > tokenization.ts
453 > export function classNameForFontTokenDecorations(fontFamily: string, fontSize: number): string {
454 const safeFontFamily = sanitizeFontFamilyForClassName(fontFamily);
455 return cleanClassName(`font-decoration-${safeFontFamily}-${fontSize}`);
456 }
457 > tokenization.ts
458 function sanitizeFontFamilyForClassName(fontFamily: string): string {
459 const normalized = fontFamily.toLowerCase().trim();
463 return cleanClassName(normalized);
464 }
465 > tokenization.ts
466 function cleanClassName(className: string): string {
467 return className.replace(/[^a-z0-9_-]/gi, '-');

Direct hierarchy

Parents

Children