textModel.ts ×190

Frontier kind: Code frontier

unlabeled · c_dff62296a684

861 tests · 35636 LOC · 206 files · introduces 0 tests · 2948 LOC · 35 files

Introduces — evidence that enters the hierarchy at this concept

Code
618 ranges2948 lines · 35 files
Tests
0 tests

Contains — complete concept membership

All code (extent)
4021 ranges35636 lines · 206 files · Browse complete extent
All tests (intent)
861 testsBrowse complete intent

Neighbourhood graph

The orange circle is the focus. Violet and green circles are every ancestor and descendant, broader and narrower, at any distance; blue squares and pink diamonds are the introduced files and exact introduced tests of every visible concept, not only the focus's. Arrows point from broader to narrower concepts and bridge only concepts omitted from this view. Undirected links show source or test introduction. Concept and file size follows LOC; exact test nodes use test-count units.

Introduced files, introduced tests, and structurally relevant concept specialization

In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the native relationship evidence on this page.

Graph controls are ready.

Interactive rendering requires JavaScript and WebGL. Use the native relationship evidence on this page while the interactive map is unavailable.

Native relationship evidence

Every exact file and test below is linked only from the concept that introduces it.

Introduced tests

Every collected test enters the hierarchy at exactly one concept.

No tests are introduced at this concept. Its intent tests are introduced by other concepts.

Introduced code

Every collected source range enters the hierarchy at exactly one concept.

Showing the top 20 of 35 files by introduced lines: 2557 of 2948 introduced LOC and 549 of 618 ranges. Expand a file to inspect source; the > gutter marks introduced lines.

src/vs/editor/common/model/textModel.ts 780 introduced LOC · 190 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- textModel.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { pushMany } from '../../../base/common/arrays.js';
7 > import { VSBuffer, VSBufferReadableStream } from '../../../base/common/buffer.js';
8 > import { CharCode } from '../../../base/common/charCode.js';
9 > import { SetWithKey } from '../../../base/common/collections.js';
10 > import { Color } from '../../../base/common/color.js';
11 > import { BugIndicatingError, illegalArgument, onUnexpectedError } from '../../../base/common/errors.js';
12 > import { Emitter, Event } from '../../../base/common/event.js';
13 > import { IMarkdownString } from '../../../base/common/htmlContent.js';
14 > import { Disposable, IDisposable, MutableDisposable } from '../../../base/common/lifecycle.js';
15 > import { listenStream } from '../../../base/common/stream.js';
16 > import * as strings from '../../../base/common/strings.js';
17 > import { ThemeColor } from '../../../base/common/themables.js';
18 > import { Constants } from '../../../base/common/uint.js';
19 > import { URI } from '../../../base/common/uri.js';
20 > import { IInstantiationService } from '../../../platform/instantiation/common/instantiation.js';
21 > import { isDark } from '../../../platform/theme/common/theme.js';
22 > import { IColorTheme } from '../../../platform/theme/common/themeService.js';
23 > import { IUndoRedoService, ResourceEditStackSnapshot, UndoRedoGroup } from '../../../platform/undoRedo/common/undoRedo.js';
24 > import { ISingleEditOperation } from '../core/editOperation.js';
25 > import { TextEdit } from '../core/edits/textEdit.js';
26 > import { countEOL } from '../core/misc/eolCounter.js';
27 > import { normalizeIndentation } from '../core/misc/indentation.js';
28 > import { EDITOR_MODEL_DEFAULTS } from '../core/misc/textModelDefaults.js';
29 > import { IPosition, Position } from '../core/position.js';
30 > import { IRange, Range } from '../core/range.js';
31 > import { Selection } from '../core/selection.js';
32 > import { TextChange } from '../core/textChange.js';
33 > import { IWordAtPosition } from '../core/wordHelper.js';
34 > import { FormattingOptions } from '../languages.js';
35 > import { ILanguageSelection, ILanguageService } from '../languages/language.js';
36 > import { ILanguageConfigurationService } from '../languages/languageConfigurationRegistry.js';
37 > import * as model from '../model.js';
38 > import { IBracketPairsTextModelPart } from '../textModelBracketPairs.js';
39 > import { EditSources, TextModelEditSource } from '../textModelEditSource.js';
40 > import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelOptionsChangedEvent, InternalModelContentChangeEvent, LineInjectedText, ModelFontChanged, ModelFontChangedEvent, ModelInjectedTextChangedEvent, ModelLineHeightChanged, ModelLineHeightChangedEvent, ModelRawChange, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from '../textModelEvents.js';
41 > import { IGuidesTextModelPart } from '../textModelGuides.js';
42 > import { ITokenizationTextModelPart } from '../tokenizationTextModelPart.js';
43 > import { LineTokens, TokenArray } from '../tokens/lineTokens.js';
44 > import { BracketPairsTextModelPart } from './bracketPairsTextModelPart/bracketPairsImpl.js';
45 > import { ColorizedBracketPairsDecorationProvider } from './bracketPairsTextModelPart/colorizedBracketPairsDecorationProvider.js';
46 > import { EditStack } from './editStack.js';
47 > import { GuidesTextModelPart } from './guidesTextModelPart.js';
48 > import { guessIndentation } from './indentationGuesser.js';
49 > import { IntervalNode, IntervalTree, recomputeMaxEnd } from './intervalTree.js';
50 > import { PieceTreeTextBuffer } from './pieceTreeTextBuffer/pieceTreeTextBuffer.js';
51 > import { PieceTreeTextBufferBuilder } from './pieceTreeTextBuffer/pieceTreeTextBufferBuilder.js';
52 > import { SearchParams, TextModelSearch } from './textModelSearch.js';
53 > import { AttachedViews } from './tokens/abstractSyntaxTokenBackend.js';
54 > import { TokenizationFontDecorationProvider } from './tokens/tokenizationFontDecorationsProvider.js';
55 > import { LineFontChangingDecoration, LineHeightChangingDecoration } from './decorationProvider.js';
56 > import { TokenizationTextModelPart } from './tokens/tokenizationTextModelPart.js';
57 > import { IViewModel } from '../viewModel.js';
58 >
59 > export function createTextBufferFactory(text: string): model.ITextBufferFactory {
60 const builder = new PieceTreeTextBufferBuilder();
61 builder.acceptChunk(text);
62 return builder.finish();
63 }
65 > interface ITextStream {
66 > on(event: 'data', callback: (data: string) => void): void;
67 > on(event: 'error', callback: (err: Error) => void): void;
68 > on(event: 'end', callback: () => void): void;
69 > on(event: string, callback: (...args: unknown[]) => void): void;
70 > }
71 >
72 > export function createTextBufferFactoryFromStream(stream: ITextStream): Promise<model.ITextBufferFactory>;
73 > export function createTextBufferFactoryFromStream(stream: VSBufferReadableStream): Promise<model.ITextBufferFactory>;
74 > export function createTextBufferFactoryFromStream(stream: ITextStream | VSBufferReadableStream): Promise<model.ITextBufferFactory> {
75 return new Promise<model.ITextBufferFactory>((resolve, reject) => {
76 const builder = new PieceTreeTextBufferBuilder();
97 });
98 }
100 > export function createTextBufferFactoryFromSnapshot(snapshot: model.ITextSnapshot): model.ITextBufferFactory {
101 const builder = new PieceTreeTextBufferBuilder();
102
108 return builder.finish();
109 }
110 > textModel.ts
111 > export function createTextBuffer(value: string | model.ITextBufferFactory | model.ITextSnapshot, defaultEOL: model.DefaultEndOfLine): { textBuffer: model.ITextBuffer; disposable: IDisposable } {
112 let factory: model.ITextBufferFactory;
113 if (typeof value === 'string') {
120 return factory.create(defaultEOL);
121 }
122 > textModel.ts
123 > let MODEL_ID = 0;
124 >
125 > const LIMIT_FIND_COUNT = 999;
126 > const LONG_LINE_BOUNDARY = 10000;
127 > const LINE_HEIGHT_CEILING = 300;
128 >
129 > class TextModelSnapshot implements model.ITextSnapshot {
130 >
131 > private readonly _source: model.ITextSnapshot;
132 > private _eos: boolean;
133 >
134 > constructor(source: model.ITextSnapshot) {
135 this._source = source;
136 this._eos = false;
137 }
138 > textModel.ts
139 > public read(): string | null {
140 if (this._eos) {
141 return null;
169 } while (true);
170 }
171 > } textModel.ts
172 >
173 > const invalidFunc = () => { throw new Error(`Invalid change accessor`); };
174 >
175 > const enum StringOffsetValidationType {
176 > /**
177 > * Even allowed in surrogate pairs
178 > */
179 > Relaxed = 0,
180 > /**
181 > * Not allowed in surrogate pairs
182 > */
183 > SurrogatePairs = 1,
184 > }
185 >
186 > export class TextModel extends Disposable implements model.ITextModel, IDecorationsTreesHost {
187 >
188 > static _MODEL_SYNC_LIMIT = 50 * 1024 * 1024; // 50 MB, // used in tests
189 > private static readonly LARGE_FILE_SIZE_THRESHOLD = 20 * 1024 * 1024; // 20 MB;
190 > private static readonly LARGE_FILE_LINE_COUNT_THRESHOLD = 300 * 1000; // 300K lines
191 > private static readonly LARGE_FILE_HEAP_OPERATION_THRESHOLD = 256 * 1024 * 1024; // 256M characters, usually ~> 512MB memory usage
192 >
193 > public static DEFAULT_CREATION_OPTIONS: model.ITextModelCreationOptions = {
194 > isForSimpleWidget: false,
195 > tabSize: EDITOR_MODEL_DEFAULTS.tabSize,
196 > indentSize: EDITOR_MODEL_DEFAULTS.indentSize,
197 > insertSpaces: EDITOR_MODEL_DEFAULTS.insertSpaces,
198 > detectIndentation: false,
199 > defaultEOL: model.DefaultEndOfLine.LF,
200 > trimAutoWhitespace: EDITOR_MODEL_DEFAULTS.trimAutoWhitespace,
201 > largeFileOptimizations: EDITOR_MODEL_DEFAULTS.largeFileOptimizations,
202 > bracketPairColorizationOptions: EDITOR_MODEL_DEFAULTS.bracketPairColorizationOptions,
203 > };
204 >
205 > public static resolveOptions(textBuffer: model.ITextBuffer, options: model.ITextModelCreationOptions): model.TextModelResolvedOptions {
206 if (options.detectIndentation) {
207 const guessedIndentation = guessIndentation(textBuffer, options.tabSize, options.insertSpaces);
218 return new model.TextModelResolvedOptions(options);
219 }
220 > textModel.ts
221 > //#region Events
222 > private readonly _onWillDispose: Emitter<void> = this._register(new Emitter<void>());
223 > public readonly onWillDispose: Event<void> = this._onWillDispose.event;
224 >
225 > private readonly _onDidChangeDecorations: DidChangeDecorationsEmitter = this._register(new DidChangeDecorationsEmitter((affectedInjectedTextLines, affectedLineHeights, affectedFontLines) => this.handleBeforeFireDecorationsChangedEvent(affectedInjectedTextLines, affectedLineHeights, affectedFontLines)));
226 > public readonly onDidChangeDecorations: Event<IModelDecorationsChangedEvent> = this._onDidChangeDecorations.event;
227 >
228 > public get onDidChangeLanguage() { return this._tokenizationTextModelPart.onDidChangeLanguage; }
229 > public get onDidChangeLanguageConfiguration() { return this._tokenizationTextModelPart.onDidChangeLanguageConfiguration; }
230 > public get onDidChangeTokens() { return this._tokenizationTextModelPart.onDidChangeTokens; }
231 >
232 > private readonly _onDidChangeOptions: Emitter<IModelOptionsChangedEvent> = this._register(new Emitter<IModelOptionsChangedEvent>());
233 > public get onDidChangeOptions(): Event<IModelOptionsChangedEvent> { return this._onDidChangeOptions.event; }
234 >
235 > private readonly _onDidChangeAttached: Emitter<void> = this._register(new Emitter<void>());
236 > public get onDidChangeAttached(): Event<void> { return this._onDidChangeAttached.event; }
237 >
238 > private readonly _onDidChangeLineHeight: Emitter<ModelLineHeightChangedEvent> = this._register(new Emitter<ModelLineHeightChangedEvent>());
239 > public get onDidChangeLineHeight(): Event<ModelLineHeightChangedEvent> { return this._onDidChangeLineHeight.event; }
240 >
241 > private readonly _onDidChangeFont: Emitter<ModelFontChangedEvent> = this._register(new Emitter<ModelFontChangedEvent>());
242 > public get onDidChangeFont(): Event<ModelFontChangedEvent> { return this._onDidChangeFont.event; }
243 >
244 > private readonly _eventEmitter: DidChangeContentEmitter = this._register(new DidChangeContentEmitter());
245 > public onDidChangeContent(listener: (e: IModelContentChangedEvent) => void): IDisposable {
246 return this._eventEmitter.event((e: InternalModelContentChangeEvent) => listener(e.contentChangedEvent));
247 }
248 > //#endregion textModel.ts
249 >
250 > public readonly id: string;
251 > public readonly isForSimpleWidget: boolean;
252 > private readonly _associatedResource: URI;
253 > private _attachedEditorCount: number;
254 > private _buffer: model.ITextBuffer;
255 > private _bufferDisposable: IDisposable;
256 > private _options: model.TextModelResolvedOptions;
257 > private readonly _languageSelectionListener = this._register(new MutableDisposable<IDisposable>());
258 >
259 > private _isDisposed: boolean;
260 > private __isDisposing: boolean;
261 > public _isDisposing(): boolean { return this.__isDisposing; }
262 > private _versionId: number;
263 > /**
264 > * Unlike, versionId, this can go down (via undo) or go to previous values (via redo)
265 > */
266 > private _alternativeVersionId: number;
267 > private _initialUndoRedoSnapshot: ResourceEditStackSnapshot | null;
268 > private readonly _isTooLargeForSyncing: boolean;
269 > private readonly _isTooLargeForTokenization: boolean;
270 > private readonly _isTooLargeForHeapOperation: boolean;
271 >
272 > //#region Editing
273 > private readonly _commandManager: EditStack;
274 > private _isUndoing: boolean;
275 > private _isRedoing: boolean;
276 > private _trimAutoWhitespaceLines: number[] | null;
277 > //#endregion
278 >
279 > //#region Decorations
280 > /**
281 > * Used to workaround broken clients that might attempt using a decoration id generated by a different model.
282 > * It is not globally unique in order to limit it to one character.
283 > */
284 > private readonly _instanceId: string;
285 > private _deltaDecorationCallCnt: number = 0;
286 > private _lastDecorationId: number;
287 > private _decorations: { [decorationId: string]: IntervalNode };
288 > private _decorationsTree: DecorationsTrees;
289 > private readonly _decorationProvider: ColorizedBracketPairsDecorationProvider;
290 > private readonly _fontTokenDecorationsProvider: TokenizationFontDecorationProvider;
291 > //#endregion
292 >
293 > private readonly _tokenizationTextModelPart: TokenizationTextModelPart;
294 > public get tokenization(): ITokenizationTextModelPart { return this._tokenizationTextModelPart; }
295 >
296 > private readonly _bracketPairs: BracketPairsTextModelPart;
297 > public get bracketPairs(): IBracketPairsTextModelPart { return this._bracketPairs; }
298 >
299 > private readonly _guidesTextModelPart: GuidesTextModelPart;
300 > public get guides(): IGuidesTextModelPart { return this._guidesTextModelPart; }
301 >
302 > private readonly _attachedViews = this._register(new AttachedViews());
303 > private readonly _viewModels = new Set<IViewModel>();
304 >
305 > constructor(
306 source: string | model.ITextBufferFactory,
307 languageIdOrSelection: string | ILanguageSelection,
411 }));
412 }
413 > textModel.ts
414 > public override dispose(): void {
415 this.__isDisposing = true;
416 this._onWillDispose.fire();
427 this._bufferDisposable = Disposable.None;
428 }
429 > textModel.ts
430 > _hasListeners(): boolean {
431 return (
432 this._onWillDispose.hasListeners()
440 );
441 }
442 > textModel.ts
443 > private _assertNotDisposed(): void {
444 if (this._isDisposed) {
445 throw new BugIndicatingError('Model is disposed!');
446 }
447 }
448 > textModel.ts
449 > public registerViewModel(viewModel: IViewModel): void {
450 this._viewModels.add(viewModel);
451 }
452 > textModel.ts
453 > public unregisterViewModel(viewModel: IViewModel): void {
454 this._viewModels.delete(viewModel);
455 }
456 > textModel.ts
457 > public equalsTextBuffer(other: model.ITextBuffer): boolean {
458 this._assertNotDisposed();
459 return this._buffer.equals(other);
460 }
461 > textModel.ts
462 > public getTextBuffer(): model.ITextBuffer {
463 this._assertNotDisposed();
464 return this._buffer;
465 }
466 > textModel.ts
467 > private _emitContentChangedEvent(rawChange: ModelRawContentChangedEvent, change: IModelContentChangedEvent, resultingSelection: Selection[] | null = null): void {
468 if (this.__isDisposing) {
469 // Do not confuse listeners by emitting any event after disposing
481 this._eventEmitter.fire(contentChangeEvent);
482 }
483 > textModel.ts
484 > public setValue(value: string | model.ITextSnapshot, reason = EditSources.setValue()): void {
485 this._assertNotDisposed();
486
492 this._setValueFromTextBuffer(textBuffer, disposable, reason);
493 }
494 > textModel.ts
495 > private _createContentChanged2(range: Range, rangeOffset: number, rangeLength: number, rangeEndPosition: Position, text: string, isUndoing: boolean, isRedoing: boolean, isFlush: boolean, isEolChange: boolean, reason: TextModelEditSource): IModelContentChangedEvent {
496 return {
497 changes: [{
511 };
512 }
513 > textModel.ts
514 > private _setValueFromTextBuffer(textBuffer: model.ITextBuffer, textBufferDisposable: IDisposable, reason: TextModelEditSource): void {
515 this._assertNotDisposed();
516 const oldFullModelRange = this.getFullModelRange();
544 );
545 }
546 > textModel.ts
547 > public setEOL(eol: model.EndOfLineSequence): void {
548 this._assertNotDisposed();
549 const newEOL = (eol === model.EndOfLineSequence.CRLF ? '\r\n' : '\n');
575 );
576 }
577 > textModel.ts
578 > private _onBeforeEOLChange(): void {
579 // Ensure all decorations get their `range` set.
580 this._decorationsTree.ensureAllNodesHaveRanges(this);
581 }
582 > textModel.ts
583 > private _onAfterEOLChange(): void {
584 // Transform back `range` to offsets
585 const versionId = this.getVersionId();
604 }
605 }
606 > textModel.ts
607 > public onBeforeAttached(): model.IAttachedView {
608 this._attachedEditorCount++;
609 if (this._attachedEditorCount === 1) {
613 return this._attachedViews.attachView();
614 }
615 > textModel.ts
616 > public onBeforeDetached(view: model.IAttachedView): void {
617 this._attachedEditorCount--;
618 if (this._attachedEditorCount === 0) {
622 this._attachedViews.detachView(view);
623 }
624 > textModel.ts
625 > public isAttachedToEditor(): boolean {
626 return this._attachedEditorCount > 0;
627 }
628 > textModel.ts
629 > public getAttachedEditorCount(): number {
630 return this._attachedEditorCount;
631 }
632 > textModel.ts
633 > public isTooLargeForSyncing(): boolean {
634 return this._isTooLargeForSyncing;
635 }
636 > textModel.ts
637 > public isTooLargeForTokenization(): boolean {
638 return this._isTooLargeForTokenization;
639 }
640 > textModel.ts
641 > public isTooLargeForHeapOperation(): boolean {
642 return this._isTooLargeForHeapOperation;
643 }
644 > textModel.ts
645 > public isDisposed(): boolean {
646 return this._isDisposed;
647 }
648 > textModel.ts
649 > public isDominatedByLongLines(): boolean {
650 this._assertNotDisposed();
651 if (this.isTooLargeForTokenization()) {
668 return (longLineCharCount > smallLineCharCount);
669 }
670 > textModel.ts
671 > public get uri(): URI {
672 return this._associatedResource;
673 }
674 > textModel.ts
675 > //#region Options
676 >
677 > public getOptions(): model.TextModelResolvedOptions {
678 this._assertNotDisposed();
679 return this._options;
680 }
681 > textModel.ts
682 > public getFormattingOptions(): FormattingOptions {
683 return {
684 tabSize: this._options.indentSize,
686 };
687 }
688 > textModel.ts
689 > public updateOptions(_newOpts: model.ITextModelUpdateOptions): void {
690 this._assertNotDisposed();
691 const tabSize = (typeof _newOpts.tabSize !== 'undefined') ? _newOpts.tabSize : this._options.tabSize;
715 this._onDidChangeOptions.fire(e);
716 }
717 > textModel.ts
718 > public detectIndentation(defaultInsertSpaces: boolean, defaultTabSize: number): void {
719 this._assertNotDisposed();
720 const guessedIndentation = guessIndentation(this._buffer, defaultTabSize, defaultInsertSpaces);
725 });
726 }
727 > textModel.ts
728 > public normalizeIndentation(str: string): string {
729 this._assertNotDisposed();
730 return normalizeIndentation(str, this._options.indentSize, this._options.insertSpaces);
731 }
732 > textModel.ts
733 > //#endregion
734 >
735 > //#region Reading
736 >
737 > public getVersionId(): number {
738 this._assertNotDisposed();
739 return this._versionId;
740 }
741 > textModel.ts
742 > public mightContainRTL(): boolean {
743 return this._buffer.mightContainRTL();
744 }
745 > textModel.ts
746 > public mightContainUnusualLineTerminators(): boolean {
747 return this._buffer.mightContainUnusualLineTerminators();
748 }
749 > textModel.ts
750 > public removeUnusualLineTerminators(selections: Selection[] | null = null): void {
751 const matches = this.findMatches(strings.UNUSUAL_LINE_TERMINATORS.source, false, true, false, null, false, Constants.MAX_SAFE_SMALL_INTEGER);
752 this._buffer.resetMightContainUnusualLineTerminators();
753 this.pushEditOperations(selections, matches.map(m => ({ range: m.range, text: null })), () => null);
754 }
755 > textModel.ts
756 > public mightContainNonBasicASCII(): boolean {
757 return this._buffer.mightContainNonBasicASCII();
758 }
759 > textModel.ts
760 > public getAlternativeVersionId(): number {
761 this._assertNotDisposed();
762 return this._alternativeVersionId;
763 }
764 > textModel.ts
765 > public getInitialUndoRedoSnapshot(): ResourceEditStackSnapshot | null {
766 this._assertNotDisposed();
767 return this._initialUndoRedoSnapshot;
768 }
769 > textModel.ts
770 > public getOffsetAt(rawPosition: IPosition): number {
771 this._assertNotDisposed();
772 const position = this._validatePosition(rawPosition.lineNumber, rawPosition.column, StringOffsetValidationType.Relaxed);
773 return this._buffer.getOffsetAt(position.lineNumber, position.column);
774 }
775 > textModel.ts
776 > public getPositionAt(rawOffset: number): Position {
777 this._assertNotDisposed();
778 const offset = (Math.min(this._buffer.getLength(), Math.max(0, rawOffset)));
779 return this._buffer.getPositionAt(offset);
780 }
781 > textModel.ts
782 > private _increaseVersionId(): void {
783 this._versionId = this._versionId + 1;
784 this._alternativeVersionId = this._versionId;
785 }
786 > textModel.ts
787 > public _overwriteVersionId(versionId: number): void {
788 this._versionId = versionId;
789 }
790 > textModel.ts
791 > public _overwriteAlternativeVersionId(newAlternativeVersionId: number): void {
792 this._alternativeVersionId = newAlternativeVersionId;
793 }
794 > textModel.ts
795 > public _overwriteInitialUndoRedoSnapshot(newInitialUndoRedoSnapshot: ResourceEditStackSnapshot | null): void {
796 this._initialUndoRedoSnapshot = newInitialUndoRedoSnapshot;
797 }
798 > textModel.ts
799 > public getValue(eol?: model.EndOfLinePreference, preserveBOM: boolean = false): string {
800 this._assertNotDisposed();
801 if (this.isTooLargeForHeapOperation()) {
812 return fullModelValue;
813 }
814 > textModel.ts
815 > public createSnapshot(preserveBOM: boolean = false): model.ITextSnapshot {
816 return new TextModelSnapshot(this._buffer.createSnapshot(preserveBOM));
817 }
818 > textModel.ts
819 > public getValueLength(eol?: model.EndOfLinePreference, preserveBOM: boolean = false): number {
820 this._assertNotDisposed();
821 const fullModelRange = this.getFullModelRange();
828 return fullModelValue;
829 }
830 > textModel.ts
831 > public getValueInRange(rawRange: IRange, eol: model.EndOfLinePreference = model.EndOfLinePreference.TextDefined): string {
832 this._assertNotDisposed();
833 return this._buffer.getValueInRange(this.validateRange(rawRange), eol);
834 }
835 > textModel.ts
836 > public getValueLengthInRange(rawRange: IRange, eol: model.EndOfLinePreference = model.EndOfLinePreference.TextDefined): number {
837 this._assertNotDisposed();
838 return this._buffer.getValueLengthInRange(this.validateRange(rawRange), eol);
839 }
840 > textModel.ts
841 > public getCharacterCountInRange(rawRange: IRange, eol: model.EndOfLinePreference = model.EndOfLinePreference.TextDefined): number {
842 this._assertNotDisposed();
843 return this._buffer.getCharacterCountInRange(this.validateRange(rawRange), eol);
844 }
845 > textModel.ts
846 > public getLineCount(): number {
847 this._assertNotDisposed();
848 return this._buffer.getLineCount();
849 }
850 > textModel.ts
851 > public getLineContent(lineNumber: number): string {
852 this._assertNotDisposed();
853 if (lineNumber < 1 || lineNumber > this.getLineCount()) {
857 return this._buffer.getLineContent(lineNumber);
858 }
859 > textModel.ts
860 > public getLineLength(lineNumber: number): number {
861 this._assertNotDisposed();
862 if (lineNumber < 1 || lineNumber > this.getLineCount()) {
866 return this._buffer.getLineLength(lineNumber);
867 }
868 > textModel.ts
869 > public getLinesContent(): string[] {
870 this._assertNotDisposed();
871 if (this.isTooLargeForHeapOperation()) {
875 return this._buffer.getLinesContent();
876 }
877 > textModel.ts
878 > public getEOL(): string {
879 this._assertNotDisposed();
880 return this._buffer.getEOL();
881 }
882 > textModel.ts
883 > public getEndOfLineSequence(): model.EndOfLineSequence {
884 this._assertNotDisposed();
885 return (
889 );
890 }
891 > textModel.ts
892 > public getLineMinColumn(lineNumber: number): number {
893 this._assertNotDisposed();
894 return 1;
895 }
896 > textModel.ts
897 > public getLineMaxColumn(lineNumber: number): number {
898 this._assertNotDisposed();
899 if (lineNumber < 1 || lineNumber > this.getLineCount()) {
902 return this._buffer.getLineLength(lineNumber) + 1;
903 }
904 > textModel.ts
905 > public getLineFirstNonWhitespaceColumn(lineNumber: number): number {
906 this._assertNotDisposed();
907 if (lineNumber < 1 || lineNumber > this.getLineCount()) {
910 return this._buffer.getLineFirstNonWhitespaceColumn(lineNumber);
911 }
912 > textModel.ts
913 > public getLineLastNonWhitespaceColumn(lineNumber: number): number {
914 this._assertNotDisposed();
915 if (lineNumber < 1 || lineNumber > this.getLineCount()) {
918 return this._buffer.getLineLastNonWhitespaceColumn(lineNumber);
919 }
920 > textModel.ts
921 > /**
922 > * Validates `range` is within buffer bounds, but allows it to sit in between surrogate pairs, etc.
923 > * Will try to not allocate if possible.
924 > */
925 > public _validateRangeRelaxedNoAllocations(range: IRange): Range {
926 const linesCount = this._buffer.getLineCount();
927
983 return new Range(startLineNumber, startColumn, endLineNumber, endColumn);
984 }
985 > textModel.ts
986 > private _isValidPosition(lineNumber: number, column: number, validationType: StringOffsetValidationType): boolean {
987 if (typeof lineNumber !== 'number' || typeof column !== 'number') {
988 return false;
1025 return true;
1026 }
1027 > textModel.ts
1028 > private _validatePosition(_lineNumber: number, _column: number, validationType: StringOffsetValidationType): Position {
1029 const lineNumber = Math.floor((typeof _lineNumber === 'number' && !isNaN(_lineNumber)) ? _lineNumber : 1);
1030 const column = Math.floor((typeof _column === 'number' && !isNaN(_column)) ? _column : 1);
1060 return new Position(lineNumber, column);
1061 }
1062 > textModel.ts
1063 > public validatePosition(position: IPosition): Position {
1064 const validationType = StringOffsetValidationType.SurrogatePairs;
1065 this._assertNotDisposed();
1074 return this._validatePosition(position.lineNumber, position.column, validationType);
1075 }
1076 > textModel.ts
1077 > public isValidRange(range: Range): boolean {
1078 return this._isValidRange(range, StringOffsetValidationType.SurrogatePairs);
1079 }
1080 > textModel.ts
1081 > private _isValidRange(range: Range, validationType: StringOffsetValidationType): boolean {
1082 const startLineNumber = range.startLineNumber;
1083 const startColumn = range.startColumn;
1107 return true;
1108 }
1109 > textModel.ts
1110 > public validateRange(_range: IRange): Range {
1111 const validationType = StringOffsetValidationType.SurrogatePairs;
1112 this._assertNotDisposed();
1159 return new Range(startLineNumber, startColumn, endLineNumber, endColumn);
1160 }
1161 > textModel.ts
1162 > public modifyPosition(rawPosition: IPosition, offset: number): Position {
1163 this._assertNotDisposed();
1164 const candidate = this.getOffsetAt(rawPosition) + offset;
1165 return this.getPositionAt(Math.min(this._buffer.getLength(), Math.max(0, candidate)));
1166 }
1167 > textModel.ts
1168 > public getFullModelRange(): Range {
1169 this._assertNotDisposed();
1170 const lineCount = this.getLineCount();
1171 return new Range(1, 1, lineCount, this.getLineMaxColumn(lineCount));
1172 }
1173 > textModel.ts
1174 > private findMatchesLineByLine(searchRange: Range, searchData: model.SearchData, captureMatches: boolean, limitResultCount: number): model.FindMatch[] {
1175 return this._buffer.findMatchesLineByLine(searchRange, searchData, captureMatches, limitResultCount);
1176 }
1177 > textModel.ts
1178 > public findMatches(searchString: string, rawSearchScope: boolean | IRange | IRange[] | null, isRegex: boolean, matchCase: boolean, wordSeparators: string | null, captureMatches: boolean, limitResultCount: number = LIMIT_FIND_COUNT): model.FindMatch[] {
1179 this._assertNotDisposed();
1180
1224 return uniqueSearchRanges.map(matchMapper).reduce((arr, matches: model.FindMatch[]) => arr.concat(matches), []);
1225 }
1226 > textModel.ts
1227 > public findNextMatch(searchString: string, rawSearchStart: IPosition, isRegex: boolean, matchCase: boolean, wordSeparators: string, captureMatches: boolean): model.FindMatch | null {
1228 this._assertNotDisposed();
1229 const searchStart = this.validatePosition(rawSearchStart);
1256 return TextModelSearch.findNextMatch(this, new SearchParams(searchString, isRegex, matchCase, wordSeparators), searchStart, captureMatches);
1257 }
1258 > textModel.ts
1259 > public findPreviousMatch(searchString: string, rawSearchStart: IPosition, isRegex: boolean, matchCase: boolean, wordSeparators: string, captureMatches: boolean): model.FindMatch | null {
1260 this._assertNotDisposed();
1261 const searchStart = this.validatePosition(rawSearchStart);
1262 return TextModelSearch.findPreviousMatch(this, new SearchParams(searchString, isRegex, matchCase, wordSeparators), searchStart, captureMatches);
1263 }
1264 > textModel.ts
1265 > //#endregion
1266 >
1267 > //#region Editing
1268 >
1269 > public pushStackElement(): void {
1270 this._commandManager.pushStackElement();
1271 }
1272 > textModel.ts
1273 > public popStackElement(): void {
1274 this._commandManager.popStackElement();
1275 }
1276 > textModel.ts
1277 > public pushEOL(eol: model.EndOfLineSequence): void {
1278 const currentEOL = (this.getEOL() === '\n' ? model.EndOfLineSequence.LF : model.EndOfLineSequence.CRLF);
1279 if (currentEOL === eol) {
1292 }
1293 }
1294 > textModel.ts
1295 > private _validateEditOperation(rawOperation: model.IIdentifiedSingleEditOperation): model.ValidAnnotatedEditOperation {
1296 if (rawOperation instanceof model.ValidAnnotatedEditOperation) {
1297 return rawOperation;
1325 );
1326 }
1327 > textModel.ts
1328 > private _validateEditOperations(rawOperations: readonly model.IIdentifiedSingleEditOperation[]): model.ValidAnnotatedEditOperation[] {
1329 const result: model.ValidAnnotatedEditOperation[] = [];
1330 for (let i = 0, len = rawOperations.length; i < len; i++) {
1333 return result;
1334 }
1335 > textModel.ts
1336 > public edit(edit: TextEdit, options?: { reason?: TextModelEditSource }): void {
1337 this.pushEditOperations(null, edit.replacements.map(r => ({ range: r.range, text: r.text })), null);
1338 }
1339 > textModel.ts
1340 > public pushEditOperations(beforeCursorState: Selection[] | null, editOperations: model.IIdentifiedSingleEditOperation[], cursorStateComputer: model.ICursorStateComputer | null, group?: UndoRedoGroup, reason?: TextModelEditSource): Selection[] | null {
1341 try {
1342 this._onDidChangeDecorations.beginDeferredEmit();
1348 }
1349 }
1350 > textModel.ts
1351 > private _pushEditOperations(beforeCursorState: Selection[] | null, editOperations: model.ValidAnnotatedEditOperation[], cursorStateComputer: model.ICursorStateComputer | null, group?: UndoRedoGroup, reason?: TextModelEditSource): Selection[] | null {
1352 if (this._options.trimAutoWhitespace && this._trimAutoWhitespaceLines) {
1353 // Go through each saved line number and insert a trim whitespace edit
1438 return this._commandManager.pushEditOperation(beforeCursorState, editOperations, cursorStateComputer, group, reason);
1439 }
1440 > textModel.ts
1441 > _applyUndo(changes: TextChange[], eol: model.EndOfLineSequence, resultingAlternativeVersionId: number, resultingSelection: Selection[] | null): void {
1442 const edits = changes.map<ISingleEditOperation>((change) => {
1443 const rangeStart = this.getPositionAt(change.newPosition);
1450 this._applyUndoRedoEdits(edits, eol, true, false, resultingAlternativeVersionId, resultingSelection);
1451 }
1452 > textModel.ts
1453 > _applyRedo(changes: TextChange[], eol: model.EndOfLineSequence, resultingAlternativeVersionId: number, resultingSelection: Selection[] | null): void {
1454 const edits = changes.map<ISingleEditOperation>((change) => {
1455 const rangeStart = this.getPositionAt(change.oldPosition);
1462 this._applyUndoRedoEdits(edits, eol, false, true, resultingAlternativeVersionId, resultingSelection);
1463 }
1464 > textModel.ts
1465 > private _applyUndoRedoEdits(edits: ISingleEditOperation[], eol: model.EndOfLineSequence, isUndoing: boolean, isRedoing: boolean, resultingAlternativeVersionId: number, resultingSelection: Selection[] | null): void {
1466 try {
1467 this._onDidChangeDecorations.beginDeferredEmit();
1480 }
1481 }
1482 > textModel.ts
1483 > public applyEdits(operations: readonly model.IIdentifiedSingleEditOperation[]): void;
1484 > public applyEdits(operations: readonly model.IIdentifiedSingleEditOperation[], computeUndoEdits: false): void;
1485 > public applyEdits(operations: readonly model.IIdentifiedSingleEditOperation[], computeUndoEdits: true): model.IValidEditOperation[];
1486 > /** @internal */
1487 > public applyEdits(operations: readonly model.IIdentifiedSingleEditOperation[], computeUndoEdits: false, reason: TextModelEditSource): void;
1488 > /** @internal */
1489 > public applyEdits(operations: readonly model.IIdentifiedSingleEditOperation[], computeUndoEdits: true, reason: TextModelEditSource): model.IValidEditOperation[];
1490 > public applyEdits(rawOperations: readonly model.IIdentifiedSingleEditOperation[], computeUndoEdits?: boolean, reason?: TextModelEditSource): void | model.IValidEditOperation[] {
1491 try {
1492 this._onDidChangeDecorations.beginDeferredEmit();
1500 }
1501 }
1502 > textModel.ts
1503 > private _doApplyEdits(rawOperations: model.ValidAnnotatedEditOperation[], computeUndoEdits: boolean, reason: TextModelEditSource, resultingSelection: Selection[] | null = null): void | model.IValidEditOperation[] {
1504
1505 const oldLineCount = this._buffer.getLineCount();
1601 return (result.reverseEdits === null ? undefined : result.reverseEdits);
1602 }
1603 > textModel.ts
1604 > public undo(): void | Promise<void> {
1605 return this._undoRedoService.undo(this.uri);
1606 }
1607 > textModel.ts
1608 > public canUndo(): boolean {
1609 return this._undoRedoService.canUndo(this.uri);
1610 }
1611 > textModel.ts
1612 > public redo(): void | Promise<void> {
1613 return this._undoRedoService.redo(this.uri);
1614 }
1615 > textModel.ts
1616 > public canRedo(): boolean {
1617 return this._undoRedoService.canRedo(this.uri);
1618 }
1619 > textModel.ts
1620 > //#endregion
1621 >
1622 > //#region Decorations
1623 >
1624 > private handleBeforeFireDecorationsChangedEvent(affectedInjectedTextLines: Set<number> | null, affectedLineHeights: Set<LineHeightChangingDecoration> | null, affectedFontLines: Set<LineFontChangingDecoration> | null): void {
1625 // This is called before the decoration changed event is fired.
1626
1633 this._fireOnDidChangeFont(affectedFontLines);
1634 }
1635 > textModel.ts
1636 > private _fireOnDidChangeLineHeight(affectedLineHeights: Set<LineHeightChangingDecoration> | null): void {
1637 if (affectedLineHeights && affectedLineHeights.size > 0) {
1638 const affectedLines = Array.from(affectedLineHeights);
1641 }
1642 }
1643 > textModel.ts
1644 > private _fireOnDidChangeFont(affectedFontLines: Set<LineFontChangingDecoration> | null): void {
1645 if (affectedFontLines && affectedFontLines.size > 0) {
1646 const affectedLines = Array.from(affectedFontLines);
1649 }
1650 }
1651 > textModel.ts
1652 > private _onDidChangeContentOrInjectedText(e: InternalModelContentChangeEvent | ModelInjectedTextChangedEvent): void {
1653 for (const viewModel of this._viewModels) {
1654 try {
1666 }
1667 }
1668 > textModel.ts
1669 > public changeDecorations<T>(callback: (changeAccessor: model.IModelDecorationsChangeAccessor) => T, ownerId: number = 0): T | null {
1670 this._assertNotDisposed();
1671
1677 }
1678 }
1679 > textModel.ts
1680 > private _changeDecorations<T>(ownerId: number, callback: (changeAccessor: model.IModelDecorationsChangeAccessor) => T): T | null {
1681 const changeAccessor: model.IModelDecorationsChangeAccessor = {
1682 addDecoration: (range: IRange, options: model.IModelDecorationOptions): string => {
1714 return result;
1715 }
1716 > textModel.ts
1717 > public deltaDecorations(oldDecorations: string[], newDecorations: model.IModelDeltaDecoration[], ownerId: number = 0): string[] {
1718 this._assertNotDisposed();
1719 if (!oldDecorations) {
1738 }
1739 }
1740 > textModel.ts
1741 > _getTrackedRange(id: string): Range | null {
1742 return this.getDecorationRange(id);
1743 }
1744 > textModel.ts
1745 > _setTrackedRange(id: string | null, newRange: null, newStickiness: model.TrackedRangeStickiness): null;
1746 > _setTrackedRange(id: string | null, newRange: Range, newStickiness: model.TrackedRangeStickiness): string;
1747 > _setTrackedRange(id: string | null, newRange: Range | null, newStickiness: model.TrackedRangeStickiness): string | null {
1748 const node = (id ? this._decorations[id] : null);
1749
1774 return node.id;
1775 }
1776 > textModel.ts
1777 > public removeAllDecorationsWithOwnerId(ownerId: number): void {
1778 if (this._isDisposed) {
1779 return;
1787 }
1788 }
1789 > textModel.ts
1790 > public getDecorationOptions(decorationId: string): model.IModelDecorationOptions | null {
1791 const node = this._decorations[decorationId];
1792 if (!node) {
1795 return node.options;
1796 }
1797 > textModel.ts
1798 > public getDecorationRange(decorationId: string): Range | null {
1799 const node = this._decorations[decorationId];
1800 if (!node) {
1803 return this._decorationsTree.getNodeRange(this, node);
1804 }
1805 > textModel.ts
1806 > public getLineDecorations(lineNumber: number, ownerId: number = 0, filterOutValidation: boolean = false, filterFontDecorations: boolean = false): model.IModelDecoration[] {
1807 if (lineNumber < 1 || lineNumber > this.getLineCount()) {
1808 return [];
1810 return this.getLinesDecorations(lineNumber, lineNumber, ownerId, filterOutValidation, filterFontDecorations);
1811 }
1812 > textModel.ts
1813 > public getLinesDecorations(_startLineNumber: number, _endLineNumber: number, ownerId: number = 0, filterOutValidation: boolean = false, filterFontDecorations: boolean = false, onlyMarginDecorations: boolean = false): model.IModelDecoration[] {
1814 const lineCount = this.getLineCount();
1815 const startLineNumber = Math.min(lineCount, Math.max(1, _startLineNumber));
1823 return decorations;
1824 }
1825 > textModel.ts
1826 > public getDecorationsInRange(range: IRange, ownerId: number = 0, filterOutValidation: boolean = false, filterFontDecorations: boolean = false, onlyMinimapDecorations: boolean = false, onlyMarginDecorations: boolean = false): model.IModelDecoration[] {
1827 const validatedRange = this.validateRange(range);
1828
1832 return decorations;
1833 }
1834 > textModel.ts
1835 > public getOverviewRulerDecorations(ownerId: number = 0, filterOutValidation: boolean = false, filterFontDecorations: boolean = false): model.IModelDecoration[] {
1836 return this._decorationsTree.getAll(this, ownerId, filterOutValidation, filterFontDecorations, true, false);
1837 }
1838 > textModel.ts
1839 > public getInjectedTextDecorations(ownerId: number = 0): model.IModelDecoration[] {
1840 return this._decorationsTree.getAllInjectedText(this, ownerId);
1841 }
1842 > textModel.ts
1843 > public getCustomLineHeightsDecorations(ownerId: number = 0): model.IModelDecoration[] {
1844 const decs = this._decorationsTree.getAllCustomLineHeights(this, ownerId);
1845 pushMany(decs, this._fontTokenDecorationsProvider.getAllDecorations(ownerId));
1846 return decs;
1847 }
1848 > textModel.ts
1849 > public getCustomLineHeightsDecorationsInRange(range: Range, ownerId: number = 0): model.IModelDecoration[] {
1850 const decs = this._decorationsTree.getCustomLineHeightsInInterval(this, this.getOffsetAt(range.getStartPosition()), this.getOffsetAt(range.getEndPosition()), ownerId);
1851 pushMany(decs, this._fontTokenDecorationsProvider.getDecorationsInRange(range, ownerId));
1852 return decs;
1853 }
1854 > textModel.ts
1855 > public getLineInjectedText(lineNumber: number, ownerId: number = 0): LineInjectedText[] {
1856 const startOffset = this._buffer.getOffsetAt(lineNumber, 1);
1857 const endOffset = startOffset + this._buffer.getLineLength(lineNumber);
1860 return LineInjectedText.fromDecorations(result).filter(t => t.lineNumber === lineNumber);
1861 }
1862 > textModel.ts
1863 > public getFontDecorationsInRange(range: IRange, ownerId: number = 0): model.IModelDecoration[] {
1864 const startOffset = this._buffer.getOffsetAt(range.startLineNumber, range.startColumn);
1865 const endOffset = this._buffer.getOffsetAt(range.endLineNumber, range.endColumn);
1866 return this._decorationsTree.getFontDecorationsInInterval(this, startOffset, endOffset, ownerId);
1867 }
1868 > textModel.ts
1869 > public getAllDecorations(ownerId: number = 0, filterOutValidation: boolean = false, filterFontDecorations: boolean = false): model.IModelDecoration[] {
1870 let result = this._decorationsTree.getAll(this, ownerId, filterOutValidation, filterFontDecorations, false, false);
1871 result = result.concat(this._decorationProvider.getAllDecorations(ownerId, filterOutValidation));
1873 return result;
1874 }
1875 > textModel.ts
1876 > public getAllMarginDecorations(ownerId: number = 0): model.IModelDecoration[] {
1877 return this._decorationsTree.getAll(this, ownerId, false, false, false, true);
1878 }
1879 > textModel.ts
1880 > private _getDecorationsInRange(filterRange: Range, filterOwnerId: number, filterOutValidation: boolean, filterFontDecorations: boolean, onlyMarginDecorations: boolean): model.IModelDecoration[] {
1881 const startOffset = this._buffer.getOffsetAt(filterRange.startLineNumber, filterRange.startColumn);
1882 const endOffset = this._buffer.getOffsetAt(filterRange.endLineNumber, filterRange.endColumn);
1883 return this._decorationsTree.getAllInInterval(this, startOffset, endOffset, filterOwnerId, filterOutValidation, filterFontDecorations, onlyMarginDecorations);
1884 }
1885 > textModel.ts
1886 > public getRangeAt(start: number, end: number): Range {
1887 return this._buffer.getRangeAt(start, end - start);
1888 }
1889 > textModel.ts
1890 > private _changeDecorationImpl(ownerId: number, decorationId: string, _range: IRange): void {
1891 const node = this._decorations[decorationId];
1892 if (!node) {
1933 }
1934 }
1935 > textModel.ts
1936 > private _changeDecorationOptionsImpl(ownerId: number, decorationId: string, options: ModelDecorationOptions): void {
1937 const node = this._decorations[decorationId];
1938 if (!node) {
1973 }
1974 }
1975 > textModel.ts
1976 > private _deltaDecorationsImpl(ownerId: number, oldDecorationsIds: string[], newDecorations: model.IModelDeltaDecoration[], suppressEvents: boolean = false): string[] {
1977 const versionId = this.getVersionId();
1978
2077 }
2078 }
2079 > textModel.ts
2080 > //#endregion
2081 >
2082 > //#region Tokenization
2083 >
2084 > // TODO move them to the tokenization part.
2085 > public getLanguageId(): string {
2086 return this.tokenization.getLanguageId();
2087 }
2088 > textModel.ts
2089 > public setLanguage(languageIdOrSelection: string | ILanguageSelection, source?: string): void {
2090 if (typeof languageIdOrSelection === 'string') {
2091 this._languageSelectionListener.clear();
2096 }
2097 }
2098 > textModel.ts
2099 > private _setLanguage(languageId: string, source?: string): void {
2100 this.tokenization.setLanguageId(languageId, source);
2101 this._languageService.requestRichLanguageFeatures(languageId);
2102 }
2103 > textModel.ts
2104 > public getLanguageIdAtPosition(lineNumber: number, column: number): string {
2105 return this.tokenization.getLanguageIdAtPosition(lineNumber, column);
2106 }
2107 > textModel.ts
2108 > public getWordAtPosition(position: IPosition): IWordAtPosition | null {
2109 return this._tokenizationTextModelPart.getWordAtPosition(position);
2110 }
2111 > textModel.ts
2112 > public getWordUntilPosition(position: IPosition): IWordAtPosition {
2113 return this._tokenizationTextModelPart.getWordUntilPosition(position);
2114 }
2115 > textModel.ts
2116 > //#endregion
2117 > normalizePosition(position: Position, affinity: model.PositionAffinity): Position {
2118 return position;
2119 }
2120 > textModel.ts
2121 > /**
2122 > * Gets the column at which indentation stops at a given line.
2123 > * @internal
2124 > */
2125 > public getLineIndentColumn(lineNumber: number): number {
2126 // Columns start with 1.
2127 return indentOfLine(this.getLineContent(lineNumber)) + 1;
2128 }
2129 > textModel.ts
2130 > public override toString(): string {
2131 return `TextModel(${this.uri.toString()})`;
2132 }
2133 > } textModel.ts
2134 >
2135 > export function getLineTokensWithInjections(tokens: LineTokens, injectionOptions: model.InjectedTextOptions[] | null, injectionOffsets: number[] | null): LineTokens {
2136 let lineTokens: LineTokens;
2137 if (injectionOffsets) {
2163 return lineTokens;
2164 }
2165 > textModel.ts
2166 > export function indentOfLine(line: string): number {
2167 let indent = 0;
2168 for (const c of line) {
2175 return indent;
2176 }
2177 > textModel.ts
2178 > //#region Decorations
2179 >
2180 function isNodeInOverviewRuler(node: IntervalNode): boolean {
2181 return (node.options.overviewRuler && node.options.overviewRuler.color ? true : false);
2182 }
2183 > textModel.ts
2184 function isOptionsInjectedText(options: ModelDecorationOptions): boolean {
2185 return !!options.after || !!options.before;
2186 }
2187 > textModel.ts
2188 function isNodeInjectedText(node: IntervalNode): boolean {
2189 return !!node.options.after || !!node.options.before;
2190 }
2191 > textModel.ts
2192 > export interface IDecorationsTreesHost {
2193 > getVersionId(): number;
2194 > getRangeAt(start: number, end: number): Range;
2195 > }
2196 >
2197 > class DecorationsTrees {
2198 >
2199 > /**
2200 > * This tree holds decorations that do not show up in the overview ruler.
2201 > */
2202 > private readonly _decorationsTree0: IntervalTree;
2203 >
2204 > /**
2205 > * This tree holds decorations that show up in the overview ruler.
2206 > */
2207 > private readonly _decorationsTree1: IntervalTree;
2208 >
2209 > /**
2210 > * This tree holds decorations that contain injected text.
2211 > */
2212 > private readonly _injectedTextDecorationsTree: IntervalTree;
2213 >
2214 > constructor() {
2215 this._decorationsTree0 = new IntervalTree();
2216 this._decorationsTree1 = new IntervalTree();
2217 this._injectedTextDecorationsTree = new IntervalTree();
2218 }
2219 > textModel.ts
2220 > public ensureAllNodesHaveRanges(host: IDecorationsTreesHost): void {
2221 this.getAll(host, 0, false, false, false, false);
2222 }
2223 > textModel.ts
2224 > private _ensureNodesHaveRanges(host: IDecorationsTreesHost, nodes: IntervalNode[]): model.IModelDecoration[] {
2225 for (const node of nodes) {
2226 if (node.range === null) {
2230 return <model.IModelDecoration[]>nodes;
2231 }
2232 > textModel.ts
2233 > public getAllInInterval(host: IDecorationsTreesHost, start: number, end: number, filterOwnerId: number, filterOutValidation: boolean, filterFontDecorations: boolean, onlyMarginDecorations: boolean): model.IModelDecoration[] {
2234 const versionId = host.getVersionId();
2235 const result = this._intervalSearch(start, end, filterOwnerId, filterOutValidation, filterFontDecorations, versionId, onlyMarginDecorations);
2236 return this._ensureNodesHaveRanges(host, result);
2237 }
2238 > textModel.ts
2239 > private _intervalSearch(start: number, end: number, filterOwnerId: number, filterOutValidation: boolean, filterFontDecorations: boolean, cachedVersionId: number, onlyMarginDecorations: boolean): IntervalNode[] {
2240 const r0 = this._decorationsTree0.intervalSearch(start, end, filterOwnerId, filterOutValidation, filterFontDecorations, cachedVersionId, onlyMarginDecorations);
2241 const r1 = this._decorationsTree1.intervalSearch(start, end, filterOwnerId, filterOutValidation, filterFontDecorations, cachedVersionId, onlyMarginDecorations);
2243 return r0.concat(r1).concat(r2);
2244 }
2245 > textModel.ts
2246 > public getInjectedTextInInterval(host: IDecorationsTreesHost, start: number, end: number, filterOwnerId: number): model.IModelDecoration[] {
2247 const versionId = host.getVersionId();
2248 const result = this._injectedTextDecorationsTree.intervalSearch(start, end, filterOwnerId, false, false, versionId, false);
2249 return this._ensureNodesHaveRanges(host, result).filter((i) => i.options.showIfCollapsed || !i.range.isEmpty());
2250 }
2251 > textModel.ts
2252 > public getFontDecorationsInInterval(host: IDecorationsTreesHost, start: number, end: number, filterOwnerId: number): model.IModelDecoration[] {
2253 const versionId = host.getVersionId();
2254 const decorations = this._decorationsTree0.intervalSearch(start, end, filterOwnerId, false, false, versionId, false);
2255 return this._ensureNodesHaveRanges(host, decorations).filter((i) => i.options.affectsFont);
2256 }
2257 > textModel.ts
2258 > public getAllInjectedText(host: IDecorationsTreesHost, filterOwnerId: number): model.IModelDecoration[] {
2259 const versionId = host.getVersionId();
2260 const result = this._injectedTextDecorationsTree.search(filterOwnerId, false, false, versionId, false);
2261 return this._ensureNodesHaveRanges(host, result).filter((i) => i.options.showIfCollapsed || !i.range.isEmpty());
2262 }
2263 > textModel.ts
2264 > public getAllCustomLineHeights(host: IDecorationsTreesHost, filterOwnerId: number): model.IModelDecoration[] {
2265 const versionId = host.getVersionId();
2266 const result = this._search(filterOwnerId, false, false, false, versionId, false);
2267 return this._ensureNodesHaveRanges(host, result).filter((i) => typeof i.options.lineHeight === 'number');
2268 }
2269 > textModel.ts
2270 > public getCustomLineHeightsInInterval(host: IDecorationsTreesHost, start: number, end: number, filterOwnerId: number): model.IModelDecoration[] {
2271 const versionId = host.getVersionId();
2272 const result = this._intervalSearch(start, end, filterOwnerId, false, false, versionId, false);
2273 return this._ensureNodesHaveRanges(host, result).filter((i) => typeof i.options.lineHeight === 'number');
2274 }
2275 > textModel.ts
2276 > public getAll(host: IDecorationsTreesHost, filterOwnerId: number, filterOutValidation: boolean, filterFontDecorations: boolean, overviewRulerOnly: boolean, onlyMarginDecorations: boolean): model.IModelDecoration[] {
2277 const versionId = host.getVersionId();
2278 const result = this._search(filterOwnerId, filterOutValidation, filterFontDecorations, overviewRulerOnly, versionId, onlyMarginDecorations);
2279 return this._ensureNodesHaveRanges(host, result);
2280 }
2281 > textModel.ts
2282 > private _search(filterOwnerId: number, filterOutValidation: boolean, filterFontDecorations: boolean, overviewRulerOnly: boolean, cachedVersionId: number, onlyMarginDecorations: boolean): IntervalNode[] {
2283 if (overviewRulerOnly) {
2284 return this._decorationsTree1.search(filterOwnerId, filterOutValidation, filterFontDecorations, cachedVersionId, onlyMarginDecorations);
2290 }
2291 }
2292 > textModel.ts
2293 > public collectNodesFromOwner(ownerId: number): IntervalNode[] {
2294 const r0 = this._decorationsTree0.collectNodesFromOwner(ownerId);
2295 const r1 = this._decorationsTree1.collectNodesFromOwner(ownerId);
2297 return r0.concat(r1).concat(r2);
2298 }
2299 > textModel.ts
2300 > public collectNodesPostOrder(): IntervalNode[] {
2301 const r0 = this._decorationsTree0.collectNodesPostOrder();
2302 const r1 = this._decorationsTree1.collectNodesPostOrder();
2304 return r0.concat(r1).concat(r2);
2305 }
2306 > textModel.ts
2307 > public insert(node: IntervalNode): void {
2308 if (isNodeInjectedText(node)) {
2309 this._injectedTextDecorationsTree.insert(node);
2314 }
2315 }
2316 > textModel.ts
2317 > public delete(node: IntervalNode): void {
2318 if (isNodeInjectedText(node)) {
2319 this._injectedTextDecorationsTree.delete(node);
2324 }
2325 }
2326 > textModel.ts
2327 > public getNodeRange(host: IDecorationsTreesHost, node: IntervalNode): Range {
2328 const versionId = host.getVersionId();
2329 if (node.cachedVersionId !== versionId) {
2335 return node.range;
2336 }
2337 > textModel.ts
2338 > private _resolveNode(node: IntervalNode, cachedVersionId: number): void {
2339 if (isNodeInjectedText(node)) {
2340 this._injectedTextDecorationsTree.resolveNode(node, cachedVersionId);
2345 }
2346 }
2347 > textModel.ts
2348 > public acceptReplace(offset: number, length: number, textLength: number, forceMoveMarkers: boolean): void {
2349 this._decorationsTree0.acceptReplace(offset, length, textLength, forceMoveMarkers);
2350 this._decorationsTree1.acceptReplace(offset, length, textLength, forceMoveMarkers);
2351 this._injectedTextDecorationsTree.acceptReplace(offset, length, textLength, forceMoveMarkers);
2352 }
2353 > } textModel.ts
2354 >
2355 function cleanClassName(className: string): string {
2356 return className.replace(/[^a-z0-9\-_]/gi, ' ');
2357 }
2358 > textModel.ts
2359 > class DecorationOptions implements model.IDecorationOptions {
2360 > readonly color: string | ThemeColor;
2361 > readonly darkColor: string | ThemeColor;
2362 >
2363 > constructor(options: model.IDecorationOptions) {
2364 this.color = options.color || '';
2365 this.darkColor = options.darkColor || '';
2366
2367 }
2368 > } textModel.ts
2369 >
2370 > export class ModelDecorationOverviewRulerOptions extends DecorationOptions {
2371 > readonly position: model.OverviewRulerLane;
2372 > private _resolvedColor: string | null;
2373 >
2374 > constructor(options: model.IModelDecorationOverviewRulerOptions) {
2375 super(options);
2376 this._resolvedColor = null;
2377 this.position = (typeof options.position === 'number' ? options.position : model.OverviewRulerLane.Center);
2378 }
2379 > textModel.ts
2380 > public getColor(theme: IColorTheme): string {
2381 if (!this._resolvedColor) {
2382 if (isDark(theme.type) && this.darkColor) {
2388 return this._resolvedColor;
2389 }
2390 > textModel.ts
2391 > public invalidateCachedColor(): void {
2392 this._resolvedColor = null;
2393 }
2394 > textModel.ts
2395 > private _resolveColor(color: string | ThemeColor, theme: IColorTheme): string {
2396 if (typeof color === 'string') {
2397 return color;
2403 return c.toString();
2404 }
2405 > } textModel.ts
2406 >
2407 > export class ModelDecorationGlyphMarginOptions {
2408 > readonly position: model.GlyphMarginLane;
2409 > readonly persistLane: boolean | undefined;
2410 >
2411 > constructor(options: model.IModelDecorationGlyphMarginOptions | null | undefined) {
2412 this.position = options?.position ?? model.GlyphMarginLane.Center;
2413 this.persistLane = options?.persistLane;
2414 }
2415 > } textModel.ts
2416 >
2417 > export class ModelDecorationMinimapOptions extends DecorationOptions {
2418 > readonly position: model.MinimapPosition;
2419 > readonly sectionHeaderStyle: model.MinimapSectionHeaderStyle | null;
2420 > readonly sectionHeaderText: string | null;
2421 > private _resolvedColor: Color | undefined;
2422 >
2423 > constructor(options: model.IModelDecorationMinimapOptions) {
2424 super(options);
2425 this.position = options.position;
2427 this.sectionHeaderText = options.sectionHeaderText ?? null;
2428 }
2429 > textModel.ts
2430 > public getColor(theme: IColorTheme): Color | undefined {
2431 if (!this._resolvedColor) {
2432 if (isDark(theme.type) && this.darkColor) {
2439 return this._resolvedColor;
2440 }
2441 > textModel.ts
2442 > public invalidateCachedColor(): void {
2443 this._resolvedColor = undefined;
2444 }
2445 > textModel.ts
2446 > private _resolveColor(color: string | ThemeColor, theme: IColorTheme): Color | undefined {
2447 if (typeof color === 'string') {
2448 return Color.fromHex(color);
2450 return theme.getColor(color.id);
2451 }
2452 > } textModel.ts
2453 >
2454 > export class ModelDecorationInjectedTextOptions implements model.InjectedTextOptions {
2455 > public static from(options: model.InjectedTextOptions): ModelDecorationInjectedTextOptions {
2456 if (options instanceof ModelDecorationInjectedTextOptions) {
2457 return options;
2459 return new ModelDecorationInjectedTextOptions(options);
2460 }
2461 > textModel.ts
2462 > public readonly content: string;
2463 > public readonly tokens: TokenArray | null;
2464 > readonly inlineClassName: string | null;
2465 > readonly inlineClassNameAffectsLetterSpacing: boolean;
2466 > readonly attachedData: unknown | null;
2467 > readonly cursorStops: model.InjectedTextCursorStops | null;
2468 >
2469 > private constructor(options: model.InjectedTextOptions) {
2470 this.content = options.content || '';
2471 this.tokens = options.tokens ?? null;
2475 this.cursorStops = options.cursorStops || null;
2476 }
2477 > } textModel.ts
2478 >
2479 > export class ModelDecorationOptions implements model.IModelDecorationOptions {
2480 >
2481 > public static EMPTY: ModelDecorationOptions;
2482 >
2483 > public static register(options: model.IModelDecorationOptions): ModelDecorationOptions {
2484 > return new ModelDecorationOptions(options);
2485 > }
2486 >
2487 > public static createDynamic(options: model.IModelDecorationOptions): ModelDecorationOptions {
2488 return new ModelDecorationOptions(options);
2489 }
2490 > readonly description: string; textModel.ts
2491 > readonly blockClassName: string | null;
2492 > readonly blockIsAfterEnd: boolean | null;
2493 > readonly blockDoesNotCollapse?: boolean | null;
2494 > readonly blockPadding: [top: number, right: number, bottom: number, left: number] | null;
2495 > readonly stickiness: model.TrackedRangeStickiness;
2496 > readonly zIndex: number;
2497 > readonly className: string | null;
2498 > readonly shouldFillLineOnLineBreak: boolean | null;
2499 > readonly hoverMessage: IMarkdownString | IMarkdownString[] | null;
2500 > readonly glyphMarginHoverMessage: IMarkdownString | IMarkdownString[] | null;
2501 > readonly isWholeLine: boolean;
2502 > readonly lineHeight: number | null;
2503 > readonly fontSize: string | null;
2504 > readonly showIfCollapsed: boolean;
2505 > readonly collapseOnReplaceEdit: boolean;
2506 > readonly overviewRuler: ModelDecorationOverviewRulerOptions | null;
2507 > readonly minimap: ModelDecorationMinimapOptions | null;
2508 > readonly glyphMargin?: model.IModelDecorationGlyphMarginOptions | null | undefined;
2509 > readonly glyphMarginClassName: string | null;
2510 > readonly linesDecorationsClassName: string | null;
2511 > readonly lineNumberClassName: string | null;
2512 > readonly lineNumberHoverMessage: IMarkdownString | IMarkdownString[] | null;
2513 > readonly linesDecorationsTooltip: string | null;
2514 > readonly firstLineDecorationClassName: string | null;
2515 > readonly marginClassName: string | null;
2516 > readonly inlineClassName: string | null;
2517 > readonly inlineClassNameAffectsLetterSpacing: boolean;
2518 > readonly beforeContentClassName: string | null;
2519 > readonly afterContentClassName: string | null;
2520 > readonly after: ModelDecorationInjectedTextOptions | null;
2521 > readonly before: ModelDecorationInjectedTextOptions | null;
2522 > readonly hideInCommentTokens: boolean | null;
2523 > readonly hideInStringTokens: boolean | null;
2524 > readonly affectsFont: boolean | null;
2525 > readonly textDirection?: model.TextDirection | null | undefined;
2526 >
2527 > private constructor(options: model.IModelDecorationOptions) {
2528 > this.description = options.description;
2529 > this.blockClassName = options.blockClassName ? cleanClassName(options.blockClassName) : null;
2530 > this.blockDoesNotCollapse = options.blockDoesNotCollapse ?? null;
2531 > this.blockIsAfterEnd = options.blockIsAfterEnd ?? null;
2532 > this.blockPadding = options.blockPadding ?? null;
2533 > this.stickiness = options.stickiness || model.TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges;
2534 > this.zIndex = options.zIndex || 0;
2535 > this.className = options.className ? cleanClassName(options.className) : null;
2536 > this.shouldFillLineOnLineBreak = options.shouldFillLineOnLineBreak ?? null;
2537 > this.hoverMessage = options.hoverMessage || null;
2538 > this.glyphMarginHoverMessage = options.glyphMarginHoverMessage || null;
2539 > this.lineNumberHoverMessage = options.lineNumberHoverMessage || null;
2540 > this.isWholeLine = options.isWholeLine || false;
2541 > this.lineHeight = options.lineHeight ? Math.min(options.lineHeight, LINE_HEIGHT_CEILING) : null;
2542 > this.fontSize = options.fontSize || null;
2543 > this.affectsFont = !!options.fontSize || !!options.fontFamily || !!options.fontWeight || !!options.fontStyle;
2544 > this.showIfCollapsed = options.showIfCollapsed || false;
2545 > this.collapseOnReplaceEdit = options.collapseOnReplaceEdit || false;
2546 > this.overviewRuler = options.overviewRuler ? new ModelDecorationOverviewRulerOptions(options.overviewRuler) : null;
2547 > this.minimap = options.minimap ? new ModelDecorationMinimapOptions(options.minimap) : null;
2548 > this.glyphMargin = options.glyphMarginClassName ? new ModelDecorationGlyphMarginOptions(options.glyphMargin) : null;
2549 > this.glyphMarginClassName = options.glyphMarginClassName ? cleanClassName(options.glyphMarginClassName) : null;
2550 > this.linesDecorationsClassName = options.linesDecorationsClassName ? cleanClassName(options.linesDecorationsClassName) : null;
2551 > this.lineNumberClassName = options.lineNumberClassName ? cleanClassName(options.lineNumberClassName) : null;
2552 > this.linesDecorationsTooltip = options.linesDecorationsTooltip ? strings.htmlAttributeEncodeValue(options.linesDecorationsTooltip) : null;
2553 > this.firstLineDecorationClassName = options.firstLineDecorationClassName ? cleanClassName(options.firstLineDecorationClassName) : null;
2554 > this.marginClassName = options.marginClassName ? cleanClassName(options.marginClassName) : null;
2555 > this.inlineClassName = options.inlineClassName ? cleanClassName(options.inlineClassName) : null;
2556 > this.inlineClassNameAffectsLetterSpacing = options.inlineClassNameAffectsLetterSpacing || false;
2557 > this.beforeContentClassName = options.beforeContentClassName ? cleanClassName(options.beforeContentClassName) : null;
2558 > this.afterContentClassName = options.afterContentClassName ? cleanClassName(options.afterContentClassName) : null;
2559 > this.after = options.after ? ModelDecorationInjectedTextOptions.from(options.after) : null;
2560 > this.before = options.before ? ModelDecorationInjectedTextOptions.from(options.before) : null;
2561 > this.hideInCommentTokens = options.hideInCommentTokens ?? false;
2562 > this.hideInStringTokens = options.hideInStringTokens ?? false;
2563 > this.textDirection = options.textDirection ?? null;
2564 > }
2565 > }
2566 > ModelDecorationOptions.EMPTY = ModelDecorationOptions.register({ description: 'empty' });
2567 >
2568 > /**
2569 > * The order carefully matches the values of the enum.
2570 > */
2571 > const TRACKED_RANGE_OPTIONS = [
2572 > ModelDecorationOptions.register({ description: 'tracked-range-always-grows-when-typing-at-edges', stickiness: model.TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges }),
2573 > ModelDecorationOptions.register({ description: 'tracked-range-never-grows-when-typing-at-edges', stickiness: model.TrackedRangeStickiness.NeverGrowsWhenTypingAtEdges }),
2574 > ModelDecorationOptions.register({ description: 'tracked-range-grows-only-when-typing-before', stickiness: model.TrackedRangeStickiness.GrowsOnlyWhenTypingBefore }),
2575 > ModelDecorationOptions.register({ description: 'tracked-range-grows-only-when-typing-after', stickiness: model.TrackedRangeStickiness.GrowsOnlyWhenTypingAfter }),
2576 > ];
2577 >
2578 function _normalizeOptions(options: model.IModelDecorationOptions): ModelDecorationOptions {
2579 if (options instanceof ModelDecorationOptions) {
2582 return ModelDecorationOptions.createDynamic(options);
2583 }
2584 > textModel.ts
2585 >
2586 > class DidChangeDecorationsEmitter extends Disposable {
2587 >
2588 > private readonly _actual: Emitter<IModelDecorationsChangedEvent> = this._register(new Emitter<IModelDecorationsChangedEvent>());
2589 > public readonly event: Event<IModelDecorationsChangedEvent> = this._actual.event;
2590 >
2591 > private _deferredCnt: number;
2592 > private _shouldFireDeferred: boolean;
2593 > private _affectsMinimap: boolean;
2594 > private _affectsOverviewRuler: boolean;
2595 > private _affectedInjectedTextLines: Set<number> | null = null;
2596 > private _affectedLineHeights: SetWithKey<LineHeightChangingDecoration> | null = null;
2597 > private _affectedFontLines: SetWithKey<LineFontChangingDecoration> | null = null;
2598 > private _affectsGlyphMargin: boolean;
2599 > private _affectsLineNumber: boolean;
2600 >
2601 > constructor(private readonly handleBeforeFire: (affectedInjectedTextLines: Set<number> | null, affectedLineHeights: SetWithKey<LineHeightChangingDecoration> | null, affectedFontLines: SetWithKey<LineFontChangingDecoration> | null) => void) {
2602 super();
2603 this._deferredCnt = 0;
2608 this._affectsLineNumber = false;
2609 }
2610 > textModel.ts
2611 > hasListeners(): boolean {
2612 return this._actual.hasListeners();
2613 }
2614 > textModel.ts
2615 > public beginDeferredEmit(): void {
2616 this._deferredCnt++;
2617 }
2618 > textModel.ts
2619 > public endDeferredEmit(): void {
2620 this._deferredCnt--;
2621 if (this._deferredCnt === 0) {
2632 }
2633 }
2634 > textModel.ts
2635 > public recordLineAffectedByInjectedText(lineNumber: number): void {
2636 if (!this._affectedInjectedTextLines) {
2637 this._affectedInjectedTextLines = new Set();
2639 this._affectedInjectedTextLines.add(lineNumber);
2640 }
2641 > textModel.ts
2642 > public recordLineAffectedByLineHeightChange(ownerId: number, decorationId: string, lineNumber: number, lineHeight: number | null): void {
2643 if (!this._affectedLineHeights) {
2644 this._affectedLineHeights = new SetWithKey<LineHeightChangingDecoration>([], LineHeightChangingDecoration.toKey);
2646 this._affectedLineHeights.add(new LineHeightChangingDecoration(ownerId, decorationId, lineNumber, lineHeight));
2647 }
2648 > textModel.ts
2649 > public recordLineAffectedByFontChange(ownerId: number, decorationId: string, lineNumber: number): void {
2650 if (!this._affectedFontLines) {
2651 this._affectedFontLines = new SetWithKey<LineFontChangingDecoration>([], LineFontChangingDecoration.toKey);
2653 this._affectedFontLines.add(new LineFontChangingDecoration(ownerId, decorationId, lineNumber));
2654 }
2655 > textModel.ts
2656 > public checkAffectedAndFire(options: ModelDecorationOptions): void {
2657 this._affectsMinimap ||= !!options.minimap?.position;
2658 this._affectsOverviewRuler ||= !!options.overviewRuler?.color;
2661 this.tryFire();
2662 }
2663 > textModel.ts
2664 > public fire(): void {
2665 this._affectsMinimap = true;
2666 this._affectsOverviewRuler = true;
2668 this.tryFire();
2669 }
2670 > textModel.ts
2671 > private tryFire() {
2672 if (this._deferredCnt === 0) {
2673 this.doFire();
2676 }
2677 }
2678 > textModel.ts
2679 > private doFire() {
2680 this.handleBeforeFire(this._affectedInjectedTextLines, this._affectedLineHeights, this._affectedFontLines);
2681
2692 this._actual.fire(event);
2693 }
2694 > } textModel.ts
2695 >
2696 > //#endregion
2697 >
2698 > class DidChangeContentEmitter extends Disposable {
2699 >
2700 > private readonly _emitter: Emitter<InternalModelContentChangeEvent> = this._register(new Emitter<InternalModelContentChangeEvent>());
2701 > public readonly event: Event<InternalModelContentChangeEvent> = this._emitter.event;
2702 >
2703 > private _deferredCnt: number;
2704 > private _deferredEvent: InternalModelContentChangeEvent | null;
2705 >
2706 > constructor() {
2707 super();
2708 this._deferredCnt = 0;
2709 this._deferredEvent = null;
2710 }
2711 > textModel.ts
2712 > public hasListeners(): boolean {
2713 return this._emitter.hasListeners();
2714 }
2715 > textModel.ts
2716 > public beginDeferredEmit(): void {
2717 this._deferredCnt++;
2718 }
2719 > textModel.ts
2720 > public endDeferredEmit(resultingSelection: Selection[] | null = null): void {
2721 this._deferredCnt--;
2722 if (this._deferredCnt === 0) {
2729 }
2730 }
2731 > textModel.ts
2732 > public fire(e: InternalModelContentChangeEvent): void {
2733 if (this._deferredCnt > 0) {
2734 if (this._deferredEvent) {
src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase.ts 263 introduced LOC · 72 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- pieceTreeBase.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { CharCode } from '../../../../base/common/charCode.js';
7 > import { Position } from '../../core/position.js';
8 > import { Range } from '../../core/range.js';
9 > import { FindMatch, ITextSnapshot, SearchData } from '../../model.js';
10 > import { NodeColor, SENTINEL, TreeNode, fixInsert, leftest, rbDelete, righttest, updateTreeMetadata } from './rbTreeBase.js';
11 > import { Searcher, createFindMatch, isValidMatch } from '../textModelSearch.js';
12 >
13 > // const lfRegex = new RegExp(/\r\n|\r|\n/g);
14 > const AverageBufferSize = 65535;
15 >
16 function createUintArray(arr: number[]): Uint32Array | Uint16Array {
17 let r;
24 return r;
25 }
27 > class LineStarts {
28 > constructor(
29 public readonly lineStarts: Uint32Array | Uint16Array | number[],
30 public readonly cr: number,
33 public readonly isBasicASCII: boolean
34 ) { }
36 >
37 > export function createLineStartsFast(str: string, readonly: boolean = true): Uint32Array | Uint16Array | number[] {
38 const r: number[] = [0];
39 let rLength = 1;
61 }
62 }
64 > export function createLineStarts(r: number[], str: string): LineStarts {
65 r.length = 0;
66 r[0] = 0;
98 return result;
99 }
101 > interface NodePosition {
102 > /**
103 > * Piece Index
104 > */
105 > node: TreeNode;
106 > /**
107 > * remainder in current piece.
108 > */
109 > remainder: number;
110 > /**
111 > * node start offset in document.
112 > */
113 > nodeStartOffset: number;
114 > }
115 >
116 > interface BufferCursor {
117 > /**
118 > * Line number in current buffer
119 > */
120 > line: number;
121 > /**
122 > * Column number in current buffer
123 > */
124 > column: number;
125 > }
126 >
127 > export class Piece {
128 > readonly bufferIndex: number;
129 > readonly start: BufferCursor;
130 > readonly end: BufferCursor;
131 > readonly length: number;
132 > readonly lineFeedCnt: number;
133 >
134 > constructor(bufferIndex: number, start: BufferCursor, end: BufferCursor, lineFeedCnt: number, length: number) {
135 this.bufferIndex = bufferIndex;
136 this.start = start;
139 this.length = length;
140 }
142 >
143 > export class StringBuffer {
144 > buffer: string;
145 > lineStarts: Uint32Array | Uint16Array | number[];
146 >
147 > constructor(buffer: string, lineStarts: Uint32Array | Uint16Array | number[]) {
148 this.buffer = buffer;
149 this.lineStarts = lineStarts;
150 }
152 >
153 > /**
154 > * Readonly snapshot for piece tree.
155 > * In a real multiple thread environment, to make snapshot reading always work correctly, we need to
156 > * 1. Make TreeNode.piece immutable, then reading and writing can run in parallel.
157 > * 2. TreeNode/Buffers normalization should not happen during snapshot reading.
158 > */
159 > class PieceTreeSnapshot implements ITextSnapshot {
160 > private readonly _pieces: Piece[];
161 > private _index: number;
162 > private readonly _tree: PieceTreeBase;
163 > private readonly _BOM: string;
164 >
165 > constructor(tree: PieceTreeBase, BOM: string) {
166 this._pieces = [];
167 this._tree = tree;
177 }
178 }
180 > read(): string | null {
181 if (this._pieces.length === 0) {
182 if (this._index === 0) {
197 return this._tree.getPieceContent(this._pieces[this._index++]);
198 }
200 >
201 > interface CacheEntry {
202 > node: TreeNode;
203 > nodeStartOffset: number;
204 > nodeStartLineNumber?: number;
205 > }
206 >
207 > class PieceTreeSearchCache {
208 > private readonly _limit: number;
209 > private _cache: CacheEntry[];
210 >
211 > constructor(limit: number) {
212 this._limit = limit;
213 this._cache = [];
214 }
216 > public get(offset: number): CacheEntry | null {
217 for (let i = this._cache.length - 1; i >= 0; i--) {
218 const nodePos = this._cache[i];
223 return null;
224 }
226 > public get2(lineNumber: number): { node: TreeNode; nodeStartOffset: number; nodeStartLineNumber: number } | null {
227 for (let i = this._cache.length - 1; i >= 0; i--) {
228 const nodePos = this._cache[i];
233 return null;
234 }
236 > public set(nodePosition: CacheEntry) {
237 if (this._cache.length >= this._limit) {
238 this._cache.shift();
240 this._cache.push(nodePosition);
241 }
243 > public validate(offset: number) {
244 let hasInvalidVal = false;
245 const tmp: Array<CacheEntry | null> = this._cache;
264 }
265 }
267 >
268 > export class PieceTreeBase {
269 > root!: TreeNode;
270 > protected _buffers!: StringBuffer[]; // 0 is change buffer, others are readonly original buffer.
271 > protected _lineCnt!: number;
272 > protected _length!: number;
273 > protected _EOL!: '\r\n' | '\n';
274 > protected _EOLLength!: number;
275 > protected _EOLNormalized!: boolean;
276 > private _lastChangeBufferPos!: BufferCursor;
277 > private _searchCache!: PieceTreeSearchCache;
278 > private _lastVisitedLine!: { lineNumber: number; value: string };
279 >
280 > constructor(chunks: StringBuffer[], eol: '\r\n' | '\n', eolNormalized: boolean) {
281 this.create(chunks, eol, eolNormalized);
282 }
284 > create(chunks: StringBuffer[], eol: '\r\n' | '\n', eolNormalized: boolean) {
285 this._buffers = [
286 new StringBuffer('', [0])
317 this.computeBufferMetadata();
318 }
320 > normalizeEOL(eol: '\r\n' | '\n') {
321 const averageBufferSize = AverageBufferSize;
322 const min = averageBufferSize - Math.floor(averageBufferSize / 3);
351 this.create(chunks, eol, true);
352 }
354 > // #region Buffer API
355 > public getEOL(): '\r\n' | '\n' {
356 return this._EOL;
357 }
359 > public setEOL(newEOL: '\r\n' | '\n'): void {
360 this._EOL = newEOL;
361 this._EOLLength = this._EOL.length;
362 this.normalizeEOL(newEOL);
363 }
365 > public createSnapshot(BOM: string): ITextSnapshot {
366 return new PieceTreeSnapshot(this, BOM);
367 }
369 > public equal(other: PieceTreeBase): boolean {
370 if (this.getLength() !== other.getLength()) {
371 return false;
392 return ret;
393 }
395 > public getOffsetAt(lineNumber: number, column: number): number {
396 let leftLen = 0; // inorder
397
415 return leftLen;
416 }
418 > public getPositionAt(offset: number): Position {
419 offset = Math.floor(offset);
420 offset = Math.max(0, offset);
456 return new Position(1, 1);
457 }
459 > public getValueInRange(range: Range, eol?: string): string {
460 if (range.startLineNumber === range.endLineNumber && range.startColumn === range.endColumn) {
461 return '';
481 return value;
482 }
484 > public getValueInRange2(startPosition: NodePosition, endPosition: NodePosition): string {
485 if (startPosition.node === endPosition.node) {
486 const node = startPosition.node;
512 return ret;
513 }
515 > public getLinesContent(): string[] {
516 const lines: string[] = [];
517 let linesLength = 0;
602 return lines;
603 }
605 > public getLength(): number {
606 return this._length;
607 }
609 > public getLineCount(): number {
610 return this._lineCnt;
611 }
613 > public getLineContent(lineNumber: number): string {
614 if (this._lastVisitedLine.lineNumber === lineNumber) {
615 return this._lastVisitedLine.value;
628 return this._lastVisitedLine.value;
629 }
631 > private _getCharCode(nodePos: NodePosition): number {
632 if (nodePos.remainder === nodePos.node.piece.length) {
633 // the char we want to fetch is at the head of next node.
648 }
649 }
651 > public getLineCharCode(lineNumber: number, index: number): number {
652 const nodePos = this.nodeAt2(lineNumber, index + 1);
653 return this._getCharCode(nodePos);
654 }
656 > public getLineLength(lineNumber: number): number {
657 if (lineNumber === this.getLineCount()) {
658 const startOffset = this.getOffsetAt(lineNumber, 1);
661 return this.getOffsetAt(lineNumber + 1, 1) - this.getOffsetAt(lineNumber, 1) - this._EOLLength;
662 }
664 > public getCharCode(offset: number): number {
665 const nodePos = this.nodeAt(offset);
666 return this._getCharCode(nodePos);
667 }
669 > public getNearestChunk(offset: number): string {
670 const nodePos = this.nodeAt(offset);
671 if (nodePos.remainder === nodePos.node.piece.length) {
687 }
688 }
690 > public findMatchesInNode(node: TreeNode, searcher: Searcher, startLineNumber: number, startColumn: number, startCursor: BufferCursor, endCursor: BufferCursor, searchData: SearchData, captureMatches: boolean, limitResultCount: number, resultLen: number, result: FindMatch[]) {
691 const buffer = this._buffers[node.piece.bufferIndex];
692 const startOffsetInBuffer = this.offsetInBuffer(node.piece.bufferIndex, node.piece.start);
735 return resultLen;
736 }
738 > public findMatchesLineByLine(searchRange: Range, searchData: SearchData, captureMatches: boolean, limitResultCount: number): FindMatch[] {
739 const result: FindMatch[] = [];
740 let resultLen = 0;
809 return result;
810 }
812 > private _findMatchesInLine(searchData: SearchData, searcher: Searcher, text: string, lineNumber: number, deltaOffset: number, resultLen: number, result: FindMatch[], captureMatches: boolean, limitResultCount: number): number {
813 const wordSeparators = searchData.wordSeparators;
814 if (!captureMatches && searchData.simpleSearch) {
843 return resultLen;
844 }
846 > // #endregion
847 >
848 > // #region Piece Table
849 > public insert(offset: number, value: string, eolNormalized: boolean = false): void {
850 this._EOLNormalized = this._EOLNormalized && eolNormalized;
851 this._lastVisitedLine.lineNumber = 0;
944 this.computeBufferMetadata();
945 }
947 > public delete(offset: number, cnt: number): void {
948 this._lastVisitedLine.lineNumber = 0;
949 this._lastVisitedLine.value = '';
1017 this.computeBufferMetadata();
1018 }
1020 > private insertContentToNodeLeft(value: string, node: TreeNode) {
1021 // we are inserting content to the beginning of node
1022 const nodesToDel: TreeNode[] = [];
1052 this.deleteNodes(nodesToDel);
1053 }
1055 > private insertContentToNodeRight(value: string, node: TreeNode) {
1056 // we are inserting to the right of this node.
1057 if (this.adjustCarriageReturnFromNext(value, node)) {
1070 this.validateCRLFWithPrevNode(newNode);
1071 }
1073 > private positionInBuffer(node: TreeNode, remainder: number): BufferCursor;
1074 > private positionInBuffer(node: TreeNode, remainder: number, ret: BufferCursor): null;
1075 > private positionInBuffer(node: TreeNode, remainder: number, ret?: BufferCursor): BufferCursor | null {
1076 const piece = node.piece;
1077 const bufferIndex = node.piece.bufferIndex;
1120 };
1121 }
1123 > private getLineFeedCnt(bufferIndex: number, start: BufferCursor, end: BufferCursor): number {
1124 // we don't need to worry about start: abc\r|\n, or abc|\r, or abc|\n, or abc|\r\n doesn't change the fact that, there is one line break after start.
1125 // now let's take care of end: abc\r|\n, if end is in between \r and \n, we need to add line feed count by 1
1150 }
1151 }
1153 > private offsetInBuffer(bufferIndex: number, cursor: BufferCursor): number {
1154 const lineStarts = this._buffers[bufferIndex].lineStarts;
1155 return lineStarts[cursor.line] + cursor.column;
1156 }
1158 > private deleteNodes(nodes: TreeNode[]): void {
1159 for (let i = 0; i < nodes.length; i++) {
1160 rbDelete(this, nodes[i]);
1161 }
1162 }
1164 > private createNewPieces(text: string): Piece[] {
1165 if (text.length > AverageBufferSize) {
1166 // the content is large, operations like substring, charCode becomes slow
1246 return [newPiece];
1247 }
1249 > public getLinesRawContent(): string {
1250 return this.getContentOfSubTree(this.root);
1251 }
1253 > public getLineRawContent(lineNumber: number, endOffset: number = 0): string {
1254 let x = this.root;
1255
1322 return ret;
1323 }
1325 > private computeBufferMetadata() {
1326 let x = this.root;
1327
1339 this._searchCache.validate(this._length);
1340 }
1342 > // #region node operations
1343 > private getIndexOf(node: TreeNode, accumulatedValue: number): { index: number; remainder: number } {
1344 const piece = node.piece;
1345 const pos = this.positionInBuffer(node, accumulatedValue);
1357 return { index: lineCnt, remainder: pos.column };
1358 }
1360 > private getAccumulatedValue(node: TreeNode, index: number) {
1361 if (index < 0) {
1362 return 0;
1371 }
1372 }
1374 > private deleteNodeTail(node: TreeNode, pos: BufferCursor) {
1375 const piece = node.piece;
1376 const originalLFCnt = piece.lineFeedCnt;
1395 updateTreeMetadata(this, node, size_delta, lf_delta);
1396 }
1398 > private deleteNodeHead(node: TreeNode, pos: BufferCursor) {
1399 const piece = node.piece;
1400 const originalLFCnt = piece.lineFeedCnt;
1417 updateTreeMetadata(this, node, size_delta, lf_delta);
1418 }
1420 > private shrinkNode(node: TreeNode, start: BufferCursor, end: BufferCursor) {
1421 const piece = node.piece;
1422 const originalStartPos = piece.start;
1452 this.validateCRLFWithPrevNode(newNode);
1453 }
1455 > private appendToNode(node: TreeNode, value: string): void {
1456 if (this.adjustCarriageReturnFromNext(value, node)) {
1457 value += '\n';
1492 updateTreeMetadata(this, node, value.length, lf_delta);
1493 }
1495 > private nodeAt(offset: number): NodePosition {
1496 let x = this.root;
1497 const cache = this._searchCache.get(offset);
1527 return null!;
1528 }
1530 > private nodeAt2(lineNumber: number, column: number): NodePosition {
1531 let x = this.root;
1532 let nodeStartOffset = 0;
1594 return null!;
1595 }
1597 > private nodeCharCodeAt(node: TreeNode, offset: number): number {
1598 if (node.piece.lineFeedCnt < 1) {
1599 return -1;
1603 return buffer.buffer.charCodeAt(newOffset);
1604 }
1606 > private offsetOfNode(node: TreeNode): number {
1607 if (!node) {
1608 return 0;
1619 return pos;
1620 }
1622 > // #endregion
1623 >
1624 > // #region CRLF
1625 > private shouldCheckCRLF() {
1626 return !(this._EOLNormalized && this._EOL === '\n');
1627 }
1629 > private startWithLF(val: string | TreeNode): boolean {
1630 if (typeof val === 'string') {
1631 return val.charCodeAt(0) === 10;
1650 return this._buffers[piece.bufferIndex].buffer.charCodeAt(startOffset) === 10;
1651 }
1653 > private endWithCR(val: string | TreeNode): boolean {
1654 if (typeof val === 'string') {
1655 return val.charCodeAt(val.length - 1) === 13;
1662 return this.nodeCharCodeAt(val, val.piece.length - 1) === 13;
1663 }
1665 > private validateCRLFWithPrevNode(nextNode: TreeNode) {
1666 if (this.shouldCheckCRLF() && this.startWithLF(nextNode)) {
1667 const node = nextNode.prev();
1671 }
1672 }
1674 > private validateCRLFWithNextNode(node: TreeNode) {
1675 if (this.shouldCheckCRLF() && this.endWithCR(node)) {
1676 const nextNode = node.next();
1680 }
1681 }
1683 > private fixCRLF(prev: TreeNode, next: TreeNode) {
1684 const nodesToDel: TreeNode[] = [];
1685 // update node
1735 }
1736 }
1738 > private adjustCarriageReturnFromNext(value: string, node: TreeNode): boolean {
1739 if (this.shouldCheckCRLF() && this.endWithCR(value)) {
1740 const nextNode = node.next();
1767 return false;
1768 }
1770 > // #endregion
1771 >
1772 > // #endregion
1773 >
1774 > // #region Tree operations
1775 > iterate(node: TreeNode, callback: (node: TreeNode) => boolean): boolean {
1776 if (node === SENTINEL) {
1777 return callback(SENTINEL);
1785 return callback(node) && this.iterate(node.right, callback);
1786 }
1788 > private getNodeContent(node: TreeNode) {
1789 if (node === SENTINEL) {
1790 return '';
1797 return currentContent;
1798 }
1800 > getPieceContent(piece: Piece) {
1801 const buffer = this._buffers[piece.bufferIndex];
1802 const startOffset = this.offsetInBuffer(piece.bufferIndex, piece.start);
1805 return currentContent;
1806 }
1808 > /**
1809 > * node node
1810 > * / \ / \
1811 > * a b <---- a b
1812 > * /
1813 > * z
1814 > */
1815 > private rbInsertRight(node: TreeNode | null, p: Piece): TreeNode {
1816 const z = new TreeNode(p, NodeColor.Red);
1817 z.left = SENTINEL;
1837 return z;
1838 }
1840 > /**
1841 > * node node
1842 > * / \ / \
1843 > * a b ----> a b
1844 > * \
1845 > * z
1846 > */
1847 > private rbInsertLeft(node: TreeNode | null, p: Piece): TreeNode {
1848 const z = new TreeNode(p, NodeColor.Red);
1849 z.left = SENTINEL;
1868 return z;
1869 }
1871 > private getContentOfSubTree(node: TreeNode): string {
1872 let str = '';
1873
src/vs/editor/common/model/tokens/treeSitter/treeSitterTokenizationImpl.ts 158 introduced LOC · 39 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- treeSitterTokenizationImpl.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { Emitter, Event } from '../../../../../base/common/event.js';
7 > import { Disposable } from '../../../../../base/common/lifecycle.js';
8 > import { setTimeout0 } from '../../../../../base/common/platform.js';
9 > import { StopWatch } from '../../../../../base/common/stopwatch.js';
10 > import { LanguageId } from '../../../encodedTokenAttributes.js';
11 > import { ILanguageIdCodec, QueryCapture } from '../../../languages.js';
12 > import { IModelContentChangedEvent, IModelTokensChangedEvent } from '../../../textModelEvents.js';
13 > import { findLikelyRelevantLines } from '../../textModelTokens.js';
14 > import { TokenStore, TokenUpdate, TokenQuality } from './tokenStore.js';
15 > import { TreeSitterTree, RangeChange, RangeWithOffsets } from './treeSitterTree.js';
16 > import type * as TreeSitter from '@vscode/tree-sitter-wasm';
17 > import { autorun, autorunHandleChanges, IObservable, recordChanges, runOnChange } from '../../../../../base/common/observable.js';
18 > import { LineRange } from '../../../core/ranges/lineRange.js';
19 > import { LineTokens } from '../../../tokens/lineTokens.js';
20 > import { Position } from '../../../core/position.js';
21 > import { Range } from '../../../core/range.js';
22 > import { isDefined } from '../../../../../base/common/types.js';
23 > import { ITreeSitterThemeService } from '../../../services/treeSitter/treeSitterThemeService.js';
24 > import { BugIndicatingError } from '../../../../../base/common/errors.js';
25 >
26 > export class TreeSitterTokenizationImpl extends Disposable {
27 > private readonly _tokenStore: TokenStore;
28 > private _accurateVersion: number;
29 > private _guessVersion: number;
30 >
31 > private readonly _onDidChangeTokens: Emitter<{ changes: IModelTokensChangedEvent }> = this._register(new Emitter());
32 > public readonly onDidChangeTokens: Event<{ changes: IModelTokensChangedEvent }> = this._onDidChangeTokens.event;
33 > private readonly _onDidCompleteBackgroundTokenization: Emitter<void> = this._register(new Emitter());
34 > public readonly onDidChangeBackgroundTokenization: Event<void> = this._onDidCompleteBackgroundTokenization.event;
35 >
36 > private _encodedLanguageId: LanguageId;
37 >
38 > private get _textModel() {
39 > return this._tree.textModel;
40 > }
41 >
42 > constructor(
43 private readonly _tree: TreeSitterTree,
44 private readonly _highlightingQueries: TreeSitter.Query,
97 }));
98 }
100 > public handleContentChanged(e: IModelContentChangedEvent): void {
101 this._guessVersion = e.versionId;
102 for (const change of e.changes) {
124 }
125 }
127 > public getLineTokens(lineNumber: number) {
128 const content = this._textModel.getLineContent(lineNumber);
129 const rawTokens = this.getTokens(lineNumber);
130 return new LineTokens(rawTokens, content, this._languageIdCodec);
131 }
133 > private _createEmptyTokens() {
134 const emptyToken = this._emptyToken();
135 const modelEndOffset = this._textModel.getValueLength();
138 return emptyTokens;
139 }
141 > private _emptyToken() {
142 return this._treeSitterThemeService.findMetadata([], this._encodedLanguageId, false, undefined);
143 }
145 > private _emptyTokensForOffsetAndLength(offset: number, length: number, emptyToken: number): TokenUpdate {
146 return { token: emptyToken, length: offset + length, startOffsetInclusive: 0 };
147 }
149 > public hasAccurateTokensForLine(lineNumber: number): boolean {
150 return this.hasTokens(new Range(lineNumber, 1, lineNumber, this._textModel.getLineMaxColumn(lineNumber)));
151 }
153 > public tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {
154 const rawLineTokens = this._guessTokensForLinesContent(lineNumber, lines);
155 const lineTokens: LineTokens[] = [];
162 return lineTokens;
163 }
165 > private _rangeHasTokens(range: Range, minimumTokenQuality: TokenQuality): boolean {
166 return this._tokenStore.rangeHasTokens(this._textModel.getOffsetAt(range.getStartPosition()), this._textModel.getOffsetAt(range.getEndPosition()), minimumTokenQuality);
167 }
169 > public hasTokens(accurateForRange?: Range): boolean {
170 if (!accurateForRange || (this._guessVersion === this._accurateVersion)) {
171 return true;
174 return !this._tokenStore.rangeNeedsRefresh(this._textModel.getOffsetAt(accurateForRange.getStartPosition()), this._textModel.getOffsetAt(accurateForRange.getEndPosition()));
175 }
177 > public getTokens(line: number): Uint32Array {
178 const lineStartOffset = this._textModel.getOffsetAt({ lineNumber: line, column: 1 });
179 const lineEndOffset = this._textModel.getOffsetAt({ lineNumber: line, column: this._textModel.getLineLength(line) + 1 });
186 return result;
187 }
189 > getTokensInRange(range: Range, rangeStartOffset: number, rangeEndOffset: number, captures?: QueryCapture[]): TokenUpdate[] | undefined {
190 const tokens = captures ? this._tokenizeCapturesWithMetadata(captures, rangeStartOffset, rangeEndOffset) : this._tokenize(range, rangeStartOffset, rangeEndOffset);
191 if (tokens?.endOffsetsAndMetadata) {
194 return undefined;
195 }
197 > private _updateTokensInStore(version: number, updates: { oldRangeLength?: number; newTokens: TokenUpdate[] }[], tokenQuality: TokenQuality): void {
198 this._accurateVersion = version;
199 for (const update of updates) {
210 }
211 }
213 > private _markForRefresh(range: Range): void {
214 this._tokenStore.markForRefresh(this._textModel.getOffsetAt(range.getStartPosition()), this._textModel.getOffsetAt(range.getEndPosition()));
215 }
217 > private _getNeedsRefresh(): { range: Range; startOffset: number; endOffset: number }[] {
218 const needsRefreshOffsetRanges = this._tokenStore.getNeedsRefresh();
219 if (!needsRefreshOffsetRanges) {
226 }));
227 }
229 >
230 > private _parseAndTokenizeViewPort(lineRanges: readonly LineRange[]) {
231 const viewportRanges = lineRanges.map(r => r.toInclusiveRange()).filter(isDefined);
232 for (const range of viewportRanges) {
251 }
252 }
254 > private _guessTokensForLinesContent(lineNumber: number, lines: string[]): Uint32Array[] | undefined {
255 if (lines.length === 0) {
256 return undefined;
293 return tokensByLine;
294 }
296 > private _forceParseAndTokenizeContent(range: Range, startOffsetOfRangeInDocument: number, endOffsetOfRangeInDocument: number, content: string, asUpdate: true): TokenUpdate[] | undefined;
297 > private _forceParseAndTokenizeContent(range: Range, startOffsetOfRangeInDocument: number, endOffsetOfRangeInDocument: number, content: string, asUpdate: false): EndOffsetToken[] | undefined;
298 > private _forceParseAndTokenizeContent(range: Range, startOffsetOfRangeInDocument: number, endOffsetOfRangeInDocument: number, content: string, asUpdate: boolean): EndOffsetToken[] | TokenUpdate[] | undefined {
299 const likelyRelevantLines = findLikelyRelevantLines(this._textModel, range.startLineNumber).likelyRelevantLines;
300 const likelyRelevantPrefix = likelyRelevantLines.join(this._textModel.getEOL());
320 }
321 }
323 >
324 > private _firstTreeUpdate(versionId: number) {
325 return this._setViewPortTokens(versionId);
326 }
328 > private _setViewPortTokens(versionId: number) {
329 const rangeChanges = this._visibleLineRanges.get().map<RangeChange | undefined>(lineRange => {
330 const range = lineRange.toInclusiveRange();
341 return this._handleTreeUpdate(rangeChanges, versionId);
342 }
344 > /**
345 > * Do not await in this method, it will cause a race
346 > */
347 > private _handleTreeUpdate(ranges: RangeChange[], versionId: number) {
348 const rangeChanges: RangeWithOffsets[] = [];
349 const chunkSize = 1000;
407 });
408 }
410 > private async _updateTreeForRanges(rangeChanges: RangeWithOffsets[], versionId: number, captures: QueryCapture[][]) {
411 let tokenUpdate: { newTokens: TokenUpdate[] } | undefined;
412
436 this._onDidCompleteBackgroundTokenization.fire();
437 }
439 > private _refreshNeedsRefresh(versionId: number) {
440 const rangesToRefresh = this._getNeedsRefresh();
441 if (rangesToRefresh.length === 0) {
455 this._handleTreeUpdate(rangeChanges, versionId);
456 }
458 > private _rangeTokensAsUpdates(rangeOffset: number, endOffsetToken: EndOffsetToken[], startingOffsetInArray?: number) {
459 const updates: TokenUpdate[] = [];
460 let lastEnd = 0;
474 return updates;
475 }
477 > private _updateTheme() {
478 const modelRange = this._textModel.getFullModelRange();
479 this._markForRefresh(modelRange);
480 this._parseAndTokenizeViewPort(this._visibleLineRanges.get());
481 }
483 > // Was used for inspect editor tokens command
484 > captureAtPosition(lineNumber: number, column: number): QueryCapture[] {
485 const captures = this.captureAtRangeWithInjections(new Range(lineNumber, column, lineNumber, column + 1));
486 return captures;
487 }
489 > // Was used for the colorization tests
490 > captureAtRangeTree(range: Range): QueryCapture[] {
491 const captures = this.captureAtRangeWithInjections(range);
492 return captures;
493 }
495 > private captureAtRange(range: Range): QueryCapture[] {
496 const tree = this._tree.tree.get();
497 if (!tree) {
519 ));
520 }
522 > private captureAtRangeWithInjections(range: Range): QueryCapture[] {
523 const captures: QueryCapture[] = this.captureAtRange(range);
524 for (let i = 0; i < captures.length; i++) {
544 return captures;
545 }
547 > /**
548 > * Gets the tokens for a given line.
549 > * Each token takes 2 elements in the array. The first element is the offset of the end of the token *in the line, not in the document*, and the second element is the metadata.
550 > *
551 > * @param lineNumber
552 > * @returns
553 > */
554 > public tokenizeEncoded(lineNumber: number) {
555 const tokens = this._tokenizeEncoded(lineNumber);
556 if (!tokens) {
562 }
563 }
565 > public tokenizeEncodedInstrumented(lineNumber: number): { result: Uint32Array; captureTime: number; metadataTime: number } | undefined {
566 const tokens = this._tokenizeEncoded(lineNumber);
567 if (!tokens) {
570 return { result: this._endOffsetTokensToUint32Array(tokens.result), captureTime: tokens.captureTime, metadataTime: tokens.metadataTime };
571 }
573 > private _getCaptures(range: Range): QueryCapture[] {
574 const captures = this.captureAtRangeWithInjections(range);
575 return captures;
576 }
578 > private _tokenize(range: Range, rangeStartOffset: number, rangeEndOffset: number): { endOffsetsAndMetadata: { endOffset: number; metadata: number }[]; versionId: number; captureTime: number; metadataTime: number } | undefined {
579 const captures = this._getCaptures(range);
580 const result = this._tokenizeCapturesWithMetadata(captures, rangeStartOffset, rangeEndOffset);
584 return { ...result, versionId: this._tree.treeLastParsedVersion.get() };
585 }
587 > private _createTokensFromCaptures(captures: QueryCapture[], rangeStartOffset: number, rangeEndOffset: number): { endOffsets: EndOffsetAndScopes[]; captureTime: number } | undefined {
588 const tree = this._tree.tree.get();
589 const stopwatch = StopWatch.create();
734 return { endOffsets: endOffsetsAndScopes as { endOffset: number; scopes: string[]; encodedLanguageId: LanguageId }[], captureTime };
735 }
737 > private _getInjectionCaptures(parentCapture: QueryCapture, range: Range): QueryCapture[] {
738 /*
739 const injection = textModelTreeSitter.getInjection(parentCapture.node.startIndex, this._treeSitterModel.languageId);
749 return [];
750 }
752 > private _tokenizeCapturesWithMetadata(captures: QueryCapture[], rangeStartOffset: number, rangeEndOffset: number): { endOffsetsAndMetadata: EndOffsetToken[]; captureTime: number; metadataTime: number } | undefined {
753 const stopwatch = StopWatch.create();
754 const emptyTokens = this._createTokensFromCaptures(captures, rangeStartOffset, rangeEndOffset);
765 return { endOffsetsAndMetadata: endOffsetsAndScopes as { endOffset: number; scopes: string[]; metadata: number }[], captureTime: emptyTokens.captureTime, metadataTime };
766 }
768 > private _tokenizeEncoded(lineNumber: number): { result: EndOffsetToken[]; captureTime: number; metadataTime: number; versionId: number } | undefined {
769 const lineOffset = this._textModel.getOffsetAt({ lineNumber: lineNumber, column: 1 });
770 const maxLine = this._textModel.getLineCount();
778 return { result: result.endOffsetsAndMetadata, captureTime: result.captureTime, metadataTime: result.metadataTime, versionId: result.versionId };
779 }
781 > private _endOffsetTokensToUint32Array(endOffsetsAndMetadata: EndOffsetToken[]): Uint32Array {
782
783 const uint32Array = new Uint32Array(endOffsetsAndMetadata.length * 2);
788 return uint32Array;
789 }
791 >
792 >
793 > interface EndOffsetToken {
794 > endOffset: number;
795 > metadata: number;
796 > }
797 >
798 > interface EndOffsetAndScopes {
799 > endOffset: number;
800 > scopes: string[];
801 > bracket?: number[];
802 > encodedLanguageId: LanguageId;
803 > }
804 >
805 > interface EndOffsetWithMeta extends EndOffsetAndScopes {
806 > metadata?: number;
807 > }
808 > export const TREESITTER_BASE_SCOPES: Record<string, string> = {
809 > 'css': 'source.css',
810 > 'typescript': 'source.ts',
811 > 'ini': 'source.ini',
812 > 'regex': 'source.regex',
813 > };
814 >
815 > const BRACKETS = /[\{\}\[\]\<\>\(\)]/g;
src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts 128 introduced LOC · 40 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- pieceTreeTextBuffer.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { Emitter, Event } from '../../../../base/common/event.js';
7 > import * as strings from '../../../../base/common/strings.js';
8 > import { Position } from '../../core/position.js';
9 > import { Range } from '../../core/range.js';
10 > import { ApplyEditsResult, EndOfLinePreference, FindMatch, IInternalModelContentChange, ISingleEditOperationIdentifier, ITextBuffer, ITextSnapshot, ValidAnnotatedEditOperation, IValidEditOperation, SearchData } from '../../model.js';
11 > import { PieceTreeBase, StringBuffer } from './pieceTreeBase.js';
12 > import { countEOL, StringEOL } from '../../core/misc/eolCounter.js';
13 > import { TextChange } from '../../core/textChange.js';
14 > import { Disposable } from '../../../../base/common/lifecycle.js';
15 >
16 > export interface IValidatedEditOperation {
17 > sortIndex: number;
18 > identifier: ISingleEditOperationIdentifier | null;
19 > range: Range;
20 > rangeOffset: number;
21 > rangeLength: number;
22 > text: string;
23 > eolCount: number;
24 > firstLineLength: number;
25 > lastLineLength: number;
26 > forceMoveMarkers: boolean;
27 > isAutoWhitespaceEdit: boolean;
28 > }
29 >
30 > interface IReverseSingleEditOperation extends IValidEditOperation {
31 > sortIndex: number;
32 > }
33 >
34 > export class PieceTreeTextBuffer extends Disposable implements ITextBuffer {
35 > private _pieceTree: PieceTreeBase;
36 > private readonly _BOM: string;
37 > private _mightContainRTL: boolean;
38 > private _mightContainUnusualLineTerminators: boolean;
39 > private _mightContainNonBasicASCII: boolean;
40 >
41 > private readonly _onDidChangeContent: Emitter<void> = this._register(new Emitter<void>());
42 > public get onDidChangeContent(): Event<void> { return this._onDidChangeContent.event; }
43 >
44 > constructor(chunks: StringBuffer[], BOM: string, eol: '\r\n' | '\n', containsRTL: boolean, containsUnusualLineTerminators: boolean, isBasicASCII: boolean, eolNormalized: boolean) {
45 super();
46 this._BOM = BOM;
50 this._pieceTree = new PieceTreeBase(chunks, eol, eolNormalized);
51 }
53 > // #region TextBuffer
54 > public equals(other: ITextBuffer): boolean {
55 if (!(other instanceof PieceTreeTextBuffer)) {
56 return false;
64 return this._pieceTree.equal(other._pieceTree);
65 }
66 > public mightContainRTL(): boolean { pieceTreeTextBuffer.ts
67 return this._mightContainRTL;
68 }
69 > public mightContainUnusualLineTerminators(): boolean { pieceTreeTextBuffer.ts
70 return this._mightContainUnusualLineTerminators;
71 }
72 > public resetMightContainUnusualLineTerminators(): void { pieceTreeTextBuffer.ts
73 this._mightContainUnusualLineTerminators = false;
74 }
75 > public mightContainNonBasicASCII(): boolean { pieceTreeTextBuffer.ts
76 return this._mightContainNonBasicASCII;
77 }
78 > public getBOM(): string { pieceTreeTextBuffer.ts
79 return this._BOM;
80 }
81 > public getEOL(): '\r\n' | '\n' { pieceTreeTextBuffer.ts
82 return this._pieceTree.getEOL();
83 }
85 > public createSnapshot(preserveBOM: boolean): ITextSnapshot {
86 return this._pieceTree.createSnapshot(preserveBOM ? this._BOM : '');
87 }
89 > public getOffsetAt(lineNumber: number, column: number): number {
90 return this._pieceTree.getOffsetAt(lineNumber, column);
91 }
93 > public getPositionAt(offset: number): Position {
94 return this._pieceTree.getPositionAt(offset);
95 }
97 > public getRangeAt(start: number, length: number): Range {
98 const end = start + length;
99 const startPosition = this.getPositionAt(start);
101 return new Range(startPosition.lineNumber, startPosition.column, endPosition.lineNumber, endPosition.column);
102 }
104 > public getValueInRange(range: Range, eol: EndOfLinePreference = EndOfLinePreference.TextDefined): string {
105 if (range.isEmpty()) {
106 return '';
110 return this._pieceTree.getValueInRange(range, lineEnding);
111 }
113 > public getValueLengthInRange(range: Range, eol: EndOfLinePreference = EndOfLinePreference.TextDefined): number {
114 if (range.isEmpty()) {
115 return 0;
136 return endOffset - startOffset + eolOffsetCompensation;
137 }
139 > public getCharacterCountInRange(range: Range, eol: EndOfLinePreference = EndOfLinePreference.TextDefined): number {
140 if (this._mightContainNonBasicASCII) {
141 // we must count by iterating
167 return this.getValueLengthInRange(range, eol);
168 }
170 > public getNearestChunk(offset: number): string {
171 return this._pieceTree.getNearestChunk(offset);
172 }
174 > public getLength(): number {
175 return this._pieceTree.getLength();
176 }
178 > public getLineCount(): number {
179 return this._pieceTree.getLineCount();
180 }
182 > public getLinesContent(): string[] {
183 return this._pieceTree.getLinesContent();
184 }
186 > public getLineContent(lineNumber: number): string {
187 return this._pieceTree.getLineContent(lineNumber);
188 }
190 > public getLineCharCode(lineNumber: number, index: number): number {
191 return this._pieceTree.getLineCharCode(lineNumber, index);
192 }
194 > public getCharCode(offset: number): number {
195 return this._pieceTree.getCharCode(offset);
196 }
198 > public getLineLength(lineNumber: number): number {
199 return this._pieceTree.getLineLength(lineNumber);
200 }
202 > public getLineMinColumn(lineNumber: number): number {
203 return 1;
204 }
206 > public getLineMaxColumn(lineNumber: number): number {
207 return this.getLineLength(lineNumber) + 1;
208 }
210 > public getLineFirstNonWhitespaceColumn(lineNumber: number): number {
211 const result = strings.firstNonWhitespaceIndex(this.getLineContent(lineNumber));
212 if (result === -1) {
215 return result + 1;
216 }
218 > public getLineLastNonWhitespaceColumn(lineNumber: number): number {
219 const result = strings.lastNonWhitespaceIndex(this.getLineContent(lineNumber));
220 if (result === -1) {
223 return result + 2;
224 }
226 > private _getEndOfLine(eol: EndOfLinePreference): string {
227 switch (eol) {
228 case EndOfLinePreference.LF:
236 }
237 }
239 > public setEOL(newEOL: '\r\n' | '\n'): void {
240 this._pieceTree.setEOL(newEOL);
241 }
243 > public applyEdits(rawOperations: ValidAnnotatedEditOperation[], recordTrimAutoWhitespace: boolean, computeUndoEdits: boolean): ApplyEditsResult {
244 let mightContainRTL = this._mightContainRTL;
245 let mightContainUnusualLineTerminators = this._mightContainUnusualLineTerminators;
413 );
414 }
416 > /**
417 > * Transform operations such that they represent the same logic edit,
418 > * but that they also do not cause OOM crashes.
419 > */
420 > private _reduceOperations(operations: IValidatedEditOperation[]): IValidatedEditOperation[] {
421 if (operations.length < 1000) {
422 // We know from empirical testing that a thousand edits work fine regardless of their shape.
431 return [this._toSingleEditOperation(operations)];
432 }
434 > _toSingleEditOperation(operations: IValidatedEditOperation[]): IValidatedEditOperation {
435 let forceMoveMarkers = false;
436 const firstEditRange = operations[0].range;
476 };
477 }
479 > private _doApplyEdits(operations: IValidatedEditOperation[]): IInternalModelContentChange[] {
480 operations.sort(PieceTreeTextBuffer._sortOpsDescending);
481
517 return contentChanges;
518 }
520 > findMatchesLineByLine(searchRange: Range, searchData: SearchData, captureMatches: boolean, limitResultCount: number): FindMatch[] {
521 return this._pieceTree.findMatchesLineByLine(searchRange, searchData, captureMatches, limitResultCount);
522 }
524 > // #endregion
525 >
526 > // #region helper
527 > // testing purpose.
528 > public getPieceTree(): PieceTreeBase {
529 return this._pieceTree;
530 }
532 > public static _getInverseEditRange(range: Range, text: string) {
533 const startLineNumber = range.startLineNumber;
534 const startColumn = range.startColumn;
554 return resultRange;
555 }
557 > /**
558 > * Assumes `operations` are validated and sorted ascending
559 > */
560 > public static _getInverseEditRanges(operations: IValidatedEditOperation[]): Range[] {
561 const result: Range[] = [];
562
610 return result;
611 }
613 > private static _sortOpsAscending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
614 const r = Range.compareRangesUsingEnds(a.range, b.range);
615 if (r === 0) {
618 return r;
619 }
621 > private static _sortOpsDescending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
622 const r = Range.compareRangesUsingEnds(a.range, b.range);
623 if (r === 0) {
src/vs/editor/common/model/tokens/tokenizationTextModelPart.ts 124 introduced LOC · 29 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- tokenizationTextModelPart.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { CharCode } from '../../../../base/common/charCode.js';
7 > import { BugIndicatingError } from '../../../../base/common/errors.js';
8 > import { Emitter, Event } from '../../../../base/common/event.js';
9 > import { countEOL } from '../../core/misc/eolCounter.js';
10 > import { IPosition, Position } from '../../core/position.js';
11 > import { Range } from '../../core/range.js';
12 > import { IWordAtPosition, getWordAtText } from '../../core/wordHelper.js';
13 > import { StandardTokenType } from '../../encodedTokenAttributes.js';
14 > import { ILanguageService } from '../../languages/language.js';
15 > import { ILanguageConfigurationService, LanguageConfigurationServiceChangeEvent, ResolvedLanguageConfiguration } from '../../languages/languageConfigurationRegistry.js';
16 > import { BracketPairsTextModelPart } from '../bracketPairsTextModelPart/bracketPairsImpl.js';
17 > import { TextModel } from '../textModel.js';
18 > import { TextModelPart } from '../textModelPart.js';
19 > import { AbstractSyntaxTokenBackend, AttachedViews } from './abstractSyntaxTokenBackend.js';
20 > import { TreeSitterSyntaxTokenBackend } from './treeSitter/treeSitterSyntaxTokenBackend.js';
21 > import { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent, IModelFontTokensChangedEvent } from '../../textModelEvents.js';
22 > import { ITokenizationTextModelPart } from '../../tokenizationTextModelPart.js';
23 > import { LineTokens } from '../../tokens/lineTokens.js';
24 > import { SparseMultilineTokens } from '../../tokens/sparseMultilineTokens.js';
25 > import { SparseTokensStore } from '../../tokens/sparseTokensStore.js';
26 > import { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js';
27 > import { TokenizerSyntaxTokenBackend } from './tokenizerSyntaxTokenBackend.js';
28 > import { ITreeSitterLibraryService } from '../../services/treeSitter/treeSitterLibraryService.js';
29 > import { derived, IObservable, ISettableObservable, observableValue } from '../../../../base/common/observable.js';
30 >
31 > export class TokenizationTextModelPart extends TextModelPart implements ITokenizationTextModelPart {
32 > private readonly _semanticTokens: SparseTokensStore;
33 >
34 > private readonly _onDidChangeLanguage: Emitter<IModelLanguageChangedEvent>;
35 > public readonly onDidChangeLanguage: Event<IModelLanguageChangedEvent>;
36 >
37 > private readonly _onDidChangeLanguageConfiguration: Emitter<IModelLanguageConfigurationChangedEvent>;
38 > public readonly onDidChangeLanguageConfiguration: Event<IModelLanguageConfigurationChangedEvent>;
39 >
40 > private readonly _onDidChangeTokens: Emitter<IModelTokensChangedEvent>;
41 > public readonly onDidChangeTokens: Event<IModelTokensChangedEvent>;
42 >
43 > private readonly _onDidChangeFontTokens: Emitter<IModelFontTokensChangedEvent> = this._register(new Emitter<IModelFontTokensChangedEvent>());
44 > public readonly onDidChangeFontTokens: Event<IModelFontTokensChangedEvent> = this._onDidChangeFontTokens.event;
45 >
46 > public readonly tokens: IObservable<AbstractSyntaxTokenBackend>;
47 > private readonly _useTreeSitter: IObservable<boolean>;
48 > private readonly _languageIdObs: ISettableObservable<string>;
49 >
50 > constructor(
51 private readonly _textModel: TextModel,
52 private readonly _bracketPairsTextModelPart: BracketPairsTextModelPart,
116 this.onDidChangeFontTokens = this._onDidChangeFontTokens.event;
117 }
119 > _hasListeners(): boolean {
120 // Note: _onDidChangeFontTokens is intentionally excluded because it's an internal event
121 // that TokenizationFontDecorationProvider subscribes to during TextModel construction
124 || this._onDidChangeTokens.hasListeners());
125 }
127 > public handleLanguageConfigurationServiceChange(e: LanguageConfigurationServiceChangeEvent): void {
128 if (e.affects(this._languageId)) {
129 this._onDidChangeLanguageConfiguration.fire({});
130 }
131 }
133 > public handleDidChangeContent(e: IModelContentChangedEvent): void {
134 if (e.isFlush) {
135 this._semanticTokens.flush();
150 this.tokens.get().handleDidChangeContent(e);
151 }
153 > public handleDidChangeAttached(): void {
154 this.tokens.get().handleDidChangeAttached();
155 }
157 > /**
158 > * Includes grammar and semantic tokens.
159 > */
160 > public getLineTokens(lineNumber: number): LineTokens {
161 this.validateLineNumber(lineNumber);
162 const syntacticTokens = this.tokens.get().getLineTokens(lineNumber);
163 return this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);
164 }
166 > private _emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void {
167 if (!this._textModel._isDisposing()) {
168 this._bracketPairsTextModelPart.handleDidChangeTokens(e);
170 }
171 }
173 > // #region Grammar Tokens
174 >
175 > private validateLineNumber(lineNumber: number): void {
176 if (lineNumber < 1 || lineNumber > this._textModel.getLineCount()) {
177 throw new BugIndicatingError('Illegal value for lineNumber');
178 }
179 }
181 > public get hasTokens(): boolean {
182 return this.tokens.get().hasTokens;
183 }
185 > public resetTokenization() {
186 this.tokens.get().todo_resetTokenization();
187 }
189 > public get backgroundTokenizationState() {
190 return this.tokens.get().backgroundTokenizationState;
191 }
193 > public forceTokenization(lineNumber: number): void {
194 this.validateLineNumber(lineNumber);
195 this.tokens.get().forceTokenization(lineNumber);
196 }
198 > public hasAccurateTokensForLine(lineNumber: number): boolean {
199 this.validateLineNumber(lineNumber);
200 return this.tokens.get().hasAccurateTokensForLine(lineNumber);
201 }
203 > public isCheapToTokenize(lineNumber: number): boolean {
204 this.validateLineNumber(lineNumber);
205 return this.tokens.get().isCheapToTokenize(lineNumber);
206 }
208 > public tokenizeIfCheap(lineNumber: number): void {
209 this.validateLineNumber(lineNumber);
210 this.tokens.get().tokenizeIfCheap(lineNumber);
211 }
213 > public getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {
214 return this.tokens.get().getTokenTypeIfInsertingCharacter(lineNumber, column, character);
215 }
217 > public tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {
218 return this.tokens.get().tokenizeLinesAt(lineNumber, lines);
219 }
221 > // #endregion
222 >
223 > // #region Semantic Tokens
224 >
225 > public setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void {
226 this._semanticTokens.set(tokens, isComplete, this._textModel);
227
231 });
232 }
234 > public hasCompleteSemanticTokens(): boolean {
235 return this._semanticTokens.isComplete();
236 }
238 > public hasSomeSemanticTokens(): boolean {
239 return !this._semanticTokens.isEmpty();
240 }
242 > public setPartialSemanticTokens(range: Range, tokens: SparseMultilineTokens[]): void {
243 if (this.hasCompleteSemanticTokens()) {
244 return;
258 });
259 }
261 > // #endregion
262 >
263 > // #region Utility Methods
264 >
265 > public getWordAtPosition(_position: IPosition): IWordAtPosition | null {
266 this.assertNotDisposed();
267
313 return null;
314 }
316 > private getLanguageConfiguration(languageId: string): ResolvedLanguageConfiguration {
317 return this._languageConfigurationService.getLanguageConfiguration(languageId);
318 }
320 > private static _findLanguageBoundaries(lineTokens: LineTokens, tokenIndex: number): [number, number] {
321 const languageId = lineTokens.getLanguageId(tokenIndex);
322
339 return [startOffset, endOffset];
340 }
342 > public getWordUntilPosition(position: IPosition): IWordAtPosition {
343 const wordAtPosition = this.getWordAtPosition(position);
344 if (!wordAtPosition) {
351 };
352 }
354 > // #endregion
355 >
356 > // #region Language Id handling
357 >
358 > public getLanguageId(): string {
359 return this._languageId;
360 }
362 > public getLanguageIdAtPosition(lineNumber: number, column: number): string {
363 const position = this._textModel.validatePosition(new Position(lineNumber, column));
364 const lineTokens = this.getLineTokens(position.lineNumber);
365 return lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1));
366 }
368 > public setLanguageId(languageId: string, source: string = 'api'): void {
369 if (this._languageId === languageId) {
370 // There's nothing to do
src/vs/editor/common/model/tokens/abstractSyntaxTokenBackend.ts 116 introduced LOC · 14 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- abstractSyntaxTokenBackend.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { equals } from '../../../../base/common/arrays.js';
7 > import { RunOnceScheduler } from '../../../../base/common/async.js';
8 > import { Emitter, Event } from '../../../../base/common/event.js';
9 > import { Disposable, IDisposable } from '../../../../base/common/lifecycle.js';
10 > import { LineRange } from '../../core/ranges/lineRange.js';
11 > import { StandardTokenType } from '../../encodedTokenAttributes.js';
12 > import { ILanguageIdCodec } from '../../languages.js';
13 > import { IAttachedView } from '../../model.js';
14 > import { TextModel } from '../textModel.js';
15 > import { IModelContentChangedEvent, IModelTokensChangedEvent, IModelFontTokensChangedEvent } from '../../textModelEvents.js';
16 > import { BackgroundTokenizationState } from '../../tokenizationTextModelPart.js';
17 > import { LineTokens } from '../../tokens/lineTokens.js';
18 > import { derivedOpts, IObservable, ISettableObservable, observableSignal, observableValueOpts } from '../../../../base/common/observable.js';
19 > import { equalsIfDefinedC, thisEqualsC, arrayEqualsC } from '../../../../base/common/equals.js';
20 >
21 > /**
22 > * @internal
23 > */
24 > export class AttachedViews implements IDisposable {
25 > private readonly _onDidChangeVisibleRanges = new Emitter<{ view: IAttachedView; state: AttachedViewState | undefined }>();
26 > public readonly onDidChangeVisibleRanges = this._onDidChangeVisibleRanges.event;
27 >
28 > private readonly _views = new Set<AttachedViewImpl>();
29 > private readonly _viewsChanged = observableSignal(this);
30 >
31 > public readonly visibleLineRanges: IObservable<readonly LineRange[]>;
32 >
33 > constructor() {
34 this.visibleLineRanges = derivedOpts({
35 owner: this,
43 });
44 }
46 > public attachView(): IAttachedView {
47 const view = new AttachedViewImpl((state) => {
48 this._onDidChangeVisibleRanges.fire({ view, state });
52 return view;
53 }
55 > public detachView(view: IAttachedView): void {
56 this._views.delete(view as AttachedViewImpl);
57 this._onDidChangeVisibleRanges.fire({ view, state: undefined });
58 this._viewsChanged.trigger(undefined);
59 }
61 > public dispose(): void {
62 this._onDidChangeVisibleRanges.dispose();
63 }
65 >
66 > /**
67 > * @internal
68 > */
69 > export class AttachedViewState {
70 > constructor(
71 readonly visibleLineRanges: readonly LineRange[],
72 readonly stabilized: boolean,
73 ) { }
75 > public equals(other: AttachedViewState): boolean {
76 if (this === other) {
77 return true;
85 return true;
86 }
88 >
89 > class AttachedViewImpl implements IAttachedView {
90 > private readonly _state: ISettableObservable<AttachedViewState | undefined>;
91 > public get state(): IObservable<AttachedViewState | undefined> { return this._state; }
92 >
93 > constructor(
94 private readonly handleStateChange: (state: AttachedViewState) => void
95 ) {
96 this._state = observableValueOpts<AttachedViewState | undefined>({ owner: this, equalsFn: equalsIfDefinedC((a, b) => a.equals(b)) }, undefined);
97 }
99 > setVisibleLines(visibleLines: { startLineNumber: number; endLineNumber: number }[], stabilized: boolean): void {
100 const visibleLineRanges = visibleLines.map((line) => new LineRange(line.startLineNumber, line.endLineNumber + 1));
101 const state = new AttachedViewState(visibleLineRanges, stabilized);
103 this.handleStateChange(state);
104 }
106 >
107 >
108 > export class AttachedViewHandler extends Disposable {
109 > private readonly runner = this._register(new RunOnceScheduler(() => this.update(), 50));
110 >
111 > private _computedLineRanges: readonly LineRange[] = [];
112 > private _lineRanges: readonly LineRange[] = [];
113 > public get lineRanges(): readonly LineRange[] { return this._lineRanges; }
114 >
115 > constructor(private readonly _refreshTokens: () => void) {
116 super();
117 }
119 > private update(): void {
120 if (equals(this._computedLineRanges, this._lineRanges, (a, b) => a.equals(b))) {
121 return;
124 this._refreshTokens();
125 }
127 > public handleStateChange(state: AttachedViewState): void {
128 this._lineRanges = state.visibleLineRanges;
129 if (state.stabilized) {
134 }
135 }
137 >
138 > export abstract class AbstractSyntaxTokenBackend extends Disposable {
139 > protected abstract _backgroundTokenizationState: BackgroundTokenizationState;
140 > public get backgroundTokenizationState(): BackgroundTokenizationState {
141 > return this._backgroundTokenizationState;
142 > }
143 >
144 > protected abstract readonly _onDidChangeBackgroundTokenizationState: Emitter<void>;
145 > /** @internal, should not be exposed by the text model! */
146 > public abstract readonly onDidChangeBackgroundTokenizationState: Event<void>;
147 >
148 > protected readonly _onDidChangeTokens = this._register(new Emitter<IModelTokensChangedEvent>());
149 > /** @internal, should not be exposed by the text model! */
150 > public readonly onDidChangeTokens: Event<IModelTokensChangedEvent> = this._onDidChangeTokens.event;
151 >
152 > protected readonly _onDidChangeFontTokens: Emitter<IModelFontTokensChangedEvent> = this._register(new Emitter<IModelFontTokensChangedEvent>());
153 > /** @internal, should not be exposed by the text model! */
154 > public readonly onDidChangeFontTokens: Event<IModelFontTokensChangedEvent> = this._onDidChangeFontTokens.event;
155 >
156 > constructor(
157 protected readonly _languageIdCodec: ILanguageIdCodec,
158 protected readonly _textModel: TextModel,
160 super();
161 }
163 > public abstract todo_resetTokenization(fireTokenChangeEvent?: boolean): void;
164 >
165 > public abstract handleDidChangeAttached(): void;
166 >
167 > public abstract handleDidChangeContent(e: IModelContentChangedEvent): void;
168 >
169 > public abstract forceTokenization(lineNumber: number): void;
170 >
171 > public abstract hasAccurateTokensForLine(lineNumber: number): boolean;
172 >
173 > public abstract isCheapToTokenize(lineNumber: number): boolean;
174 >
175 > public tokenizeIfCheap(lineNumber: number): void {
176 if (this.isCheapToTokenize(lineNumber)) {
177 this.forceTokenization(lineNumber);
178 }
179 }
181 > public abstract getLineTokens(lineNumber: number): LineTokens;
182 >
183 > public abstract getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType;
184 >
185 > public abstract tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null;
186 >
187 > public abstract get hasTokens(): boolean;
188 > }
src/vs/editor/common/tokenizationTextModelPart.ts 102 introduced LOC · 1 range

Open complete file

1 > /*--------------------------------------------------------------------------------------------- tokenizationTextModelPart.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { Range } from './core/range.js';
7 > import { StandardTokenType } from './encodedTokenAttributes.js';
8 > import { LineTokens } from './tokens/lineTokens.js';
9 > import { SparseMultilineTokens } from './tokens/sparseMultilineTokens.js';
10 >
11 > /**
12 > * Provides tokenization related functionality of the text model.
13 > */
14 > export interface ITokenizationTextModelPart {
15 > readonly hasTokens: boolean;
16 >
17 > /**
18 > * Replaces all semantic tokens with the provided `tokens`.
19 > * @internal
20 > */
21 > setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void;
22 >
23 > /**
24 > * Merges the provided semantic tokens into existing semantic tokens.
25 > * @internal
26 > */
27 > setPartialSemanticTokens(range: Range, tokens: SparseMultilineTokens[] | null): void;
28 >
29 > /**
30 > * @internal
31 > */
32 > hasCompleteSemanticTokens(): boolean;
33 >
34 > /**
35 > * @internal
36 > */
37 > hasSomeSemanticTokens(): boolean;
38 >
39 > /**
40 > * Flush all tokenization state.
41 > * @internal
42 > */
43 > resetTokenization(): void;
44 >
45 > /**
46 > * Force tokenization information for `lineNumber` to be accurate.
47 > * @internal
48 > */
49 > forceTokenization(lineNumber: number): void;
50 >
51 > /**
52 > * If it is cheap, force tokenization information for `lineNumber` to be accurate.
53 > * This is based on a heuristic.
54 > * @internal
55 > */
56 > tokenizeIfCheap(lineNumber: number): void;
57 >
58 > /**
59 > * Check if tokenization information is accurate for `lineNumber`.
60 > * @internal
61 > */
62 > hasAccurateTokensForLine(lineNumber: number): boolean;
63 >
64 > /**
65 > * Check if calling `forceTokenization` for this `lineNumber` will be cheap (time-wise).
66 > * This is based on a heuristic.
67 > * @internal
68 > */
69 > isCheapToTokenize(lineNumber: number): boolean;
70 >
71 > /**
72 > * Get the tokens for the line `lineNumber`.
73 > * The tokens might be inaccurate. Use `forceTokenization` to ensure accurate tokens.
74 > * @internal
75 > */
76 > getLineTokens(lineNumber: number): LineTokens;
77 >
78 > /**
79 > * Returns the standard token type for a character if the character were to be inserted at
80 > * the given position. If the result cannot be accurate, it returns null.
81 > * @internal
82 > */
83 > getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType;
84 >
85 > /**
86 > * Tokens the lines as if they were inserted at [lineNumber, lineNumber).
87 > * @internal
88 > */
89 > tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null;
90 >
91 > getLanguageId(): string;
92 > getLanguageIdAtPosition(lineNumber: number, column: number): string;
93 >
94 > setLanguageId(languageId: string, source?: string): void;
95 >
96 > readonly backgroundTokenizationState: BackgroundTokenizationState;
97 > }
98 >
99 > export const enum BackgroundTokenizationState {
100 > InProgress = 1,
101 > Completed = 2,
102 > }
src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsImpl.ts 96 introduced LOC · 25 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- bracketPairsImpl.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { CallbackIterable, compareBy } from '../../../../base/common/arrays.js';
7 > import { Emitter } from '../../../../base/common/event.js';
8 > import { Disposable, DisposableStore, IDisposable, IReference, MutableDisposable } from '../../../../base/common/lifecycle.js';
9 > import { IPosition, Position } from '../../core/position.js';
10 > import { Range } from '../../core/range.js';
11 > import { ILanguageConfigurationService, LanguageConfigurationServiceChangeEvent } from '../../languages/languageConfigurationRegistry.js';
12 > import { ignoreBracketsInToken } from '../../languages/supports.js';
13 > import { LanguageBracketsConfiguration } from '../../languages/supports/languageBracketsConfiguration.js';
14 > import { BracketsUtils, RichEditBracket, RichEditBrackets } from '../../languages/supports/richEditBrackets.js';
15 > import { BracketPairsTree } from './bracketPairsTree/bracketPairsTree.js';
16 > import { TextModel } from '../textModel.js';
17 > import { BracketInfo, BracketPairInfo, BracketPairWithMinIndentationInfo, IBracketPairsTextModelPart, IFoundBracket } from '../../textModelBracketPairs.js';
18 > import { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent } from '../../textModelEvents.js';
19 > import { LineTokens } from '../../tokens/lineTokens.js';
20 >
21 > export class BracketPairsTextModelPart extends Disposable implements IBracketPairsTextModelPart {
22 > private readonly bracketPairsTree = this._register(new MutableDisposable<IReference<BracketPairsTree>>());
23 >
24 > private readonly onDidChangeEmitter = this._register(new Emitter<void>());
25 > public readonly onDidChange = this.onDidChangeEmitter.event;
26 >
27 > private get canBuildAST() {
28 > const maxSupportedDocumentLength = /* max lines */ 50_000 * /* average column count */ 100;
29 > return this.textModel.getValueLength() <= maxSupportedDocumentLength;
30 > }
31 >
32 > private bracketsRequested = false;
33 >
34 > public constructor(
35 private readonly textModel: TextModel,
36 private readonly languageConfigurationService: ILanguageConfigurationService
38 super();
39 }
41 > //#region TextModel events
42 >
43 > public handleLanguageConfigurationServiceChange(e: LanguageConfigurationServiceChangeEvent): void {
44 if (!e.languageId || this.bracketPairsTree.value?.object.didLanguageChange(e.languageId)) {
45 this.bracketPairsTree.clear();
47 }
48 }
50 > public handleDidChangeOptions(e: IModelOptionsChangedEvent): void {
51 this.bracketPairsTree.clear();
52 this.updateBracketPairsTree();
53 }
55 > public handleDidChangeLanguage(e: IModelLanguageChangedEvent): void {
56 this.bracketPairsTree.clear();
57 this.updateBracketPairsTree();
58 }
60 > public handleDidChangeContent(change: IModelContentChangedEvent) {
61 this.bracketPairsTree.value?.object.handleContentChanged(change);
62 }
64 > public handleDidChangeBackgroundTokenizationState(): void {
65 this.bracketPairsTree.value?.object.handleDidChangeBackgroundTokenizationState();
66 }
68 > public handleDidChangeTokens(e: IModelTokensChangedEvent): void {
69 this.bracketPairsTree.value?.object.handleDidChangeTokens(e);
70 }
72 > //#endregion
73 >
74 > private updateBracketPairsTree() {
75 if (this.bracketsRequested && this.canBuildAST) {
76 if (!this.bracketPairsTree.value) {
96 }
97 }
99 > /**
100 > * Returns all bracket pairs that intersect the given range.
101 > * The result is sorted by the start position.
102 > */
103 > public getBracketPairsInRange(range: Range): CallbackIterable<BracketPairInfo> {
104 this.bracketsRequested = true;
105 this.updateBracketPairsTree();
106 return this.bracketPairsTree.value?.object.getBracketPairsInRange(range, false) || CallbackIterable.empty;
107 }
109 > public getBracketPairsInRangeWithMinIndentation(range: Range): CallbackIterable<BracketPairWithMinIndentationInfo> {
110 this.bracketsRequested = true;
111 this.updateBracketPairsTree();
112 return this.bracketPairsTree.value?.object.getBracketPairsInRange(range, true) || CallbackIterable.empty;
113 }
115 > public getBracketsInRange(range: Range, onlyColorizedBrackets: boolean = false): CallbackIterable<BracketInfo> {
116 this.bracketsRequested = true;
117 this.updateBracketPairsTree();
118 return this.bracketPairsTree.value?.object.getBracketsInRange(range, onlyColorizedBrackets) || CallbackIterable.empty;
119 }
121 > public findMatchingBracketUp(_bracket: string, _position: IPosition, maxDuration?: number): Range | null {
122 const position = this.textModel.validatePosition(_position);
123 const languageId = this.textModel.getLanguageIdAtPosition(position.lineNumber, position.column);
159 }
160 }
162 > public matchBracket(position: IPosition, maxDuration?: number): [Range, Range] | null {
163 if (this.canBuildAST) {
164 const bracketPair =
189 }
190 }
192 > private _establishBracketSearchOffsets(position: Position, lineTokens: LineTokens, modeBrackets: RichEditBrackets, tokenIndex: number) {
193 const tokenCount = lineTokens.getCount();
194 const currentLanguageId = lineTokens.getLanguageId(tokenIndex);
222 return { searchStartOffset, searchEndOffset };
223 }
225 > private _matchBracket(position: Position, continueSearchPredicate: ContinueBracketSearchPredicate): [Range, Range] | null {
226 const lineNumber = position.lineNumber;
227 const lineTokens = this.textModel.tokenization.getLineTokens(lineNumber);
297 return null;
298 }
300 > private _matchFoundBracket(foundBracket: Range, data: RichEditBracket, isOpen: boolean, continueSearchPredicate: ContinueBracketSearchPredicate): [Range, Range] | null | BracketSearchCanceled {
301 if (!data) {
302 return null;
319 return [foundBracket, matched];
320 }
322 > private _findMatchingBracketUp(bracket: RichEditBracket, position: Position, continueSearchPredicate: ContinueBracketSearchPredicate): Range | null | BracketSearchCanceled {
323 // console.log('_findMatchingBracketUp: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));
324
406 return null;
407 }
409 > private _findMatchingBracketDown(bracket: RichEditBracket, position: Position, continueSearchPredicate: ContinueBracketSearchPredicate): Range | null | BracketSearchCanceled {
410 // console.log('_findMatchingBracketDown: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));
411
494 return null;
495 }
497 > public findPrevBracket(_position: IPosition): IFoundBracket | null {
498 const position = this.textModel.validatePosition(_position);
499
580 return null;
581 }
583 > public findNextBracket(_position: IPosition): IFoundBracket | null {
584 const position = this.textModel.validatePosition(_position);
585
667 return null;
668 }
670 > public findEnclosingBrackets(_position: IPosition, maxDuration?: number): [Range, Range] | null {
671 const position = this.textModel.validatePosition(_position);
672
803 return null;
804 }
806 > private _toFoundBracket(bracketConfig: LanguageBracketsConfiguration, r: Range): IFoundBracket | null {
807 if (!r) {
808 return null;
822 };
823 }
825 >
826 function createDisposableRef<T>(object: T, disposable?: IDisposable): IReference<T> {
827 return {
830 };
831 }
833 > type ContinueBracketSearchPredicate = (() => boolean);
834 >
835 function createTimeBasedContinueBracketSearchPredicate(maxDuration: number | undefined): ContinueBracketSearchPredicate {
836 if (typeof maxDuration === 'undefined') {
843 }
844 }
846 > class BracketSearchCanceled {
847 > public static INSTANCE = new BracketSearchCanceled();
848 > _searchCanceledBrand = undefined;
849 > private constructor() { }
850 > }
851 >
852 function stripBracketSearchCanceled<T>(result: T | null | BracketSearchCanceled): T | null {
853 if (result instanceof BracketSearchCanceled) {
src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/bracketPairsTree.ts 89 introduced LOC · 16 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- bracketPairsTree.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { Emitter } from '../../../../../base/common/event.js';
7 > import { Disposable } from '../../../../../base/common/lifecycle.js';
8 > import { Range } from '../../../core/range.js';
9 > import { ITextModel } from '../../../model.js';
10 > import { BracketInfo, BracketPairWithMinIndentationInfo, IFoundBracket } from '../../../textModelBracketPairs.js';
11 > import { TextModel } from '../../textModel.js';
12 > import { IModelContentChangedEvent, IModelTokensChangedEvent } from '../../../textModelEvents.js';
13 > import { ResolvedLanguageConfiguration } from '../../../languages/languageConfigurationRegistry.js';
14 > import { AstNode, AstNodeKind } from './ast.js';
15 > import { TextEditInfo } from './beforeEditPositionMapper.js';
16 > import { LanguageAgnosticBracketTokens } from './brackets.js';
17 > import { Length, lengthAdd, lengthGreaterThanEqual, lengthLessThan, lengthLessThanEqual, lengthsToRange, lengthZero, positionToLength, toLength } from './length.js';
18 > import { parseDocument } from './parser.js';
19 > import { DenseKeyProvider } from './smallImmutableSet.js';
20 > import { FastTokenizer, TextBufferTokenizer } from './tokenizer.js';
21 > import { BackgroundTokenizationState } from '../../../tokenizationTextModelPart.js';
22 > import { Position } from '../../../core/position.js';
23 > import { CallbackIterable } from '../../../../../base/common/arrays.js';
24 > import { combineTextEditInfos } from './combineTextEditInfos.js';
25 > import { ClosingBracketKind, OpeningBracketKind } from '../../../languages/supports/languageBracketsConfiguration.js';
26 >
27 > export class BracketPairsTree extends Disposable {
28 > private readonly didChangeEmitter;
29 >
30 > /*
31 > There are two trees:
32 > * The initial tree that has no token information and is used for performant initial bracket colorization.
33 > * The tree that used token information to detect bracket pairs.
34 >
35 > To prevent flickering, we only switch from the initial tree to tree with token information
36 > when tokenization completes.
37 > Since the text can be edited while background tokenization is in progress, we need to update both trees.
38 > */
39 > private initialAstWithoutTokens: AstNode | undefined;
40 > private astWithTokens: AstNode | undefined;
41 >
42 > private readonly denseKeyProvider;
43 > private readonly brackets;
44 >
45 > public didLanguageChange(languageId: string): boolean {
46 > return this.brackets.didLanguageChange(languageId);
47 > }
48 >
49 > public readonly onDidChange;
50 > private queuedTextEditsForInitialAstWithoutTokens: TextEditInfo[];
51 > private queuedTextEdits: TextEditInfo[];
52 >
53 > public constructor(
54 private readonly textModel: TextModel,
55 private readonly getLanguageConfiguration: (languageId: string) => ResolvedLanguageConfiguration
79 }
80 }
82 > //#region TextModel events
83 >
84 > public handleDidChangeBackgroundTokenizationState(): void {
85 if (this.textModel.tokenization.backgroundTokenizationState === BackgroundTokenizationState.Completed) {
86 const wasUndefined = this.initialAstWithoutTokens === undefined;
92 }
93 }
95 > public handleDidChangeTokens({ ranges }: IModelTokensChangedEvent): void {
96 const edits = ranges.map(r =>
97 new TextEditInfo(
108 }
109 }
111 > public handleContentChanged(change: IModelContentChangedEvent) {
112 const edits = TextEditInfo.fromModelContentChanges(change.changes);
113 this.handleEdits(edits, false);
114 }
116 > private handleEdits(edits: TextEditInfo[], tokenChange: boolean): void {
117 // Lazily queue the edits and only apply them when the tree is accessed.
118 const result = combineTextEditInfos(this.queuedTextEdits, edits);
123 }
124 }
126 > //#endregion
127 >
128 > private flushQueue() {
129 if (this.queuedTextEdits.length > 0) {
130 this.astWithTokens = this.parseDocumentFromTextBuffer(this.queuedTextEdits, this.astWithTokens, false);
138 }
139 }
141 > /**
142 > * @pure (only if isPure = true)
143 > */
144 > private parseDocumentFromTextBuffer(edits: TextEditInfo[], previousAst: AstNode | undefined, immutable: boolean): AstNode {
145 // Is much faster if `isPure = false`.
146 const isPure = false;
150 return result;
151 }
153 > public getBracketsInRange(range: Range, onlyColorizedBrackets: boolean): CallbackIterable<BracketInfo> {
154 this.flushQueue();
155
161 });
162 }
164 > public getBracketPairsInRange(range: Range, includeMinIndentation: boolean): CallbackIterable<BracketPairWithMinIndentationInfo> {
165 this.flushQueue();
166
174 });
175 }
177 > public getFirstBracketAfter(position: Position): IFoundBracket | null {
178 this.flushQueue();
179
181 return getFirstBracketAfter(node, lengthZero, node.length, positionToLength(position));
182 }
184 > public getFirstBracketBefore(position: Position): IFoundBracket | null {
185 this.flushQueue();
186
188 return getFirstBracketBefore(node, lengthZero, node.length, positionToLength(position));
189 }
191 >
192 function getFirstBracketBefore(node: AstNode, nodeOffsetStart: Length, nodeOffsetEnd: Length, position: Length): IFoundBracket | null {
193 if (node.kind === AstNodeKind.List || node.kind === AstNodeKind.Pair) {
219 return null;
220 }
222 function getFirstBracketAfter(node: AstNode, nodeOffsetStart: Length, nodeOffsetEnd: Length, position: Length): IFoundBracket | null {
223 if (node.kind === AstNodeKind.List || node.kind === AstNodeKind.Pair) {
244 return null;
245 }
247 function collectBrackets(
248 node: AstNode,
373 }
374 }
376 > class CollectBracketPairsContext {
377 > constructor(
378 public readonly push: (item: BracketPairWithMinIndentationInfo) => boolean,
379 public readonly includeMinIndentation: boolean,
381 ) {
382 }
384 >
385 function collectBracketPairs(
386 node: AstNode,
src/vs/editor/common/textModelBracketPairs.ts 88 introduced LOC · 6 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- textModelBracketPairs.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { CallbackIterable } from '../../base/common/arrays.js';
7 > import { Event } from '../../base/common/event.js';
8 > import { IPosition } from './core/position.js';
9 > import { IRange, Range } from './core/range.js';
10 > import { ClosingBracketKind, OpeningBracketKind } from './languages/supports/languageBracketsConfiguration.js';
11 > import { PairAstNode } from './model/bracketPairsTextModelPart/bracketPairsTree/ast.js';
12 >
13 > export interface IBracketPairsTextModelPart {
14 > /**
15 > * Is fired when bracket pairs change, either due to a text or a settings change.
16 > */
17 > readonly onDidChange: Event<void>;
18 >
19 > /**
20 > * Gets all bracket pairs that intersect the given position.
21 > * The result is sorted by the start position.
22 > */
23 > getBracketPairsInRange(range: IRange): CallbackIterable<BracketPairInfo>;
24 >
25 > /**
26 > * Gets all bracket pairs that intersect the given position.
27 > * The result is sorted by the start position.
28 > */
29 > getBracketPairsInRangeWithMinIndentation(range: IRange): CallbackIterable<BracketPairWithMinIndentationInfo>;
30 >
31 > getBracketsInRange(range: IRange, onlyColorizedBrackets?: boolean): CallbackIterable<BracketInfo>;
32 >
33 > /**
34 > * Find the matching bracket of `request` up, counting brackets.
35 > * @param request The bracket we're searching for
36 > * @param position The position at which to start the search.
37 > * @return The range of the matching bracket, or null if the bracket match was not found.
38 > */
39 > findMatchingBracketUp(bracket: string, position: IPosition, maxDuration?: number): Range | null;
40 >
41 > /**
42 > * Find the first bracket in the model before `position`.
43 > * @param position The position at which to start the search.
44 > * @return The info for the first bracket before `position`, or null if there are no more brackets before `positions`.
45 > */
46 > findPrevBracket(position: IPosition): IFoundBracket | null;
47 >
48 > /**
49 > * Find the first bracket in the model after `position`.
50 > * @param position The position at which to start the search.
51 > * @return The info for the first bracket after `position`, or null if there are no more brackets after `positions`.
52 > */
53 > findNextBracket(position: IPosition): IFoundBracket | null;
54 >
55 > /**
56 > * Find the enclosing brackets that contain `position`.
57 > * @param position The position at which to start the search.
58 > */
59 > findEnclosingBrackets(position: IPosition, maxDuration?: number): [Range, Range] | null;
60 >
61 > /**
62 > * Given a `position`, if the position is on top or near a bracket,
63 > * find the matching bracket of that bracket and return the ranges of both brackets.
64 > * @param position The position at which to look for a bracket.
65 > */
66 > matchBracket(position: IPosition, maxDuration?: number): [Range, Range] | null;
67 > }
68 >
69 > export interface IFoundBracket {
70 > range: Range;
71 > bracketInfo: OpeningBracketKind | ClosingBracketKind;
72 > }
73 >
74 > export class BracketInfo {
75 > constructor(
76 public readonly range: Range,
77 /** 0-based level */
80 public readonly isInvalid: boolean,
81 ) { }
83 >
84 > export class BracketPairInfo {
85 > constructor(
86 public readonly range: Range,
87 public readonly openingBracketRange: Range,
94 ) {
95 }
97 > public get openingBracketInfo(): OpeningBracketKind {
98 return this.bracketPairNode.openingBracket.bracketInfo as OpeningBracketKind;
99 }
101 > public get closingBracketInfo(): ClosingBracketKind | undefined {
102 return this.bracketPairNode.closingBracket?.bracketInfo as ClosingBracketKind | undefined;
103 }
105 >
106 > export class BracketPairWithMinIndentationInfo extends BracketPairInfo {
107 > constructor(
108 range: Range,
109 openingBracketRange: Range,
122 super(range, openingBracketRange, closingBracketRange, nestingLevel, nestingLevelOfEqualBracketType, bracketPairNode);
123 }
src/vs/editor/common/model/tokens/treeSitter/treeSitterTree.ts 87 introduced LOC · 18 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- treeSitterTree.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 > import type * as TreeSitter from '@vscode/tree-sitter-wasm';
6 > import { TaskQueue } from '../../../../../base/common/async.js';
7 > import { Disposable, toDisposable } from '../../../../../base/common/lifecycle.js';
8 > import { IObservable, observableValue, transaction, IObservableWithChange } from '../../../../../base/common/observable.js';
9 > import { setTimeout0 } from '../../../../../base/common/platform.js';
10 > import { ILogService } from '../../../../../platform/log/common/log.js';
11 > import { ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js';
12 > import { TextLength } from '../../../core/text/textLength.js';
13 > import { IModelContentChangedEvent } from '../../../textModelEvents.js';
14 > import { IModelContentChange } from '../../mirrorTextModel.js';
15 > import { TextModel } from '../../textModel.js';
16 > import { gotoParent, getClosestPreviousNodes, nextSiblingOrParentSibling, gotoNthChild } from './cursorUtils.js';
17 > import { Range } from '../../../core/range.js';
18 >
19 > export class TreeSitterTree extends Disposable {
20 >
21 > private readonly _tree = observableValue<TreeSitter.Tree | undefined, TreeParseUpdateEvent>(this, undefined);
22 > public readonly tree: IObservableWithChange<TreeSitter.Tree | undefined, TreeParseUpdateEvent> = this._tree;
23 >
24 > private readonly _treeLastParsedVersion = observableValue(this, -1);
25 > public readonly treeLastParsedVersion: IObservable<number> = this._treeLastParsedVersion;
26 >
27 > private _lastFullyParsed: TreeSitter.Tree | undefined;
28 > private _lastFullyParsedWithEdits: TreeSitter.Tree | undefined;
29 >
30 > private _onDidChangeContentQueue: TaskQueue = new TaskQueue();
31 >
32 > constructor(
33 public readonly languageId: string,
34 private _ranges: TreeSitter.Range[] | undefined,
55 this.handleContentChange(undefined, this._ranges);
56 }
58 > public handleContentChange(e: IModelContentChangedEvent | undefined, ranges?: TreeSitter.Range[]): void {
59 const version = this.textModel.getVersionId();
60 let newRanges: TreeSitter.Range[] = [];
102 });
103 }
105 > get ranges(): TreeSitter.Range[] | undefined {
106 return this._ranges;
107 }
109 > public getInjectionTrees(startIndex: number, languageId: string): TreeSitterTree | undefined {
110 // TODO
111 return undefined;
112 }
114 > private _applyEdits(changes: IModelContentChange[]) {
115 for (const change of changes) {
116 const originalTextLength = TextLength.ofRange(Range.lift(change.range));
129 }
130 }
132 > private _findChangedNodes(newTree: TreeSitter.Tree, oldTree: TreeSitter.Tree): TreeSitter.Range[] | undefined {
133 if ((this._ranges && this._ranges.every(range => range.startPosition.row !== newTree.rootNode.startPosition.row)) || newTree.rootNode.startPosition.row !== 0) {
134 return [];
183 return nodes;
184 }
186 > private _findTreeChanges(newTree: TreeSitter.Tree, changedNodes: TreeSitter.Range[], newRanges: TreeSitter.Range[]): RangeChange[] {
187 let newRangeIndex = 0;
188 const mergedChanges: RangeChange[] = [];
259 return this._constrainRanges(mergedChanges);
260 }
262 > private _constrainRanges(changes: RangeChange[]): RangeChange[] {
263 if (!this._ranges) {
264 return changes;
300 return constrainedChanges;
301 }
303 > private async _parseAndUpdateTree(version: number): Promise<TreeSitter.Tree | undefined> {
304 const tree = await this._parse();
305 if (tree) {
317 return undefined;
318 }
320 > private _parse(): Promise<TreeSitter.Tree | undefined> {
321 let parseType: TelemetryParseType = TelemetryParseType.Full;
322 if (this._tree.get()) {
325 return this._parseAndYield(parseType);
326 }
328 > private async _parseAndYield(parseType: TelemetryParseType): Promise<TreeSitter.Tree | undefined> {
329 let time: number = 0;
330 let passes: number = 0;
349 return (newTree && (inProgressVersion === this.textModel.getVersionId())) ? newTree : undefined;
350 }
352 > private _parseCallback(index: number): string | undefined {
353 try {
354 return this.textModel.getTextBuffer().getNearestChunk(index);
358 return undefined;
359 }
361 > private _setRanges(newRanges: TreeSitter.Range[]): TreeSitter.Range[] {
362 const unKnownRanges: TreeSitter.Range[] = [];
363 // If we have existing ranges, find the parts of the new ranges that are not included in the existing ones
387 return unKnownRanges;
388 }
390 > private _sendParseTimeTelemetry(parseType: TelemetryParseType, time: number, passes: number): void {
391 this._logService.debug(`Tree parsing (${parseType}) took ${time} ms and ${passes} passes.`);
392 type ParseTimeClassification = {
403 }
404 }
406 > public createParsedTreeSync(src: string): TreeSitter.Tree | undefined {
407 const parser = new this._parserClass();
408 parser.setLanguage(this._parser.language);
411 return tree ?? undefined;
412 }
414 >
415 > const enum TelemetryParseType {
416 > Full = 'fullParse',
417 > Incremental = 'incrementalParse'
418 > }
419 >
420 > export interface TreeParseUpdateEvent {
421 > ranges: RangeChange[];
422 > versionId: number;
423 > }
424 >
425 > export interface RangeWithOffsets {
426 > range: Range;
427 > startOffset: number;
428 > endOffset: number;
429 > }
430 >
431 > export interface RangeChange {
432 > newRange: Range;
433 > newRangeStartOffset: number;
434 > newRangeEndOffset: number;
435 > }
436 >
437 function newTimeOutProgressCallback(): (state: TreeSitter.ParseState) => void {
438 let lastYieldTime: number = performance.now();
446 };
447 }
448 > export function rangesEqual(a: TreeSitter.Range, b: TreeSitter.Range) { treeSitterTree.ts
449 return (a.startPosition.row === b.startPosition.row)
450 && (a.startPosition.column === b.startPosition.column)
454 && (a.endIndex === b.endIndex);
455 }
457 > export function rangesIntersect(a: TreeSitter.Range, b: TreeSitter.Range) {
458 return (a.startIndex <= b.startIndex && a.endIndex >= b.startIndex) ||
459 (b.startIndex <= a.startIndex && b.endIndex >= a.startIndex);
src/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.ts 75 introduced LOC · 17 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- tokenizerSyntaxTokenBackend.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { onUnexpectedError } from '../../../../base/common/errors.js';
7 > import { Emitter, Event } from '../../../../base/common/event.js';
8 > import { MutableDisposable, DisposableMap } from '../../../../base/common/lifecycle.js';
9 > import { countEOL } from '../../core/misc/eolCounter.js';
10 > import { Position } from '../../core/position.js';
11 > import { LineRange } from '../../core/ranges/lineRange.js';
12 > import { StandardTokenType } from '../../encodedTokenAttributes.js';
13 > import { IBackgroundTokenizer, IState, ILanguageIdCodec, TokenizationRegistry, ITokenizationSupport, IBackgroundTokenizationStore } from '../../languages.js';
14 > import { IAttachedView } from '../../model.js';
15 > import { FontTokensUpdate, IModelContentChangedEvent } from '../../textModelEvents.js';
16 > import { BackgroundTokenizationState } from '../../tokenizationTextModelPart.js';
17 > import { ContiguousMultilineTokens } from '../../tokens/contiguousMultilineTokens.js';
18 > import { ContiguousMultilineTokensBuilder } from '../../tokens/contiguousMultilineTokensBuilder.js';
19 > import { ContiguousTokensStore } from '../../tokens/contiguousTokensStore.js';
20 > import { LineTokens } from '../../tokens/lineTokens.js';
21 > import { TextModel } from '../textModel.js';
22 > import { TokenizerWithStateStoreAndTextModel, DefaultBackgroundTokenizer, TrackingTokenizationStateStore } from '../textModelTokens.js';
23 > import { AbstractSyntaxTokenBackend, AttachedViewHandler, AttachedViews } from './abstractSyntaxTokenBackend.js';
24 >
25 > /** For TextMate */
26 > export class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend {
27 > private _tokenizer: TokenizerWithStateStoreAndTextModel | null = null;
28 > protected _backgroundTokenizationState: BackgroundTokenizationState = BackgroundTokenizationState.InProgress;
29 > protected readonly _onDidChangeBackgroundTokenizationState: Emitter<void> = this._register(new Emitter<void>());
30 > public readonly onDidChangeBackgroundTokenizationState: Event<void> = this._onDidChangeBackgroundTokenizationState.event;
31 >
32 > private _defaultBackgroundTokenizer: DefaultBackgroundTokenizer | null = null;
33 > private readonly _backgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());
34 >
35 > private readonly _tokens = new ContiguousTokensStore(this._languageIdCodec);
36 > private _debugBackgroundTokens: ContiguousTokensStore | undefined;
37 > private _debugBackgroundStates: TrackingTokenizationStateStore<IState> | undefined;
38 >
39 > private readonly _debugBackgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());
40 >
41 > private readonly _attachedViewStates = this._register(new DisposableMap<IAttachedView, AttachedViewHandler>());
42 >
43 > constructor(
44 languageIdCodec: ILanguageIdCodec,
45 textModel: TextModel,
72 }));
73 }
75 > public todo_resetTokenization(fireTokenChangeEvent: boolean = true): void {
76 this._tokens.flush();
77 this._debugBackgroundTokens?.flush();
182 this.refreshAllVisibleLineTokens();
183 }
185 > public handleDidChangeAttached() {
186 this._defaultBackgroundTokenizer?.handleChanges();
187 }
189 > public handleDidChangeContent(e: IModelContentChangedEvent): void {
190 if (e.isFlush) {
191 // Don't fire the event, as the view might not have got the text change event yet
206 }
207 }
209 > private setTokens(tokens: ContiguousMultilineTokens[]): { changes: { fromLineNumber: number; toLineNumber: number }[] } {
210 const { changes } = this._tokens.setMultilineTokens(tokens, this._textModel);
211
216 return { changes: changes };
217 }
219 > private setFontInfo(changes: FontTokensUpdate): void {
220 this._onDidChangeFontTokens.fire({ changes });
221 }
223 > private refreshAllVisibleLineTokens(): void {
224 const ranges = LineRange.joinMany([...this._attachedViewStates].map(([_, s]) => s.lineRanges));
225 this.refreshRanges(ranges);
226 }
228 > private refreshRanges(ranges: readonly LineRange[]): void {
229 for (const range of ranges) {
230 this.refreshRange(range.startLineNumber, range.endLineNumberExclusive - 1);
231 }
232 }
234 > private refreshRange(startLineNumber: number, endLineNumber: number): void {
235 if (!this._tokenizer) {
236 return;
255 this._defaultBackgroundTokenizer?.checkFinished();
256 }
258 > public forceTokenization(lineNumber: number): void {
259 const builder = new ContiguousMultilineTokensBuilder();
260 this._tokenizer?.updateTokensUntilLine(builder, lineNumber);
262 this._defaultBackgroundTokenizer?.checkFinished();
263 }
265 > public hasAccurateTokensForLine(lineNumber: number): boolean {
266 if (!this._tokenizer) {
267 return true;
269 return this._tokenizer.hasAccurateTokensForLine(lineNumber);
270 }
272 > public isCheapToTokenize(lineNumber: number): boolean {
273 if (!this._tokenizer) {
274 return true;
276 return this._tokenizer.isCheapToTokenize(lineNumber);
277 }
279 > public getLineTokens(lineNumber: number): LineTokens {
280 const lineText = this._textModel.getLineContent(lineNumber);
281 const result = this._tokens.getTokens(
298 return result;
299 }
301 > public getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {
302 if (!this._tokenizer) {
303 return StandardTokenType.Other;
308 return this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);
309 }
311 >
312 > public tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {
313 if (!this._tokenizer) {
314 return null;
src/vs/editor/common/services/treeSitter/treeSitterLibraryService.ts 67 introduced LOC · 1 range

Open complete file

1 > /*--------------------------------------------------------------------------------------------- treeSitterLibraryService.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import type { Language, Parser, Query } from '@vscode/tree-sitter-wasm';
7 > import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
8 > import { IReader } from '../../../../base/common/observable.js';
9 >
10 > export const ITreeSitterLibraryService = createDecorator<ITreeSitterLibraryService>('treeSitterLibraryService');
11 >
12 > export interface ITreeSitterLibraryService {
13 > readonly _serviceBrand: undefined;
14 >
15 > /**
16 > * Gets the tree sitter Parser constructor.
17 > */
18 > getParserClass(): Promise<typeof Parser>;
19 >
20 > /**
21 > * Checks whether a language is supported and available based setting enablement.
22 > * @param languageId The language identifier to check.
23 > * @param reader Optional observable reader.
24 > */
25 > supportsLanguage(languageId: string, reader: IReader | undefined): boolean;
26 >
27 > /**
28 > * Gets the tree sitter Language object synchronously.
29 > * @param languageId The language identifier to retrieve.
30 > * @param ignoreSupportsCheck Whether to ignore the supportsLanguage check.
31 > * @param reader Optional observable reader.
32 > */
33 > getLanguage(languageId: string, ignoreSupportsCheck: boolean, reader: IReader | undefined): Language | undefined;
34 >
35 > /**
36 > * Gets the language as a promise, as opposed to via observables. This ignores the automatic
37 > * supportsLanguage check.
38 > *
39 > * Warning: This approach is generally not recommended as it's not reactive, but it's the only
40 > * way to catch and handle import errors when the grammar fails to load.
41 > * @param languageId The language identifier to retrieve.
42 > */
43 > getLanguagePromise(languageId: string): Promise<Language | undefined>;
44 >
45 > /**
46 > * Gets the injection queries for a language. A return value of `null`
47 > * indicates that there are no highlights queries for this language.
48 > * @param languageId The language identifier to retrieve queries for.
49 > * @param reader Optional observable reader.
50 > */
51 > getInjectionQueries(languageId: string, reader: IReader | undefined): Query | null | undefined;
52 >
53 > /**
54 > * Gets the highlighting queries for a language. A return value of `null`
55 > * indicates that there are no highlights queries for this language.
56 > * @param languageId The language identifier to retrieve queries for.
57 > * @param reader Optional observable reader.
58 > */
59 > getHighlightingQueries(languageId: string, reader: IReader | undefined): Query | null | undefined;
60 >
61 > /**
62 > * Creates a one-off custom query for a language.
63 > * @param language The Language to create the query for.
64 > * @param querySource The query source string to compile.
65 > */
66 > createQuery(language: Language, querySource: string): Promise<Query>;
67 > }
src/vs/editor/common/model/pieceTreeTextBuffer/rbTreeBase.ts 64 introduced LOC · 14 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- rbTreeBase.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { Piece, PieceTreeBase } from './pieceTreeBase.js';
7 >
8 > export class TreeNode {
9 > parent: TreeNode;
10 > left: TreeNode;
11 > right: TreeNode;
12 > color: NodeColor;
13 >
14 > // Piece
15 > piece: Piece;
16 > size_left: number; // size of the left subtree (not inorder)
17 > lf_left: number; // line feeds cnt in the left subtree (not in order)
18 >
19 > constructor(piece: Piece, color: NodeColor) {
20 > this.piece = piece;
21 > this.color = color;
22 > this.size_left = 0;
23 > this.lf_left = 0;
24 > this.parent = this;
25 > this.left = this;
26 > this.right = this;
27 > }
28 >
29 > public next(): TreeNode {
30 if (this.right !== SENTINEL) {
31 return leftest(this.right);
48 }
49 }
51 > public prev(): TreeNode {
52 if (this.left !== SENTINEL) {
53 return righttest(this.left);
70 }
71 }
73 > public detach(): void {
74 this.parent = null!;
75 this.left = null!;
76 this.right = null!;
77 }
78 > } rbTreeBase.ts
79 >
80 > export const enum NodeColor {
81 > Black = 0,
82 > Red = 1,
83 > }
84 >
85 > export const SENTINEL: TreeNode = new TreeNode(null!, NodeColor.Black);
86 > SENTINEL.parent = SENTINEL;
87 > SENTINEL.left = SENTINEL;
88 > SENTINEL.right = SENTINEL;
89 > SENTINEL.color = NodeColor.Black;
90 >
91 > export function leftest(node: TreeNode): TreeNode {
92 while (node.left !== SENTINEL) {
93 node = node.left;
95 return node;
96 }
98 > export function righttest(node: TreeNode): TreeNode {
99 while (node.right !== SENTINEL) {
100 node = node.right;
102 return node;
103 }
105 function calculateSize(node: TreeNode): number {
106 if (node === SENTINEL) {
110 return node.size_left + node.piece.length + calculateSize(node.right);
111 }
113 function calculateLF(node: TreeNode): number {
114 if (node === SENTINEL) {
118 return node.lf_left + node.piece.lineFeedCnt + calculateLF(node.right);
119 }
121 function resetSentinel(): void {
122 SENTINEL.parent = SENTINEL;
123 }
125 > export function leftRotate(tree: PieceTreeBase, x: TreeNode) {
126 const y = x.right;
127
145 x.parent = y;
146 }
148 > export function rightRotate(tree: PieceTreeBase, y: TreeNode) {
149 const x = y.left;
150 y.left = x.right;
169 y.parent = x;
170 }
172 > export function rbDelete(tree: PieceTreeBase, z: TreeNode) {
173 let x: TreeNode;
174 let y: TreeNode;
330 resetSentinel();
331 }
333 > export function fixInsert(tree: PieceTreeBase, x: TreeNode) {
334 recomputeTreeMetadata(tree, x);
335
375 tree.root.color = NodeColor.Black;
376 }
378 > export function updateTreeMetadata(tree: PieceTreeBase, x: TreeNode, delta: number, lineFeedCntDelta: number): void {
379 // node length change or line feed count change
380 while (x !== tree.root && x !== SENTINEL) {
387 }
388 }
390 > export function recomputeTreeMetadata(tree: PieceTreeBase, x: TreeNode) {
391 let delta = 0;
392 let lf_delta = 0;
src/vs/editor/common/core/edits/lengthEdit.ts 59 introduced LOC · 16 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- lengthEdit.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { OffsetRange } from '../ranges/offsetRange.js';
7 > import { AnyEdit, BaseEdit, BaseReplacement } from './edit.js';
8 >
9 > /**
10 > * Like a normal edit, but only captures the length information.
11 > */
12 > export class LengthEdit extends BaseEdit<LengthReplacement, LengthEdit> {
13 > public static readonly empty = new LengthEdit([]);
14 >
15 > public static fromEdit(edit: AnyEdit): LengthEdit {
16 return new LengthEdit(edit.replacements.map(r => new LengthReplacement(r.replaceRange, r.getNewLength())));
17 }
19 > public static create(replacements: readonly LengthReplacement[]): LengthEdit {
20 return new LengthEdit(replacements);
21 }
23 > public static single(replacement: LengthReplacement): LengthEdit {
24 return new LengthEdit([replacement]);
25 }
27 > public static replace(range: OffsetRange, newLength: number): LengthEdit {
28 return new LengthEdit([new LengthReplacement(range, newLength)]);
29 }
31 > public static insert(offset: number, newLength: number): LengthEdit {
32 return new LengthEdit([new LengthReplacement(OffsetRange.emptyAt(offset), newLength)]);
33 }
35 > public static delete(range: OffsetRange): LengthEdit {
36 return new LengthEdit([new LengthReplacement(range, 0)]);
37 }
39 > public static compose(edits: readonly LengthEdit[]): LengthEdit {
40 let e = LengthEdit.empty;
41 for (const edit of edits) {
44 return e;
45 }
47 > /**
48 > * Creates an edit that reverts this edit.
49 > */
50 > public inverse(): LengthEdit {
51 const edits: LengthReplacement[] = [];
52 let offset = 0;
60 return new LengthEdit(edits);
61 }
63 > protected override _createNew(replacements: readonly LengthReplacement[]): LengthEdit {
64 return new LengthEdit(replacements);
65 }
67 > public applyArray<T>(arr: readonly T[], fillItem: T): T[] {
68 const newArr = new Array(this.getNewDataLength(arr.length));
69
93 return newArr;
94 }
95 > } lengthEdit.ts
96 >
97 > export class LengthReplacement extends BaseReplacement<LengthReplacement> {
98 > public static create(
99 > startOffset: number,
100 > endOffsetExclusive: number,
101 > newLength: number,
102 > ): LengthReplacement {
103 > return new LengthReplacement(new OffsetRange(startOffset, endOffsetExclusive), newLength);
104 > }
105 >
106 > constructor(
107 range: OffsetRange,
108 public readonly newLength: number,
110 super(range);
111 }
113 > override equals(other: LengthReplacement): boolean {
114 return this.replaceRange.equals(other.replaceRange) && this.newLength === other.newLength;
115 }
117 > getNewLength(): number { return this.newLength; }
118 >
119 > tryJoinTouching(other: LengthReplacement): LengthReplacement | undefined {
120 return new LengthReplacement(this.replaceRange.joinRightTouching(other.replaceRange), this.newLength + other.newLength);
121 }
123 > slice(range: OffsetRange, rangeInReplacement: OffsetRange): LengthReplacement {
124 return new LengthReplacement(range, rangeInReplacement.length);
125 }
127 > override toString() {
128 return `[${this.replaceRange.start}, +${this.replaceRange.length}) -> +${this.newLength}}`;
129 }
130 > } lengthEdit.ts
src/vs/editor/common/tokens/contiguousTokensStore.ts 56 introduced LOC · 15 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- contiguousTokensStore.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import * as arrays from '../../../base/common/arrays.js';
7 > import { Position } from '../core/position.js';
8 > import { IRange } from '../core/range.js';
9 > import { ContiguousTokensEditing, EMPTY_LINE_TOKENS, toUint32Array } from './contiguousTokensEditing.js';
10 > import { LineTokens } from './lineTokens.js';
11 > import { ILanguageIdCodec } from '../languages.js';
12 > import { LanguageId, FontStyle, ColorId, StandardTokenType, MetadataConsts, TokenMetadata } from '../encodedTokenAttributes.js';
13 > import { ITextModel } from '../model.js';
14 > import { ContiguousMultilineTokens } from './contiguousMultilineTokens.js';
15 >
16 > /**
17 > * Represents contiguous tokens in a text model.
18 > */
19 > export class ContiguousTokensStore {
20 > private _lineTokens: (Uint32Array | ArrayBuffer | null)[];
21 > private _len: number;
22 > private readonly _languageIdCodec: ILanguageIdCodec;
23 >
24 > constructor(languageIdCodec: ILanguageIdCodec) {
25 this._lineTokens = [];
26 this._len = 0;
27 this._languageIdCodec = languageIdCodec;
28 }
30 > public flush(): void {
31 this._lineTokens = [];
32 this._len = 0;
33 }
35 > get hasTokens(): boolean {
36 return this._lineTokens.length > 0;
37 }
39 > public getTokens(topLevelLanguageId: string, lineIndex: number, lineText: string): LineTokens {
40 let rawLineTokens: Uint32Array | ArrayBuffer | null = null;
41 if (lineIndex < this._len) {
52 return new LineTokens(lineTokens, lineText, this._languageIdCodec);
53 }
55 > private static _massageTokens(topLevelLanguageId: LanguageId, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null): Uint32Array | ArrayBuffer {
56
57 const tokens = _tokens ? toUint32Array(_tokens) : null;
84 return tokens;
85 }
87 > private _ensureLine(lineIndex: number): void {
88 while (lineIndex >= this._len) {
89 this._lineTokens[this._len] = null;
91 }
92 }
94 > private _deleteLines(start: number, deleteCount: number): void {
95 if (deleteCount === 0) {
96 return;
102 this._len -= deleteCount;
103 }
105 > private _insertLines(insertIndex: number, insertCount: number): void {
106 if (insertCount === 0) {
107 return;
114 this._len += insertCount;
115 }
117 > public setTokens(topLevelLanguageId: string, lineIndex: number, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null, checkEquality: boolean): boolean {
118 const tokens = ContiguousTokensStore._massageTokens(this._languageIdCodec.encodeLanguageId(topLevelLanguageId), lineTextLength, _tokens);
119 this._ensureLine(lineIndex);
126 return false;
127 }
129 > private static _equals(_a: Uint32Array | ArrayBuffer | null, _b: Uint32Array | ArrayBuffer | null) {
130 if (!_a || !_b) {
131 return !_a && !_b;
145 return true;
146 }
148 > //#region Editing
149 >
150 > public acceptEdit(range: IRange, eolCount: number, firstLineLength: number): void {
151 this._acceptDeleteRange(range);
152 this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength);
153 }
155 > private _acceptDeleteRange(range: IRange): void {
156
157 const firstLineIndex = range.startLineNumber - 1;
184 this._deleteLines(range.startLineNumber, range.endLineNumber - range.startLineNumber);
185 }
187 > private _acceptInsertText(position: Position, eolCount: number, firstLineLength: number): void {
188
189 if (eolCount === 0 && firstLineLength === 0) {
208 this._insertLines(position.lineNumber, eolCount);
209 }
211 > //#endregion
212 >
213 > public setMultilineTokens(tokens: ContiguousMultilineTokens[], textModel: ITextModel): { changes: { fromLineNumber: number; toLineNumber: number }[] } {
214 if (tokens.length === 0) {
215 return { changes: [] };
243 return { changes: ranges };
244 }
246 >
247 function getDefaultMetadata(topLevelLanguageId: LanguageId): number {
248 return (
src/vs/editor/common/model/tokens/treeSitter/treeSitterSyntaxTokenBackend.ts 55 introduced LOC · 14 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- treeSitterSyntaxTokenBackend.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { Emitter, Event } from '../../../../../base/common/event.js';
7 > import { toDisposable } from '../../../../../base/common/lifecycle.js';
8 > import { StandardTokenType } from '../../../encodedTokenAttributes.js';
9 > import { ILanguageIdCodec } from '../../../languages.js';
10 > import { IModelContentChangedEvent } from '../../../textModelEvents.js';
11 > import { BackgroundTokenizationState } from '../../../tokenizationTextModelPart.js';
12 > import { LineTokens } from '../../../tokens/lineTokens.js';
13 > import { TextModel } from '../../textModel.js';
14 > import { AbstractSyntaxTokenBackend } from '../abstractSyntaxTokenBackend.js';
15 > import { autorun, derived, IObservable, ObservablePromise } from '../../../../../base/common/observable.js';
16 > import { TreeSitterTree } from './treeSitterTree.js';
17 > import { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js';
18 > import { TreeSitterTokenizationImpl } from './treeSitterTokenizationImpl.js';
19 > import { ITreeSitterLibraryService } from '../../../services/treeSitter/treeSitterLibraryService.js';
20 > import { LineRange } from '../../../core/ranges/lineRange.js';
21 >
22 > export class TreeSitterSyntaxTokenBackend extends AbstractSyntaxTokenBackend {
23 > protected _backgroundTokenizationState: BackgroundTokenizationState = BackgroundTokenizationState.InProgress;
24 > protected readonly _onDidChangeBackgroundTokenizationState: Emitter<void> = this._register(new Emitter<void>());
25 > public readonly onDidChangeBackgroundTokenizationState: Event<void> = this._onDidChangeBackgroundTokenizationState.event;
26 >
27 > private readonly _tree: IObservable<TreeSitterTree | undefined>;
28 > private readonly _tokenizationImpl: IObservable<TreeSitterTokenizationImpl | undefined>;
29 >
30 > constructor(
31 private readonly _languageIdObs: IObservable<string>,
32 languageIdCodec: ILanguageIdCodec,
103 }));
104 }
106 > get tree(): IObservable<TreeSitterTree | undefined> {
107 return this._tree;
108 }
110 > get tokenizationImpl(): IObservable<TreeSitterTokenizationImpl | undefined> {
111 return this._tokenizationImpl;
112 }
114 > public getLineTokens(lineNumber: number): LineTokens {
115 const model = this._tokenizationImpl.get();
116 if (!model) {
120 return model.getLineTokens(lineNumber);
121 }
123 > public todo_resetTokenization(fireTokenChangeEvent: boolean = true): void {
124 if (fireTokenChangeEvent) {
125 this._onDidChangeTokens.fire({
134 }
135 }
137 > public override handleDidChangeAttached(): void {
138 // TODO @alexr00 implement for background tokenization
139 }
141 > public override handleDidChangeContent(e: IModelContentChangedEvent): void {
142 if (e.isFlush) {
143 // Don't fire the event, as the view might not have got the text change event yet
151 treeModel?.handleContentChange(e);
152 }
154 > public override forceTokenization(lineNumber: number): void {
155 const model = this._tokenizationImpl.get();
156 if (!model) {
161 }
162 }
164 > public override hasAccurateTokensForLine(lineNumber: number): boolean {
165 const model = this._tokenizationImpl.get();
166 if (!model) {
169 return model.hasAccurateTokensForLine(lineNumber);
170 }
172 > public override isCheapToTokenize(lineNumber: number): boolean {
173 // TODO @alexr00 determine what makes it cheap to tokenize?
174 return true;
175 }
177 > public override getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {
178 // TODO @alexr00 implement once we have custom parsing and don't just feed in the whole text model value
179 return StandardTokenType.Other;
180 }
182 > public override tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {
183 const model = this._tokenizationImpl.get();
184 if (!model) {
187 return model.tokenizeLinesAt(lineNumber, lines);
188 }
190 > public override get hasTokens(): boolean {
191 const model = this._tokenizationImpl.get();
192 if (!model) {
src/vs/editor/common/textModelGuides.ts 51 introduced LOC · 3 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- textModelGuides.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { IPosition } from './core/position.js';
7 >
8 > export interface IGuidesTextModelPart {
9 > /**
10 > * @internal
11 > */
12 > getActiveIndentGuide(lineNumber: number, minLineNumber: number, maxLineNumber: number): IActiveIndentGuideInfo;
13 >
14 > /**
15 > * @internal
16 > */
17 > getLinesIndentGuides(startLineNumber: number, endLineNumber: number): number[];
18 >
19 > /**
20 > * Requests the indent guides for the given range of lines.
21 > * `result[i]` will contain the indent guides of the `startLineNumber + i`th line.
22 > * @internal
23 > */
24 > getLinesBracketGuides(startLineNumber: number, endLineNumber: number, activePosition: IPosition | null, options: BracketGuideOptions): IndentGuide[][];
25 > }
26 >
27 > export interface IActiveIndentGuideInfo {
28 > startLineNumber: number;
29 > endLineNumber: number;
30 > indent: number;
31 > }
32 >
33 > export enum HorizontalGuidesState {
34 > Disabled,
35 > EnabledForActive,
36 > Enabled
37 > }
38 >
39 > export interface BracketGuideOptions {
40 > includeInactive: boolean;
41 > horizontalGuides: HorizontalGuidesState;
42 > highlightActive: boolean;
43 > }
44 >
45 > export class IndentGuide {
46 > constructor(
47 public readonly visibleColumn: number | -1,
48 public readonly column: number | -1,
63 }
64 }
66 >
67 > export class IndentGuideHorizontalLine {
68 > constructor(
69 public readonly top: boolean,
70 public readonly endColumn: number,
71 ) { }
src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder.ts 50 introduced LOC · 11 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- pieceTreeTextBufferBuilder.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { CharCode } from '../../../../base/common/charCode.js';
7 > import { IDisposable } from '../../../../base/common/lifecycle.js';
8 > import * as strings from '../../../../base/common/strings.js';
9 > import { DefaultEndOfLine, ITextBuffer, ITextBufferBuilder, ITextBufferFactory } from '../../model.js';
10 > import { StringBuffer, createLineStarts, createLineStartsFast } from './pieceTreeBase.js';
11 > import { PieceTreeTextBuffer } from './pieceTreeTextBuffer.js';
12 >
13 > class PieceTreeTextBufferFactory implements ITextBufferFactory {
14 >
15 > constructor(
16 private readonly _chunks: StringBuffer[],
17 private readonly _bom: string,
24 private readonly _normalizeEOL: boolean
25 ) { }
27 > private _getEOL(defaultEOL: DefaultEndOfLine): '\r\n' | '\n' {
28 const totalEOLCount = this._cr + this._lf + this._crlf;
29 const totalCRCount = this._cr + this._crlf;
39 return '\n';
40 }
42 > public create(defaultEOL: DefaultEndOfLine): { textBuffer: ITextBuffer; disposable: IDisposable } {
43 const eol = this._getEOL(defaultEOL);
44 const chunks = this._chunks;
59 return { textBuffer: textBuffer, disposable: textBuffer };
60 }
62 > public getFirstLineText(lengthLimit: number): string {
63 return this._chunks[0].buffer.substr(0, lengthLimit).split(/\r\n|\r|\n/)[0];
64 }
66 >
67 > export class PieceTreeTextBufferBuilder implements ITextBufferBuilder {
68 > private readonly chunks: StringBuffer[];
69 > private BOM: string;
70 >
71 > private _hasPreviousChar: boolean;
72 > private _previousChar: number;
73 > private readonly _tmpLineStarts: number[];
74 >
75 > private cr: number;
76 > private lf: number;
77 > private crlf: number;
78 > private containsRTL: boolean;
79 > private containsUnusualLineTerminators: boolean;
80 > private isBasicASCII: boolean;
81 >
82 > constructor() {
83 this.chunks = [];
84 this.BOM = '';
95 this.isBasicASCII = true;
96 }
98 > public acceptChunk(chunk: string): void {
99 if (chunk.length === 0) {
100 return;
120 }
121 }
123 > private _acceptChunk1(chunk: string, allowEmptyStrings: boolean): void {
124 if (!allowEmptyStrings && chunk.length === 0) {
125 // Nothing to do
133 }
134 }
136 > private _acceptChunk2(chunk: string): void {
137 const lineStarts = createLineStarts(this._tmpLineStarts, chunk);
138
153 }
154 }
156 > public finish(normalizeEOL: boolean = true): PieceTreeTextBufferFactory {
157 this._finish();
158 return new PieceTreeTextBufferFactory(
168 );
169 }
171 > private _finish(): void {
172 if (this.chunks.length === 0) {
173 this._acceptChunk1('', true);
src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/parser.ts 49 introduced LOC · 8 ranges

Open complete file

1 > /*--------------------------------------------------------------------------------------------- parser.ts
2 > * Copyright (c) Microsoft Corporation. All rights reserved.
3 > * Licensed under the MIT License. See License.txt in the project root for license information.
4 > *--------------------------------------------------------------------------------------------*/
5 >
6 > import { AstNode, AstNodeKind, BracketAstNode, InvalidBracketAstNode, ListAstNode, PairAstNode, TextAstNode } from './ast.js';
7 > import { BeforeEditPositionMapper, TextEditInfo } from './beforeEditPositionMapper.js';
8 > import { SmallImmutableSet } from './smallImmutableSet.js';
9 > import { lengthIsZero, lengthLessThan } from './length.js';
10 > import { concat23Trees, concat23TreesOfSameHeight } from './concat23Trees.js';
11 > import { NodeReader } from './nodeReader.js';
12 > import { OpeningBracketId, Tokenizer, TokenKind } from './tokenizer.js';
13 >
14 > /**
15 > * Non incrementally built ASTs are immutable.
16 > */
17 > export function parseDocument(tokenizer: Tokenizer, edits: TextEditInfo[], oldNode: AstNode | undefined, createImmutableLists: boolean): AstNode {
18 const parser = new Parser(tokenizer, edits, oldNode, createImmutableLists);
19 return parser.parseDocument();
20 }
21 > parser.ts
22 > /**
23 > * Non incrementally built ASTs are immutable.
24 > */
25 > class Parser {
26 > private readonly oldNodeReader?: NodeReader;
27 > private readonly positionMapper: BeforeEditPositionMapper;
28 > private _itemsConstructed: number = 0;
29 > private _itemsFromCache: number = 0;
30 >
31 > /**
32 > * Reports how many nodes were constructed in the last parse operation.
33 > */
34 > get nodesConstructed() {
35 > return this._itemsConstructed;
36 > }
37 >
38 > /**
39 > * Reports how many nodes were reused in the last parse operation.
40 > */
41 > get nodesReused() {
42 return this._itemsFromCache;
43 }
44 > parser.ts
45 > constructor(
46 private readonly tokenizer: Tokenizer,
47 edits: TextEditInfo[],
56 this.positionMapper = new BeforeEditPositionMapper(edits);
57 }
58 > parser.ts
59 > parseDocument(): AstNode {
60 this._itemsConstructed = 0;
61 this._itemsFromCache = 0;
68 return result;
69 }
70 > parser.ts
71 > private parseList(
72 openedBracketIds: SmallImmutableSet<OpeningBracketId>,
73 level: number,
102 return result;
103 }
104 > parser.ts
105 > private tryReadChildFromCache(openedBracketIds: SmallImmutableSet<number>): AstNode | undefined {
106 if (this.oldNodeReader) {
107 const maxCacheableLength = this.positionMapper.getDistanceToNextChange(this.tokenizer.offset);
128 return undefined;
129 }
130 > parser.ts
131 > private parseChild(
132 openedBracketIds: SmallImmutableSet<number>,
133 level: number,