src/vs/workbench/services/search/common/textSearchManager.ts
394 LOC · 353 covered · 41 uncovered · 90 ranges · 72 concepts · 15 introducers · 34 tests
File neighbourhood
The centred file is linked to every concept that introduces one of its ranges, every test that runs code from the file, and the gray connector concepts standing between those tests and the file's own introducer concepts. Undirected links join concepts to every file where they introduce source and concepts to the tests they introduce; arrows show specialization between the displayed concepts and bridge only concepts omitted from this view. Concept colors match the source ranges below; connector concepts have no source color and are shown in gray.
Focused file, its introducer and connector concepts, their introduced files, and tests that run code from the file
In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the related-file, concept, and source links on this page.
Graph controls are ready.
Interactive rendering requires JavaScript and WebGL. Use the related-file, concept, and source links on this page while the interactive map is unavailable.
1
>
/*---------------------------------------------------------------------------------------------
textSearchManager.ts ×23
2
>
* Copyright (c) Microsoft Corporation. All rights reserved.
3
>
* Licensed under the MIT License. See License.txt in the project root for license information.
4
>
*--------------------------------------------------------------------------------------------*/
5
>
6
>
import { isThenable } from '../../../../base/common/async.js';
7
>
import { CancellationToken, CancellationTokenSource } from '../../../../base/common/cancellation.js';
8
>
import { toErrorMessage } from '../../../../base/common/errorMessage.js';
9
>
import { Schemas } from '../../../../base/common/network.js';
10
>
import * as path from '../../../../base/common/path.js';
11
>
import * as resources from '../../../../base/common/resources.js';
12
>
import { URI } from '../../../../base/common/uri.js';
13
>
import { FolderQuerySearchTree } from './folderQuerySearchTree.js';
14
>
import { DEFAULT_MAX_SEARCH_RESULTS, hasSiblingPromiseFn, IAITextQuery, IFileMatch, IFolderQuery, excludeToGlobPattern, IPatternInfo, ISearchCompleteStats, ITextQuery, ITextSearchContext, ITextSearchMatch, ITextSearchResult, ITextSearchStats, QueryGlobTester, QueryType, resolvePatternsForProvider, ISearchRange, DEFAULT_TEXT_SEARCH_PREVIEW_OPTIONS } from './search.js';
15
>
import { TextSearchComplete2, TextSearchMatch2, TextSearchProviderFolderOptions, TextSearchProvider2, TextSearchProviderOptions, TextSearchQuery2, TextSearchResult2, AITextSearchProvider, AISearchResult, AISearchKeyword } from './searchExtTypes.js';
16
>
17
>
export interface IFileUtils {
18
>
readdir: (resource: URI) => Promise<string[]>;
19
>
toCanonicalName: (encoding: string) => string;
20
>
}
21
>
interface IAITextQueryProviderPair {
22
>
query: IAITextQuery; provider: AITextSearchProvider;
23
>
}
24
>
25
>
interface ITextQueryProviderPair {
26
>
query: ITextQuery; provider: TextSearchProvider2;
27
>
}
28
>
interface FolderQueryInfo {
29
>
queryTester: QueryGlobTester;
30
>
folder: URI;
31
>
folderIdx: number;
32
>
}
33
>
34
>
export class TextSearchManager {
35
>
36
>
private collector: TextSearchResultsCollector | null = null;
37
>
38
>
private isLimitHit = false;
39
>
private resultCount = 0;
40
>
41
>
constructor(private queryProviderPair: IAITextQueryProviderPair | ITextQueryProviderPair,
43
>
private processType: ITextSearchStats['type']) { }
45
>
private get query() {
47
>
}
49
>
search(onProgress: (matches: IFileMatch[]) => void, token: CancellationToken, onKeywordResult?: (keyword: AISearchKeyword) => void): Promise<ISearchCompleteStats> {
51
>
const tokenSource = new CancellationTokenSource(token);
52
>
53
>
return new Promise<ISearchCompleteStats>((resolve, reject) => {
54
>
this.collector = new TextSearchResultsCollector(onProgress);
55
>
56
>
let isCanceled = false;
57
>
const onResult = (result: TextSearchResult2, folderIdx: number) => {
59
// Already processed by the callback.
60
return;
61
}
64
>
}
66
>
if (!this.isLimitHit) {
67
>
const resultSize = this.resultSize(result);
68
>
if (result instanceof TextSearchMatch2 && typeof this.query.maxResults === 'number' && this.resultCount + resultSize > this.query.maxResults) {
70
>
isCanceled = true;
71
>
tokenSource.cancel();
72
>
73
>
result = this.trimResultToSize(result, this.query.maxResults - this.resultCount);
74
>
}
76
>
const newResultSize = this.resultSize(result);
77
>
this.resultCount += newResultSize;
78
>
const a = result instanceof TextSearchMatch2;
79
>
80
>
if (newResultSize > 0 || !a) {
81
>
this.collector!.add(result, folderIdx);
82
>
}
83
>
}
84
>
};
86
>
// For each root folder
87
>
this.doSearch(folderQueries, onResult, tokenSource.token, onKeywordResult).then(result => {
89
>
this.collector!.flush();
90
>
91
>
resolve({
92
>
limitHit: this.isLimitHit || result?.limitHit,
93
>
messages: this.getMessagesFromResults(result),
94
>
stats: {
95
>
type: this.processType
96
>
}
97
>
});
100
>
const errMsg = toErrorMessage(err);
101
>
reject(new Error(errMsg));
103
>
});
104
>
}
106
>
private getMessagesFromResults(result: TextSearchComplete2 | null | undefined) {
109
return [result.message];
112
>
private resultSize(result: TextSearchResult2): number {
114
>
return Array.isArray(result.ranges) ?
115
>
result.ranges.length :
116
1;
118
else {
119
// #104400 context lines shoudn't count towards result count
120
return 0;
121
}
124
>
private trimResultToSize(result: TextSearchMatch2, size: number): TextSearchMatch2 {
125
>
return new TextSearchMatch2(result.uri, result.ranges.slice(0, size), result.previewText);
textSearchManager.ts ×2
126
>
}
128
>
private async doSearch(folderQueries: IFolderQuery<URI>[], onResult: (result: TextSearchResult2, folderIdx: number) => void, token: CancellationToken, onKeywordResult?: (keyword: AISearchKeyword) => void): Promise<TextSearchComplete2 | null | undefined> {
129
>
const folderMappings: FolderQuerySearchTree<FolderQueryInfo> = new FolderQuerySearchTree<FolderQueryInfo>(
textSearchManager.ts ×16
130
>
folderQueries,
131
>
(fq, i) => {
132
>
const queryTester = new QueryGlobTester(this.query, fq);
133
>
return { queryTester, folder: fq.folder, folderIdx: i };
134
>
},
135
>
() => true
136
>
);
137
>
138
>
const testingPs: Promise<void>[] = [];
139
>
const progress = {
140
>
report: (result: TextSearchResult2 | AISearchResult) => {
142
onKeywordResult?.(result);
144
>
if (result.uri === undefined) {
145
throw Error('Text search result URI is undefined. Please check provider implementation.');
146
}
147
>
const folderQuery = folderMappings.findQueryFragmentAwareSubstr(result.uri);
textSearchManager.ts ×25
148
>
if (folderQuery?.folder?.scheme) {
149
>
const hasSibling = folderQuery.folder.scheme === Schemas.file ?
155
>
const relativePath = resources.relativePath(folderQuery.folder, result.uri);
156
>
if (relativePath) {
157
>
// This method is only async when the exclude contains sibling clauses
158
>
const included = folderQuery.queryTester.includedInQuery(relativePath, path.basename(relativePath), hasSibling);
159
>
if (isThenable(included)) {
160
>
testingPs.push(
161
>
included.then(isIncluded => {
162
>
if (isIncluded) {
163
>
onResult(result, folderQuery.folderIdx);
164
>
}
165
>
}));
166
>
} else if (included) {
167
onResult(result, folderQuery.folderIdx);
168
}
170
>
}
171
>
}
172
>
}
174
>
175
>
const folderOptions = folderQueries.map(fq => this.getSearchOptionsForFolder(fq));
176
>
const searchOptions: TextSearchProviderOptions = {
177
>
folderOptions,
178
>
maxFileSize: this.query.maxFileSize,
179
>
maxResults: this.query.maxResults ?? DEFAULT_MAX_SEARCH_RESULTS,
180
>
previewOptions: this.query.previewOptions ?? DEFAULT_TEXT_SEARCH_PREVIEW_OPTIONS,
181
>
surroundingContext: this.query.surroundingContext ?? 0,
182
>
};
183
>
184
>
let result;
185
>
if (this.queryProviderPair.query.type === QueryType.aiText) {
186
result = await (this.queryProviderPair as IAITextQueryProviderPair).provider.provideAITextSearchResults(this.queryProviderPair.query.contentPattern, searchOptions, progress, token);
188
>
result = await (this.queryProviderPair as ITextQueryProviderPair).provider.provideTextSearchResults(patternInfoToQuery(this.queryProviderPair.query.contentPattern), searchOptions, progress, token);
190
>
if (testingPs.length) {
192
>
}
194
>
return result;
197
>
private getSearchOptionsForFolder(fq: IFolderQuery<URI>): TextSearchProviderFolderOptions {
198
>
const includes = resolvePatternsForProvider(this.query.includePattern, fq.includePattern);
textSearchManager.ts ×16
199
>
200
>
let excludePattern = fq.excludePattern?.map(e => ({
202
>
patterns: resolvePatternsForProvider(this.query.excludePattern, e.pattern)
204
>
205
>
if (!excludePattern || excludePattern.length === 0) {
207
>
folder: undefined,
208
>
patterns: resolvePatternsForProvider(this.query.excludePattern, undefined)
209
>
}];
210
>
}
212
>
213
>
const options = {
214
>
folder: URI.from(fq.folder),
215
>
excludes,
216
>
includes,
217
>
useIgnoreFiles: {
218
>
local: !fq.disregardIgnoreFiles,
219
>
parent: !fq.disregardParentIgnoreFiles,
220
>
global: !fq.disregardGlobalIgnoreFiles
221
>
},
222
>
followSymlinks: !fq.ignoreSymlinks,
223
>
encoding: (fq.fileEncoding && this.fileUtils.toCanonicalName(fq.fileEncoding)) ?? '',
224
>
ignoreGlobCase: this.query.ignoreGlobCase || fq.ignoreGlobCase,
225
>
};
226
>
return options;
227
>
}
229
>
230
>
function patternInfoToQuery(patternInfo: IPatternInfo): TextSearchQuery2 {
textSearchManager.ts ×16
231
>
return {
232
>
isCaseSensitive: patternInfo.isCaseSensitive || false,
233
>
isRegExp: patternInfo.isRegExp || false,
234
>
isWordMatch: patternInfo.isWordMatch || false,
235
>
isMultiline: patternInfo.isMultiline || false,
236
>
pattern: patternInfo.pattern
237
>
};
238
>
}
240
>
export class TextSearchResultsCollector {
241
>
private _batchedCollector: BatchedCollector<IFileMatch>;
242
>
243
>
private _currentFolderIdx: number = -1;
244
>
private _currentUri: URI | undefined;
245
>
private _currentFileMatch: IFileMatch | null = null;
246
>
247
>
constructor(private _onResult: (result: IFileMatch[]) => void) {
248
>
this._batchedCollector = new BatchedCollector<IFileMatch>(512, items => this.sendItems(items));
textSearchManager.ts ×16
249
>
}
251
>
add(data: TextSearchResult2, folderIdx: number): void {
252
>
// Collects TextSearchResults into IInternalFileMatches and collates using BatchedCollector.
textSearchManager.ts ×25
253
>
// This is efficient for ripgrep which sends results back one file at a time. It wouldn't be efficient for other search
254
>
// providers that send results in random order. We could do this step afterwards instead.
255
>
if (this._currentFileMatch && (this._currentFolderIdx !== folderIdx || !resources.isEqual(this._currentUri, data.uri))) {
257
>
this._currentFileMatch = null;
258
>
}
260
>
if (!this._currentFileMatch) {
261
>
this._currentFolderIdx = folderIdx;
262
>
this._currentUri = data.uri;
263
>
this._currentFileMatch = {
264
>
resource: data.uri,
265
>
results: []
266
>
};
267
>
}
268
>
269
>
this._currentFileMatch.results!.push(extensionResultToFrontendResult(data));
270
>
}
272
>
private pushToCollector(): void {
273
>
const size = this._currentFileMatch && this._currentFileMatch.results ?
textSearchManager.ts ×12
277
>
}
279
>
flush(): void {
281
>
this._batchedCollector.flush();
282
>
}
284
>
private sendItems(items: IFileMatch[]): void {
286
>
}
288
>
289
>
function extensionResultToFrontendResult(data: TextSearchResult2): ITextSearchResult {
textSearchManager.ts ×25
290
>
// Warning: result from RipgrepTextSearchEH has fake Range. Don't depend on any other props beyond these...
291
>
if (data instanceof TextSearchMatch2) {
292
>
return {
293
>
previewText: data.previewText,
294
>
rangeLocations: data.ranges.map(r => ({
295
>
preview: {
296
>
startLineNumber: r.previewRange.start.line,
297
>
startColumn: r.previewRange.start.character,
298
>
endLineNumber: r.previewRange.end.line,
299
>
endColumn: r.previewRange.end.character
300
>
} satisfies ISearchRange,
301
>
source: {
302
>
startLineNumber: r.sourceRange.start.line,
303
>
startColumn: r.sourceRange.start.character,
304
>
endLineNumber: r.sourceRange.end.line,
305
>
endColumn: r.sourceRange.end.character
306
>
} satisfies ISearchRange,
307
>
})),
308
>
} satisfies ITextSearchMatch;
309
>
} else {
310
return {
311
text: data.text,
312
lineNumber: data.lineNumber
313
} satisfies ITextSearchContext;
314
}
317
>
318
>
/**
319
>
* Collects items that have a size - before the cumulative size of collected items reaches START_BATCH_AFTER_COUNT, the callback is called for every
320
>
* set of items collected.
321
>
* But after that point, the callback is called with batches of maxBatchSize.
322
>
* If the batch isn't filled within some time, the callback is also called.
323
>
*/
324
>
export class BatchedCollector<T> {
325
>
private static readonly TIMEOUT = 4000;
326
>
327
>
// After START_BATCH_AFTER_COUNT items have been collected, stop flushing on timeout
328
>
private static readonly START_BATCH_AFTER_COUNT = 50;
329
>
330
>
private totalNumberCompleted = 0;
331
>
private batch: T[] = [];
332
>
private batchSize = 0;
333
>
private timeoutHandle: Timeout | undefined;
334
>
335
>
constructor(private maxBatchSize: number, private cb: (items: T[]) => void) {
338
>
addItem(item: T, size: number): void {
341
>
}
343
>
this.addItemToBatch(item, size);
346
>
addItems(items: T[], size: number): void {
347
if (!items) {
348
return;
349
}
350
351
this.addItemsToBatch(items, size);
352
}
354
>
private addItemToBatch(item: T, size: number): void {
356
>
this.batchSize += size;
357
>
this.onUpdate();
358
>
}
360
>
private addItemsToBatch(item: T[], size: number): void {
361
this.batch = this.batch.concat(item);
362
this.batchSize += size;
363
this.onUpdate();
364
}
366
>
private onUpdate(): void {
367
>
if (this.totalNumberCompleted < BatchedCollector.START_BATCH_AFTER_COUNT) {
textSearchManager.ts ×25
368
>
// Flush because we aren't batching yet
369
>
this.flush();
370
>
} else if (this.batchSize >= this.maxBatchSize) {
371
// Flush because the batch is full
372
this.flush();
373
} else if (!this.timeoutHandle) {
374
// No timeout running, start a timeout to flush
375
this.timeoutHandle = setTimeout(() => {
376
this.flush();
377
}, BatchedCollector.TIMEOUT);
378
}
381
>
flush(): void {
384
>
this.cb(this.batch);
385
>
this.batch = [];
386
>
this.batchSize = 0;
387
>
388
>
if (this.timeoutHandle) {
389
clearTimeout(this.timeoutHandle);
390
this.timeoutHandle = undefined;
391
}