src/vs/workbench/services/search/node/rawSearchService.ts
449 LOC · 89 covered · 360 uncovered · 22 ranges · 60 concepts · 1 introducers · 32 tests
File neighbourhood
The centred file is linked to every concept that introduces one of its ranges, every test that runs code from the file, and the gray connector concepts standing between those tests and the file's own introducer concepts. Undirected links join concepts to every file where they introduce source and concepts to the tests they introduce; arrows show specialization between the displayed concepts and bridge only concepts omitted from this view. Concept colors match the source ranges below; connector concepts have no source color and are shown in gray.
Focused file, its introducer and connector concepts, their introduced files, and tests that run code from the file
In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the related-file, concept, and source links on this page.
Graph controls are ready.
Interactive rendering requires JavaScript and WebGL. Use the related-file, concept, and source links on this page while the interactive map is unavailable.
1
>
/*---------------------------------------------------------------------------------------------
fileSearch.ts ×31
2
>
* Copyright (c) Microsoft Corporation. All rights reserved.
3
>
* Licensed under the MIT License. See License.txt in the project root for license information.
4
>
*--------------------------------------------------------------------------------------------*/
5
>
6
>
import * as arrays from '../../../../base/common/arrays.js';
7
>
import { CancelablePromise, createCancelablePromise } from '../../../../base/common/async.js';
8
>
import { CancellationToken } from '../../../../base/common/cancellation.js';
9
>
import { canceled } from '../../../../base/common/errors.js';
10
>
import { Emitter, Event } from '../../../../base/common/event.js';
11
>
import { compareItemsByFuzzyScore, FuzzyScorerCache, IItemAccessor, prepareQuery } from '../../../../base/common/fuzzyScorer.js';
12
>
import { revive } from '../../../../base/common/marshalling.js';
13
>
import { basename, dirname, join, sep } from '../../../../base/common/path.js';
14
>
import { StopWatch } from '../../../../base/common/stopwatch.js';
15
>
import { URI, UriComponents } from '../../../../base/common/uri.js';
16
>
import { ByteSize } from '../../../../platform/files/common/files.js';
17
>
import { DEFAULT_MAX_SEARCH_RESULTS, ICachedSearchStats, IFileQuery, IFileSearchProgressItem, IFileSearchStats, IFolderQuery, IProgressMessage, IRawFileMatch, IRawFileQuery, IRawQuery, IRawSearchService, IRawTextQuery, ISearchEngine, ISearchEngineSuccess, ISerializedFileMatch, ISerializedSearchComplete, ISerializedSearchProgressItem, ISerializedSearchSuccess, isFilePatternMatch, ITextQuery } from '../common/search.js';
18
>
import { Engine as FileSearchEngine } from './fileSearch.js';
19
>
import { TextSearchEngineAdapter } from './textSearchAdapter.js';
20
>
21
>
export type IProgressCallback = (p: ISerializedSearchProgressItem) => void;
22
>
type IFileProgressCallback = (p: IFileSearchProgressItem) => void;
23
>
24
>
export class SearchService implements IRawSearchService {
25
>
26
>
private static readonly BATCH_SIZE = 512;
27
>
28
>
private caches: { [cacheKey: string]: Cache } = Object.create(null);
29
>
30
>
constructor(private readonly processType: IFileSearchStats['type'] = 'searchProcess', private readonly getNumThreads?: () => Promise<number | undefined>) { }
31
>
32
>
fileSearch(config: IRawFileQuery): Event<ISerializedSearchProgressItem | ISerializedSearchComplete> {
33
let promise: CancelablePromise<ISerializedSearchSuccess>;
34
35
const query = reviveQuery(config);
36
const emitter = new Emitter<ISerializedSearchProgressItem | ISerializedSearchComplete>({
37
onDidAddFirstListener: () => {
38
promise = createCancelablePromise(async token => {
39
const numThreads = await this.getNumThreads?.();
40
return this.doFileSearchWithEngine(FileSearchEngine, query, p => emitter.fire(p), token, SearchService.BATCH_SIZE, numThreads);
41
});
42
43
promise.then(
44
c => emitter.fire(c),
45
err => emitter.fire({ type: 'error', error: { message: err.message, stack: err.stack } }));
46
},
47
onDidRemoveLastListener: () => {
48
promise.cancel();
49
}
50
});
51
52
return emitter.event;
53
}
55
>
textSearch(rawQuery: IRawTextQuery): Event<ISerializedSearchProgressItem | ISerializedSearchComplete> {
56
let promise: CancelablePromise<ISerializedSearchComplete>;
57
58
const query = reviveQuery(rawQuery);
59
const emitter = new Emitter<ISerializedSearchProgressItem | ISerializedSearchComplete>({
60
onDidAddFirstListener: () => {
61
promise = createCancelablePromise(token => {
62
return this.ripgrepTextSearch(query, p => emitter.fire(p), token);
63
});
64
65
promise.then(
66
c => emitter.fire(c),
67
err => emitter.fire({ type: 'error', error: { message: err.message, stack: err.stack } }));
68
},
69
onDidRemoveLastListener: () => {
70
promise.cancel();
71
}
72
});
73
74
return emitter.event;
75
}
77
>
private async ripgrepTextSearch(config: ITextQuery, progressCallback: IProgressCallback, token: CancellationToken): Promise<ISerializedSearchSuccess> {
78
config.maxFileSize = this.getPlatformFileLimits().maxFileSize;
79
const numThreads = await this.getNumThreads?.();
80
const engine = new TextSearchEngineAdapter(config, numThreads);
81
82
return engine.search(token, progressCallback, progressCallback);
83
}
85
>
private getPlatformFileLimits(): { readonly maxFileSize: number } {
86
return {
87
maxFileSize: 16 * ByteSize.GB
88
};
89
}
91
>
doFileSearch(config: IFileQuery, numThreads: number | undefined, progressCallback: IProgressCallback, token?: CancellationToken): Promise<ISerializedSearchSuccess> {
92
return this.doFileSearchWithEngine(FileSearchEngine, config, progressCallback, token, SearchService.BATCH_SIZE, numThreads);
93
}
95
>
doFileSearchWithEngine(EngineClass: { new(config: IFileQuery, numThreads?: number | undefined): ISearchEngine<IRawFileMatch> }, config: IFileQuery, progressCallback: IProgressCallback, token?: CancellationToken, batchSize = SearchService.BATCH_SIZE, threads?: number): Promise<ISerializedSearchSuccess> {
96
let resultCount = 0;
97
const fileProgressCallback: IFileProgressCallback = progress => {
98
if (Array.isArray(progress)) {
99
resultCount += progress.length;
100
progressCallback(progress.map(m => this.rawMatchToSearchItem(m)));
101
} else if ((<IRawFileMatch>progress).relativePath) {
102
resultCount++;
103
progressCallback(this.rawMatchToSearchItem(<IRawFileMatch>progress));
104
} else {
105
progressCallback(<IProgressMessage>progress);
106
}
107
};
108
109
if (config.sortByScore) {
110
let sortedSearch = this.trySortedSearchFromCache(config, fileProgressCallback, token);
111
if (!sortedSearch) {
112
const walkerConfig = config.maxResults ? Object.assign({}, config, { maxResults: null }) : config;
113
const engine = new EngineClass(walkerConfig, threads);
114
sortedSearch = this.doSortedSearch(engine, config, progressCallback, fileProgressCallback, token);
115
}
116
117
return new Promise<ISerializedSearchSuccess>((c, e) => {
118
sortedSearch.then(([result, rawMatches]) => {
119
const serializedMatches = rawMatches.map(rawMatch => this.rawMatchToSearchItem(rawMatch));
120
this.sendProgress(serializedMatches, progressCallback, batchSize);
121
c(result);
122
}, e);
123
});
124
}
125
126
const engine = new EngineClass(config, threads);
127
128
return this.doSearch(engine, fileProgressCallback, batchSize, token).then(complete => {
129
return {
130
limitHit: complete.limitHit,
131
type: 'success',
132
stats: {
133
detailStats: complete.stats,
134
type: this.processType,
135
fromCache: false,
136
resultCount,
137
sortingTime: undefined
138
},
139
messages: []
140
};
141
});
142
}
144
>
private rawMatchToSearchItem(match: IRawFileMatch): ISerializedFileMatch {
145
return { path: match.base ? join(match.base, match.relativePath) : match.relativePath };
146
}
148
>
private doSortedSearch(engine: ISearchEngine<IRawFileMatch>, config: IFileQuery, progressCallback: IProgressCallback, fileProgressCallback: IFileProgressCallback, token?: CancellationToken): Promise<[ISerializedSearchSuccess, IRawFileMatch[]]> {
149
const emitter = new Emitter<IFileSearchProgressItem>();
150
151
let allResultsPromise = createCancelablePromise(token => {
152
let results: IRawFileMatch[] = [];
153
154
const innerProgressCallback: IFileProgressCallback = progress => {
155
if (Array.isArray(progress)) {
156
results = progress;
157
} else {
158
fileProgressCallback(progress);
159
emitter.fire(progress);
160
}
161
};
162
163
return this.doSearch(engine, innerProgressCallback, -1, token)
164
.then<[ISearchEngineSuccess, IRawFileMatch[]]>(result => {
165
return [result, results];
166
});
167
});
168
169
let cache: Cache;
170
if (config.cacheKey) {
171
cache = this.getOrCreateCache(config.cacheKey);
172
const cacheRow: ICacheRow = {
173
promise: allResultsPromise,
174
event: emitter.event,
175
resolved: false
176
};
177
cache.resultsToSearchCache[config.filePattern || ''] = cacheRow;
178
allResultsPromise.then(() => {
179
cacheRow.resolved = true;
180
}, err => {
181
delete cache.resultsToSearchCache[config.filePattern || ''];
182
});
183
184
allResultsPromise = this.preventCancellation(allResultsPromise);
185
}
186
187
return allResultsPromise.then(([result, results]) => {
188
const scorerCache: FuzzyScorerCache = cache ? cache.scorerCache : Object.create(null);
189
const sortSW = (typeof config.maxResults !== 'number' || config.maxResults > 0) && StopWatch.create(false);
190
return this.sortResults(config, results, scorerCache, token)
191
.then<[ISerializedSearchSuccess, IRawFileMatch[]]>(sortedResults => {
192
// sortingTime: -1 indicates a "sorted" search that was not sorted, i.e. populating the cache when quickaccess is opened.
193
// Contrasting with findFiles which is not sorted and will have sortingTime: undefined
194
const sortingTime = sortSW ? sortSW.elapsed() : -1;
195
196
return [{
197
type: 'success',
198
stats: {
199
detailStats: result.stats,
200
sortingTime,
201
fromCache: false,
202
type: this.processType,
203
resultCount: sortedResults.length
204
},
205
messages: result.messages,
206
limitHit: result.limitHit || typeof config.maxResults === 'number' && results.length > config.maxResults
207
}, sortedResults];
208
});
209
});
210
}
212
>
private getOrCreateCache(cacheKey: string): Cache {
213
const existing = this.caches[cacheKey];
214
if (existing) {
215
return existing;
216
}
217
return this.caches[cacheKey] = new Cache();
218
}
220
>
private trySortedSearchFromCache(config: IFileQuery, progressCallback: IFileProgressCallback, token?: CancellationToken): Promise<[ISerializedSearchSuccess, IRawFileMatch[]]> | undefined {
221
const cache = config.cacheKey && this.caches[config.cacheKey];
222
if (!cache) {
223
return undefined;
224
}
225
226
const cached = this.getResultsFromCache(cache, config.filePattern || '', progressCallback, token);
227
if (cached) {
228
return cached.then(([result, results, cacheStats]) => {
229
const sortSW = StopWatch.create(false);
230
return this.sortResults(config, results, cache.scorerCache, token)
231
.then<[ISerializedSearchSuccess, IRawFileMatch[]]>(sortedResults => {
232
const sortingTime = sortSW.elapsed();
233
const stats: IFileSearchStats = {
234
fromCache: true,
235
detailStats: cacheStats,
236
type: this.processType,
237
resultCount: results.length,
238
sortingTime
239
};
240
241
return [
242
{
243
type: 'success',
244
limitHit: result.limitHit || typeof config.maxResults === 'number' && results.length > config.maxResults,
245
stats,
246
messages: [],
247
} satisfies ISerializedSearchSuccess,
248
sortedResults
249
];
250
});
251
});
252
}
253
return undefined;
254
}
256
>
private sortResults(config: IFileQuery, results: IRawFileMatch[], scorerCache: FuzzyScorerCache, token?: CancellationToken): Promise<IRawFileMatch[]> {
257
// we use the same compare function that is used later when showing the results using fuzzy scoring
258
// this is very important because we are also limiting the number of results by config.maxResults
259
// and as such we want the top items to be included in this result set if the number of items
260
// exceeds config.maxResults.
261
const query = prepareQuery(config.filePattern || '');
262
const compare = (matchA: IRawFileMatch, matchB: IRawFileMatch) => compareItemsByFuzzyScore(matchA, matchB, query, true, FileMatchItemAccessor, scorerCache);
263
264
const maxResults = typeof config.maxResults === 'number' ? config.maxResults : DEFAULT_MAX_SEARCH_RESULTS;
265
return arrays.topAsync(results, compare, maxResults, 10000, token);
266
}
268
>
private sendProgress(results: ISerializedFileMatch[], progressCb: IProgressCallback, batchSize: number) {
269
if (batchSize && batchSize > 0) {
270
for (let i = 0; i < results.length; i += batchSize) {
271
progressCb(results.slice(i, i + batchSize));
272
}
273
} else {
274
progressCb(results);
275
}
276
}
278
>
private getResultsFromCache(cache: Cache, searchValue: string, progressCallback: IFileProgressCallback, token?: CancellationToken): Promise<[ISearchEngineSuccess, IRawFileMatch[], ICachedSearchStats]> | null {
279
const cacheLookupSW = StopWatch.create(false);
280
281
// Find cache entries by prefix of search value
282
const hasPathSep = searchValue.indexOf(sep) >= 0;
283
let cachedRow: ICacheRow | undefined;
284
for (const previousSearch in cache.resultsToSearchCache) {
285
// If we narrow down, we might be able to reuse the cached results
286
if (searchValue.startsWith(previousSearch)) {
287
if (hasPathSep && previousSearch.indexOf(sep) < 0 && previousSearch !== '') {
288
continue; // since a path character widens the search for potential more matches, require it in previous search too
289
}
290
291
const row = cache.resultsToSearchCache[previousSearch];
292
cachedRow = {
293
promise: this.preventCancellation(row.promise),
294
event: row.event,
295
resolved: row.resolved
296
};
297
break;
298
}
299
}
300
301
if (!cachedRow) {
302
return null;
303
}
304
305
const cacheLookupTime = cacheLookupSW.elapsed();
306
const cacheFilterSW = StopWatch.create(false);
307
308
const listener = cachedRow.event(progressCallback);
309
if (token) {
310
token.onCancellationRequested(() => {
311
listener.dispose();
312
});
313
}
314
315
return cachedRow.promise.then<[ISearchEngineSuccess, IRawFileMatch[], ICachedSearchStats]>(([complete, cachedEntries]) => {
316
if (token && token.isCancellationRequested) {
317
throw canceled();
318
}
319
320
// Pattern match on results
321
const results: IRawFileMatch[] = [];
322
const normalizedSearchValueLowercase = prepareQuery(searchValue).normalizedLowercase;
323
for (const entry of cachedEntries) {
324
325
// Check if this entry is a match for the search value
326
if (!isFilePatternMatch(entry, normalizedSearchValueLowercase)) {
327
continue;
328
}
329
330
results.push(entry);
331
}
332
333
return [complete, results, {
334
cacheWasResolved: cachedRow.resolved,
335
cacheLookupTime,
336
cacheFilterTime: cacheFilterSW.elapsed(),
337
cacheEntryCount: cachedEntries.length
338
}];
339
});
340
}
342
>
343
>
344
>
private doSearch(engine: ISearchEngine<IRawFileMatch>, progressCallback: IFileProgressCallback, batchSize: number, token?: CancellationToken): Promise<ISearchEngineSuccess> {
345
return new Promise<ISearchEngineSuccess>((c, e) => {
346
let batch: IRawFileMatch[] = [];
347
token?.onCancellationRequested(() => engine.cancel());
348
349
engine.search((match) => {
350
if (match) {
351
if (batchSize) {
352
batch.push(match);
353
if (batchSize > 0 && batch.length >= batchSize) {
354
progressCallback(batch);
355
batch = [];
356
}
357
} else {
358
progressCallback(match);
359
}
360
}
361
}, (progress) => {
362
progressCallback(progress);
363
}, (error, complete) => {
364
if (batch.length) {
365
progressCallback(batch);
366
}
367
368
if (error) {
369
progressCallback({ message: 'Search finished. Error: ' + error.message });
370
e(error);
371
} else {
372
progressCallback({ message: 'Search finished. Stats: ' + JSON.stringify(complete.stats) });
373
c(complete);
374
}
375
});
376
});
377
}
379
>
clearCache(cacheKey: string): Promise<void> {
380
delete this.caches[cacheKey];
381
return Promise.resolve(undefined);
382
}
384
>
/**
385
>
* Return a CancelablePromise which is not actually cancelable
386
>
* TODO@rob - Is this really needed?
387
>
*/
388
>
private preventCancellation<C>(promise: CancelablePromise<C>): CancelablePromise<C> {
389
return new class implements CancelablePromise<C> {
390
get [Symbol.toStringTag]() { return this.toString(); }
391
cancel() {
392
// Do nothing
393
}
394
then<TResult1 = C, TResult2 = never>(resolve?: ((value: C) => TResult1 | Promise<TResult1>) | undefined | null, reject?: ((reason: any) => TResult2 | Promise<TResult2>) | undefined | null): Promise<TResult1 | TResult2> {
395
return promise.then(resolve, reject);
396
}
397
catch(reject?: any) {
398
return this.then(undefined, reject);
399
}
400
finally(onFinally: any) {
401
return promise.finally(onFinally);
402
}
403
};
404
}
406
>
407
>
interface ICacheRow {
408
>
// TODO@roblou - never actually canceled
409
>
promise: CancelablePromise<[ISearchEngineSuccess, IRawFileMatch[]]>;
410
>
resolved: boolean;
411
>
readonly event: Event<IFileSearchProgressItem>;
412
>
}
413
>
414
class Cache {
415
416
resultsToSearchCache: { [searchValue: string]: ICacheRow } = Object.create(null);
417
418
scorerCache: FuzzyScorerCache = Object.create(null);
420
>
421
>
const FileMatchItemAccessor = new class implements IItemAccessor<IRawFileMatch> {
422
>
423
>
getItemLabel(match: IRawFileMatch): string {
424
return basename(match.relativePath); // e.g. myFile.txt
425
}
427
>
getItemDescription(match: IRawFileMatch): string {
428
return dirname(match.relativePath); // e.g. some/path/to/file
429
}
431
>
getItemPath(match: IRawFileMatch): string {
432
return match.relativePath; // e.g. some/path/to/file/myFile.txt
433
}
435
>
436
function reviveQuery<U extends IRawQuery>(rawQuery: U): U extends IRawTextQuery ? ITextQuery : IFileQuery {
437
return {
438
// eslint-disable-next-line local/code-no-any-casts
439
...<any>rawQuery, // TODO
440
...{
441
folderQueries: rawQuery.folderQueries && rawQuery.folderQueries.map(reviveFolderQuery),
442
extraFileResources: rawQuery.extraFileResources && rawQuery.extraFileResources.map(components => URI.revive(components))
443
}
444
};
445
}
447
function reviveFolderQuery(rawFolderQuery: IFolderQuery<UriComponents>): IFolderQuery<URI> {
448
return revive(rawFolderQuery);
449
}