1
>
/*---------------------------------------------------------------------------------------------
editSurvivalReporter.ts
2
>
* Copyright (c) Microsoft Corporation. All rights reserved.
3
>
* Licensed under the MIT License. See License.txt in the project root for license information.
4
>
*--------------------------------------------------------------------------------------------*/
5
>
6
>
import { TimeoutTimer } from '../../../../base/common/async.js';
7
>
import { Disposable, type IDisposable } from '../../../../base/common/lifecycle.js';
8
>
import { extname } from '../../../../base/common/path.js';
9
>
import { URI } from '../../../../base/common/uri.js';
10
>
import { FileOperationResult, IFileService, toFileOperationResult } from '../../../files/common/files.js';
11
>
import { createDecorator } from '../../../instantiation/common/instantiation.js';
12
>
import { ILogService } from '../../../log/common/log.js';
13
>
import { ITelemetryService } from '../../../telemetry/common/telemetry.js';
14
>
import { AgentSession } from '../../common/agentService.js';
15
>
import { isAhpChatChannel, parseRequiredSessionUriFromChatUri } from '../../common/state/sessionState.js';
16
>
import { computeChunkedEditSurvival, computeWholeFileEditSurvival } from './editSurvivalTracker.js';
17
>
18
>
/**
19
>
* Parameters describing a single completed tool-driven file edit that the
20
>
* agent host wants to follow over time.
21
>
*
22
>
* Only first-party edit tools that go through `FileEditTracker` produce
23
>
* these — file modifications from shell tools like `Bash` are not
24
>
* observable here.
25
>
*
26
>
* Notebook tools (`NotebookEdit`) are skipped by the launcher for now
27
>
* to avoid the complexity of scoring against notebook JSON. We may
28
>
* revisit when we have a notebook-aware tracker.
29
>
*/
30
>
export interface IEditSurvivalReporterLaunchParams {
31
>
/** Full session URI string (e.g. `claude:/abc123`). */
32
>
readonly sessionUri: string;
33
>
readonly turnId: string;
34
>
readonly toolCallId: string;
35
>
/** Absolute file path on the agent host's local file system. */
36
>
readonly filePath: string;
37
>
/** File content snapshotted before the tool ran (empty for creates). */
38
>
readonly beforeText: string;
39
>
/** File content after the tool ran (the AI's output). */
40
>
readonly afterText: string;
41
>
/** Whether the tool created a new file (no prior content existed). */
42
>
readonly isCreate: boolean;
43
>
/** Name of the edit tool, e.g. `Edit`, `apply_patch`. Empty if unknown. */
44
>
readonly toolName?: string;
45
>
/**
46
>
* Model that produced this edit, e.g. `claude-sonnet-4.5`. Optional
47
>
* defensively, but always expected to be set
48
>
*/
49
>
readonly modelId?: string;
50
>
/**
51
>
* Explicit AI-written text chunks extracted from the tool input
52
>
* (see `editChunkExtractor.ts`). When provided, survival is scored
53
>
* against just these chunks; when omitted or empty, the reporter
54
>
* falls back to whole-file scoring and tags the event with
55
>
* `scoringMode='whole-file'`.
56
>
*/
57
>
readonly aiChunks?: readonly string[];
58
>
}
59
>
60
>
export const IEditSurvivalReporterFactory = createDecorator<IEditSurvivalReporterFactory>('editSurvivalReporterFactory');
61
>
62
>
/**
63
>
* Launches background reporters that sample the on-disk file after a tool
64
>
* edit and emit edit-survival telemetry over the next 15 minutes.
65
>
*/
66
>
export interface IEditSurvivalReporterFactory {
67
>
readonly _serviceBrand: undefined;
68
>
/**
69
>
* Begin tracking a single file edit. The returned disposable can be
70
>
* used to cancel sampling early; otherwise the reporter cleans itself
71
>
* up after the final 15-minute sample.
72
>
*/
73
>
launch(params: IEditSurvivalReporterLaunchParams): IDisposable;
74
>
}
75
>
76
>
/** No-op factory, useful for tests and environments without telemetry. */
77
>
export class NullEditSurvivalReporterFactory implements IEditSurvivalReporterFactory {
78
>
readonly _serviceBrand: undefined;
79
>
launch(_params: IEditSurvivalReporterLaunchParams): IDisposable {
80
return { dispose() { } };
81
}
83
>
84
>
interface IEditSurvivalTelemetryEvent {
85
>
provider: string;
86
>
modelId: string;
87
>
toolName: string;
88
>
agentSessionId: string;
89
>
turnId: string;
90
>
toolCallId: string;
91
>
fileExtension: string;
92
>
survivalRateFourGram: number;
93
>
survivalRateNoRevert: number;
94
>
scoringMode: string;
95
>
aiChunkCount: number;
96
>
aiCharCount: number;
97
>
timeDelayMs: number;
98
>
didFileGetDeleted: number;
99
>
isCreate: number;
100
>
beforeTextLength: number;
101
>
afterTextLength: number;
102
>
currentTextLength: number;
103
>
}
104
>
105
>
type IEditSurvivalTelemetryClassification = {
106
>
provider: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The provider handling the agent host session.' };
107
>
modelId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The model that produced the edit, e.g. "claude-sonnet-4.5" or "gpt-5-mini". Empty if the host could not determine the per-edit model.' };
108
>
toolName: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Name of the edit tool that produced the edit, e.g. "Edit", "apply_patch". Empty if unknown.' };
109
>
agentSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The agent host session identifier.' };
110
>
turnId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The agent host turn identifier this edit belongs to.' };
111
>
toolCallId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The tool call identifier that produced the edit.' };
112
>
fileExtension: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The file extension (including the leading dot) of the edited file, or empty if the file has no extension.' };
113
>
survivalRateFourGram: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'A number between 0 and 1 representing the share of 4-grams the AI wrote that are still present in the file.' };
114
>
survivalRateNoRevert: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'A number between 0 and 1; 1 means the user kept the AI edit and 0 means the user fully reverted it.' };
115
>
scoringMode: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'How survivalRateFourGram was computed: "chunked" (asymmetric, denominator bounded by the AI-written text) or "whole-file" (symmetric, denominator includes the whole file).' };
116
>
aiChunkCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of distinct AI-written text chunks contributing to chunked scoring (0 when scoringMode is "whole-file").' };
117
>
aiCharCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Sum of character lengths of the AI-written chunks (suitable for char-weighted dashboard rollups). Always 0 when scoringMode is "whole-file" because we cannot accurately determine the AI char count from a whole-file snapshot.' };
118
>
timeDelayMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Milliseconds since the edit completed when this sample was taken.' };
119
>
didFileGetDeleted: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: '1 if the file could not be read when the sample was taken (deleted or moved), otherwise 0.' };
120
>
isCreate: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: '1 if the tool call created a new file, otherwise 0.' };
121
>
beforeTextLength: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Length in characters of the file content before the AI edit.' };
122
>
afterTextLength: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Length in characters of the file content the AI wrote.' };
123
>
currentTextLength: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Length in characters of the file content at the time of the sample (0 if the file is missing).' };
124
>
owner: 'roblourens';
125
>
comment: 'Tracks how long AI-produced file edits survive in the user\'s file over the 15 minutes following a tool call in an agent host session. No file contents are reported.';
126
>
};
127
>
128
>
/**
129
>
* Schedule of samples (in milliseconds since the edit completed) at
130
>
* which we read the file again and emit a telemetry event. Matches the
131
>
* chat extension's schedule so the resulting data is comparable.
132
>
*/
133
>
const SAMPLE_SCHEDULE_MS = [0, 5_000, 30_000, 120_000, 300_000, 600_000, 900_000];
134
>
135
>
class SessionEditSurvivalReporter extends Disposable {
136
>
private readonly _startTime = Date.now();
137
>
private _samplesTaken = 0;
138
>
139
>
constructor(
140
private readonly _params: IEditSurvivalReporterLaunchParams,
141
private readonly _fileService: IFileService,