1
>
/*---------------------------------------------------------------------------------------------
agentHostTelemetryReporter.ts
2
>
* Copyright (c) Microsoft Corporation. All rights reserved.
3
>
* Licensed under the MIT License. See License.txt in the project root for license information.
4
>
*--------------------------------------------------------------------------------------------*/
5
>
6
>
import type { LanguageModelToolInvokedClassification, LanguageModelToolInvokedEvent } from '../../telemetry/common/languageModelToolTelemetry.js';
7
>
import type { ITelemetryService } from '../../telemetry/common/telemetry.js';
8
>
import { TelemetryTrustedValue } from '../../telemetry/common/telemetryUtils.js';
9
>
import { hash } from '../../../base/common/hash.js';
10
>
import { AgentSession } from '../common/agentService.js';
11
>
import type { ErrorInfo, MessageAttachment, SessionInputRequestKind, ToolDefinition } from '../common/state/protocol/state.js';
12
>
import { isAhpChatChannel, isSubagentChatUri, isSubagentSession, parseRequiredSessionUriFromChatUri, type ISessionWithDefaultChat } from '../common/state/sessionState.js';
13
>
import type { ToolInvokedResult } from './agentHostToolCallTracker.js';
14
>
import { multiplexProperties, type IAgentHostRestrictedTelemetry, type IAgentHostRestrictedTelemetryContext } from './agentHostRestrictedTelemetry.js';
15
>
16
>
export type AgentHostUserMessageSentSource = 'direct' | 'queued';
17
>
18
>
export interface IAgentHostUserMessageSentEvent {
19
>
provider: string;
20
>
agentSessionId: string;
21
>
source: AgentHostUserMessageSentSource;
22
>
isSubagentSession: boolean;
23
>
turnCount: number;
24
>
activeClientId?: string;
25
>
activeClientToolCount?: number;
26
>
activeClientCustomizationCount?: number;
27
>
attachmentCount: number;
28
>
}
29
>
30
>
export type IAgentHostUserMessageSentClassification = {
31
>
provider: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The provider handling the agent host session.' };
32
>
agentSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The agent host session identifier.' };
33
>
source: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user message was sent directly or from the queued-message flow.' };
34
>
isSubagentSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the message was sent to a subagent session.' };
35
>
turnCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of completed turns in the session when the message was sent.' };
36
>
activeClientId?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The identifier of the first active client for the session, if any.' };
37
>
activeClientToolCount?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The total number of tools provided by the active clients, if any.' };
38
>
activeClientCustomizationCount?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The total number of customizations provided by the active clients, if any.' };
39
>
attachmentCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of attachments included with the user message.' };
40
>
owner: 'roblourens';
41
>
comment: 'Tracks user messages sent from the agent host process to an agent provider.';
42
>
};
43
>
44
>
export type AgentHostTurnResult = 'success' | 'error' | 'cancelled';
45
>
export type AgentHostModelTelemetryKind = 'trusted' | 'byok' | 'unknown';
46
>
type AgentHostModelSelectionKind = 'default' | 'auto' | 'explicit';
47
>
export type AgentHostTurnFailureStage = 'validation' | 'workingDirectory' | 'modelSelection' | 'sendMessage' | 'provider';
48
>
49
>
export interface IAgentHostTurnCompletedEvent {
50
>
provider: string;
51
>
agentSessionId: string;
52
>
turnId: string;
53
>
timeToFirstProgress: number | undefined;
54
>
totalTime: number;
55
>
result: AgentHostTurnResult;
56
>
model: string | TelemetryTrustedValue<string> | undefined;
57
>
modelSelectionKind: AgentHostModelSelectionKind;
58
>
permissionLevel: string | undefined;
59
>
errorType: string | undefined;
60
>
failureStage: AgentHostTurnFailureStage | undefined;
61
>
}
62
>
63
>
export type IAgentHostTurnCompletedClassification = {
64
>
provider: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The provider handling the agent host session.' };
65
>
agentSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The agent host session identifier.' };
66
>
turnId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The identifier of the turn within the agent host session.' };
67
>
timeToFirstProgress: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Time in milliseconds from turn start to the first visible progress (text delta, response part, tool call start, or reasoning).' };
68
>
totalTime: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Total time in milliseconds from turn start to turn completion.' };
69
>
result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the turn completed successfully, with an error, or was cancelled.' };
70
>
model: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The trusted provider model identifier selected at turn start, or a generic value for BYOK and unknown models.' };
71
>
modelSelectionKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the client used the provider default, Auto, or an explicit model.' };
72
>
permissionLevel: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The tool auto-approval level configured for the session at turn start (e.g. default, autoApprove, autopilot).' };
73
>
errorType: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The structured agent host or provider error type when the turn fails.' };
74
>
failureStage: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The bounded stage at which the agent host turn failed.' };
75
>
owner: 'roblourens';
76
>
comment: 'Tracks agent host turn performance including time to first visible progress and total turn duration.';
77
>
};
78
>
79
>
export interface IAgentHostTurnFailedEvent {
80
>
provider: string;
81
>
agentSessionId: string;
82
>
turnId: string;
83
>
failureStage: AgentHostTurnFailureStage;
84
>
errorType: string;
85
>
errorName: string | undefined;
86
>
errorCode: string | undefined;
87
>
msg: string;
88
>
callstack: string | undefined;
89
>
}
90
>
91
>
export type IAgentHostTurnFailedClassification = {
92
>
provider: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The provider handling the failed agent host turn.' };
93
>
agentSessionId: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The agent host session identifier.' };
94
>
turnId: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The identifier of the failed turn within the agent host session.' };
95
>
failureStage: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The bounded stage at which the agent host turn failed.' };
96
>
errorType: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The structured agent host or provider error type.' };
97
>
errorName: { classification: 'CallstackOrException'; purpose: 'PerformanceAndHealth'; comment: 'The name of the exception, when available.' };
98
>
errorCode: { classification: 'CallstackOrException'; purpose: 'PerformanceAndHealth'; comment: 'The exception or protocol error code, when available.' };
99
>
msg: { classification: 'CallstackOrException'; purpose: 'PerformanceAndHealth'; comment: 'The error message. VS Code telemetry scrubs file paths and likely secrets before transmission.' };
100
>
callstack: { classification: 'CallstackOrException'; purpose: 'PerformanceAndHealth'; comment: 'The error stack. VS Code telemetry scrubs file paths and likely secrets before transmission.' };
101
>
owner: 'roblourens';
102
>
comment: 'Captures diagnostic details for failed agent host turns.';
103
>
};
104
>
105
>
export interface IAgentHostTurnFailure {
106
>
stage: AgentHostTurnFailureStage;
107
>
error: ErrorInfo;
108
>
errorName?: string;
109
>
errorCode?: string;
110
>
errorStack?: string;
111
>
}
112
>
113
>
export interface IAgentHostTurnCompletedReport {
114
>
provider: string;
115
>
session: string;
116
>
turnId: string;
117
>
timeToFirstProgress: number | undefined;
118
>
totalTime: number;
119
>
result: AgentHostTurnResult;
120
>
model: string | undefined;
121
>
modelTelemetryKind: AgentHostModelTelemetryKind | undefined;
122
>
permissionLevel: string | undefined;
123
>
failure: IAgentHostTurnFailure | undefined;
124
>
}
125
>
126
>
export interface IAgentHostToolInvokedReport {
127
>
provider: string;
128
>
session: string;
129
>
toolId: string;
130
>
toolSourceKind: string;
131
>
result: ToolInvokedResult;
132
>
invocationTimeMs: number;
133
>
}
134
>
135
>
export interface IAgentHostToolCallDetailsReport {
136
>
session: string;
137
>
turnId: string;
138
>
model: string | undefined;
139
>
responseType: string;
140
>
/** Count of invocations keyed by tool name, across all rounds in the turn. */
141
>
toolCounts: Record<string, number>;
142
>
/** Names of the tools offered to the model for this turn. */
143
>
availableTools: readonly string[];
144
>
/** Number of model-call rounds in the turn, including the final tool-free response round (matches the extension's `toolCallRounds.length`). */
145
>
numRequests: number;
146
>
totalToolCalls: number;
147
>
parallelToolCallRounds: number;
148
>
parallelToolCallsTotal: number;
149
>
}
150
>
151
>
export interface IAgentHostSkillContentReadReport {
152
>
/** The skill name. */
153
>
name: string;
154
>
/** Path to the SKILL.md file. */
155
>
path: string;
156
>
/** Full skill content; hashed (never sent raw), matching the extension. */
157
>
content: string;
158
>
/** Where the skill was discovered (project, personal-copilot, plugin, builtin, …). */
159
>
source: string | undefined;
160
>
/** Name of the plugin the skill came from, when applicable (AH-native analog of the extension's skill extension id). */
161
>
pluginName: string | undefined;
162
>
/** Version of the plugin the skill came from, when applicable. */
163
>
pluginVersion: string | undefined;
164
>
}
165
>
166
>
export type AgentHostRepoInfoResult = 'success' | 'filesChanged' | 'diffTooLarge' | 'noChanges' | 'tooManyChanges' | 'mergeBaseTooOld' | 'virtualFileSystem' | 'tooManyCommits';
167
>
168
>
export interface IAgentHostRepoInfoReport {
169
>
telemetryMessageId: string;
170
>
location: 'begin' | 'end';
171
>
remoteUrl: string;
172
>
repoId: string;
173
>
repoType: 'github' | 'ado';
174
>
headCommitHash: string;
175
>
headBranchName: string | undefined;
176
>
fileRelativePaths: string | undefined;
177
>
diffsJSON: string | undefined;
178
>
result: AgentHostRepoInfoResult;
179
>
isActiveRepository: 'true';
180
>
workspaceFileCount: number;
181
>
changedFileCount: number;
182
>
diffSizeBytes: number;
183
>
}
184
>
185
>
export interface IAgentHostToolCallStalledEvent {
186
>
provider: string;
187
>
agentSessionId: string;
188
>
isSubagentSession: boolean;
189
>
blockerKind: SessionInputRequestKind.ToolConfirmation | SessionInputRequestKind.ToolClientExecution | SessionInputRequestKind.ToolAuthentication;
190
>
toolId: string;
191
>
toolSourceKind: string;
192
>
stalledTimeMs: number;
193
>
}
194
>
195
>
export type IAgentHostToolCallStalledClassification = {
196
>
provider: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The provider handling the stalled agent host tool call.' };
197
>
agentSessionId: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The agent host session identifier.' };
198
>
isSubagentSession: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Whether the stalled tool call belongs to a subagent session.' };
199
>
blockerKind: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Whether the tool call is waiting for confirmation or client execution.' };
200
>
toolId: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The identifier of the stalled tool.' };
201
>
toolSourceKind: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Whether the stalled tool is provided by the agent host, an MCP server, or a client.' };
202
>
stalledTimeMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Time in milliseconds that the tool call has remained blocked.' };
203
>
owner: 'roblourens';
204
>
comment: 'Tracks agent host tool calls that remain blocked beyond the stall threshold.';
205
>
};
206
>
207
>
export interface IAgentHostToolCallStalledReport {
208
>
provider: string;
209
>
session: string;
210
>
blockerKind: SessionInputRequestKind.ToolConfirmation | SessionInputRequestKind.ToolClientExecution | SessionInputRequestKind.ToolAuthentication;
211
>
toolId: string;
212
>
toolSourceKind: string;
213
>
stalledTimeMs: number;
214
>
}
215
>
216
>
export interface IAgentHostStalledToolCallCompletedEvent {
217
>
provider: string;
218
>
agentSessionId: string;
219
>
isSubagentSession: boolean;
220
>
blockerKind: SessionInputRequestKind.ToolConfirmation | SessionInputRequestKind.ToolClientExecution | SessionInputRequestKind.ToolAuthentication;
221
>
toolId: string;
222
>
toolSourceKind: string;
223
>
result: ToolInvokedResult;
224
>
totalTimeMs: number;
225
>
timeAfterStallMs: number;
226
>
}
227
>
228
>
export type IAgentHostStalledToolCallCompletedClassification = {
229
>
provider: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The provider handling the completed agent host tool call.' };
230
>
agentSessionId: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The agent host session identifier.' };
231
>
isSubagentSession: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Whether the completed tool call belongs to a subagent session.' };
232
>
blockerKind: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Whether the tool call had stalled waiting for confirmation or client execution.' };
233
>
toolId: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The identifier of the completed tool.' };
234
>
toolSourceKind: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Whether the completed tool is provided by the agent host, an MCP server, or a client.' };
235
>
result: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Whether the stalled tool call eventually completed successfully, with an error, or through user cancellation.' };
236
>
totalTimeMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Total time in milliseconds from tool call start to completion.' };
237
>
timeAfterStallMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Time in milliseconds from the stall report to tool call completion.' };
238
>
owner: 'roblourens';
239
>
comment: 'Tracks agent host tool calls that complete after previously exceeding the stall threshold.';
240
>
};
241
>
242
>
export interface IAgentHostStalledToolCallCompletedReport {
243
>
provider: string;
244
>
session: string;
245
>
blockerKind: SessionInputRequestKind.ToolConfirmation | SessionInputRequestKind.ToolClientExecution | SessionInputRequestKind.ToolAuthentication;
246
>
toolId: string;
247
>
toolSourceKind: string;
248
>
result: ToolInvokedResult;
249
>
totalTimeMs: number;
250
>
timeAfterStallMs: number;
251
>
}
252
>
253
>
export class AgentHostTelemetryReporter {
254
>
255
>
constructor(private readonly _telemetryService: ITelemetryService) { }
256
>
257
>
/** The restricted GH/MSFT telemetry surface, present when the agent-host telemetry service is wired. */
258
>
private get _restricted(): IAgentHostRestrictedTelemetry | undefined {
259
const ts = this._telemetryService as Partial<IAgentHostRestrictedTelemetry>;
260
return typeof ts.sendEnhancedGHTelemetryEvent === 'function' ? ts as IAgentHostRestrictedTelemetry : undefined;
261
}
263
>
userMessageSent(provider: string, session: string, sessionState: ISessionWithDefaultChat | undefined, source: AgentHostUserMessageSentSource, attachments: readonly MessageAttachment[] | undefined): void {
264
const attachmentCount = attachments?.length ?? 0;
265
const activeClients = sessionState?.activeClients ?? [];