src/vs/workbench/contrib/chat/common/chatImageExtraction.ts
254 LOC · 244 covered · 10 uncovered · 54 ranges · 37 concepts · 26 introducers · 30 tests
File neighbourhood
The centred file is linked to every concept that introduces one of its ranges, every test that runs code from the file, and the gray connector concepts standing between those tests and the file's own introducer concepts. Undirected links join concepts to every file where they introduce source and concepts to the tests they introduce; arrows show specialization between the displayed concepts and bridge only concepts omitted from this view. Concept colors match the source ranges below; connector concepts have no source color and are shown in gray.
Focused file, its introducer and connector concepts, their introduced files, and tests that run code from the file
In the embedded map, ordinary wheel input scrolls the page; use the visible controls to zoom and drag to pan. Open the full-screen map for canvas navigation: wheel pans, Ctrl/Command plus wheel zooms, and arrow keys pan when this region is focused. On touch screens, open the full-screen map to pan or pinch. If JavaScript or WebGL is unavailable, use the related-file, concept, and source links on this page.
Graph controls are ready.
Interactive rendering requires JavaScript and WebGL. Use the related-file, concept, and source links on this page while the interactive map is unavailable.
1
>
/*---------------------------------------------------------------------------------------------
chatImageExtraction.ts ×8
2
>
* Copyright (c) Microsoft Corporation. All rights reserved.
3
>
* Licensed under the MIT License. See License.txt in the project root for license information.
4
>
*--------------------------------------------------------------------------------------------*/
5
>
6
>
import { decodeBase64, VSBuffer } from '../../../../base/common/buffer.js';
7
>
import { IMarkdownString } from '../../../../base/common/htmlContent.js';
8
>
import { getExtensionForMimeType, getMediaMime } from '../../../../base/common/mime.js';
9
>
import { URI } from '../../../../base/common/uri.js';
10
>
import { localize } from '../../../../nls.js';
11
>
import { isLocation } from '../../../../editor/common/languages.js';
12
>
import { IChatResponseViewModel, IChatRequestViewModel, isRequestVM } from './model/chatViewModel.js';
13
>
import { ChatResponseResource } from './model/chatModel.js';
14
>
import { IChatContentInlineReference, IChatToolInvocation, IChatToolInvocationSerialized, IToolResultOutputDetailsSerialized } from './chatService/chatService.js';
15
>
import { isToolResultInputOutputDetails, isToolResultOutputDetails, IToolResultOutputDetails } from './tools/languageModelToolsService.js';
16
>
import { getExplicitFileOrImageAttachmentSummary, type IChatRequestVariableEntry, isImageVariableEntry } from './attachments/chatVariableEntries.js';
17
>
18
>
export interface IChatExtractedImage {
19
>
readonly id: string;
20
>
readonly uri: URI;
21
>
readonly name: string;
22
>
readonly mimeType: string;
23
>
readonly data: VSBuffer;
24
>
readonly source: string;
25
>
readonly caption: string | IMarkdownString | undefined;
26
>
}
27
>
28
>
export interface IChatExtractedImageCollection {
29
>
readonly id: string;
30
>
readonly title: string;
31
>
readonly images: IChatExtractedImage[];
32
>
}
33
>
34
>
/**
35
>
* Extract all images from a chat response's tool invocations and inline references.
36
>
* Tool invocation images are extracted from output details and message URIs.
37
>
* Inline reference images (file URIs) are read via the provided {@link readFile} callback.
38
>
*/
40
>
response: IChatResponseViewModel,
41
>
readFile: (uri: URI) => Promise<VSBuffer>,
42
>
): Promise<IChatExtractedImageCollection> {
43
>
const allImages: IChatExtractedImage[] = [];
44
>
45
>
for (const item of response.response.value) {
46
>
if (item.kind === 'toolInvocation' || item.kind === 'toolInvocationSerialized') {
chatImageExtraction.ts ×3
47
>
const images = extractImagesFromToolInvocationOutputDetails(item, response.sessionResource);
chatImageExtraction.ts ×4
48
>
allImages.push(...images);
49
>
const messageImages = await extractImagesFromToolInvocationMessages(item, readFile);
50
>
allImages.push(...messageImages);
53
>
if (image) {
55
>
}
59
>
// Use the corresponding user request as the carousel title
60
>
const request = response.session.getItems().find((item): item is IChatRequestViewModel => isRequestVM(item) && item.id === response.requestId);
61
>
const title = request ? request.messageText.trim() || getExplicitFileOrImageAttachmentSummary(request.variables) || localize('chatImageExtraction.defaultTitle', "Images") : localize('chatImageExtraction.defaultTitle', "Images");
62
>
63
>
return {
64
>
id: response.sessionResource.toString() + '_' + response.id,
65
>
title,
66
>
images: allImages,
67
>
};
68
>
}
70
>
export function extractImagesFromToolInvocationOutputDetails(toolInvocation: IChatToolInvocation | IChatToolInvocationSerialized, sessionResource: URI): IChatExtractedImage[] {
72
>
73
>
const resultDetails = IChatToolInvocation.resultDetails(toolInvocation);
74
>
75
>
const caption = toolInvocation.pastTenseMessage ?? toolInvocation.invocationMessage;
76
>
const pushImage = (mimeType: string, data: VSBuffer, outputIndex: number) => {
78
>
const permalinkBasename = ext ? `file${ext}` : 'file.bin';
79
>
const uri = ChatResponseResource.createUri(sessionResource, toolInvocation.toolCallId, outputIndex, permalinkBasename);
80
>
images.push({
81
>
id: `${toolInvocation.toolCallId}_${outputIndex}`,
82
>
uri,
83
>
name: localize('chatImageExtraction.imageName', "Image {0}", images.length + 1),
84
>
mimeType,
85
>
data,
86
>
source: localize('chatImageExtraction.toolSource', "Tool: {0}", toolInvocation.toolId),
87
>
caption,
88
>
});
89
>
};
91
>
if (isToolResultInputOutputDetails(resultDetails)) {
93
>
const outputItem = resultDetails.output[i];
94
>
if (outputItem.type === 'embed' && outputItem.mimeType?.startsWith('image/') && !outputItem.isText) {
95
>
pushImage(outputItem.mimeType, decodeBase64(outputItem.value), i);
96
>
}
97
>
}
98
>
}
101
>
if (output.mimeType?.startsWith('image/')) {
102
>
const data = getImageDataFromOutputDetails(resultDetails, toolInvocation);
chatImageExtraction.ts ×3
103
>
if (data) {
104
>
pushImage(output.mimeType, data, 0);
105
>
}
106
>
}
109
>
return images;
110
>
}
113
>
toolInvocation: IChatToolInvocation | IChatToolInvocationSerialized,
114
>
readFile: (uri: URI) => Promise<VSBuffer>
115
>
): Promise<IChatExtractedImage[]> {
116
>
// Use pastTenseMessage if available, otherwise fall back to invocationMessage.
117
>
// When pastTenseMessage exists it visually replaces invocationMessage in the UI,
118
>
// so we only look at its URIs — we don't fall back to invocationMessage URIs.
119
>
const message = toolInvocation.pastTenseMessage ?? toolInvocation.invocationMessage;
120
>
if (!message || typeof message === 'string' || !message.uris || Object.keys(message.uris).length === 0) {
122
>
}
124
>
const images: IChatExtractedImage[] = [];
125
>
for (const uriComponents of Object.values(message.uris)) {
126
>
const uri = URI.revive(uriComponents);
127
>
const mimeType = getMediaMime(uri.path);
128
>
if (mimeType?.startsWith('image/')) {
130
>
try {
131
>
data = await readFile(uri);
132
>
} catch {
134
>
}
136
>
images.push({
137
>
id: uri.toString(),
138
>
uri,
139
>
name,
140
>
mimeType,
141
>
data,
142
>
source: localize('chatImageExtraction.toolSource', "Tool: {0}", toolInvocation.toolId),
143
>
caption: message,
144
>
});
145
>
}
147
>
return images;
148
>
}
150
>
function getImageDataFromOutputDetails(resultDetails: IToolResultOutputDetails, toolInvocation: IChatToolInvocation | IChatToolInvocationSerialized): VSBuffer | undefined {
chatImageExtraction.ts ×3
151
>
if (toolInvocation.kind === 'toolInvocationSerialized') {
152
>
const serializedDetails = resultDetails as unknown as IToolResultOutputDetailsSerialized;
153
>
if (serializedDetails.output.base64Data) {
154
>
return decodeBase64(serializedDetails.output.base64Data);
155
>
}
156
return undefined;
157
} else {
158
return resultDetails.output.value;
159
}
163
>
part: IChatContentInlineReference,
164
>
readFile: (uri: URI) => Promise<VSBuffer>,
165
>
): Promise<IChatExtractedImage | undefined> {
166
>
const ref = part.inlineReference;
167
>
const refUri = URI.isUri(ref) ? ref : isLocation(ref) ? ref.uri : ref.location.uri;
168
>
const mime = getMediaMime(refUri.path);
169
>
if (!mime?.startsWith('image/')) {
171
>
}
173
>
let data: VSBuffer;
174
>
try {
175
>
data = await readFile(refUri);
176
>
} catch {
178
>
}
180
>
return {
181
>
id: refUri.toString(),
182
>
uri: refUri,
183
>
name,
184
>
mimeType: mime,
185
>
data,
186
>
source: localize('chatImageExtraction.inlineReference', "File"),
187
>
caption: undefined,
188
>
};
189
>
}
191
>
export function coerceImageBuffer(value: unknown): Uint8Array | undefined {
194
>
}
197
>
}
199
return undefined;
200
}
202
>
const record = value as Record<string, unknown>;
203
>
const keys = Object.keys(record).sort((a, b) => Number(a) - Number(b));
204
>
if (keys.length === 0) {
205
return undefined;
206
}
208
>
const result = new Uint8Array(keys.length);
209
>
for (let index = 0; index < keys.length; index++) {
210
>
const byte = record[keys[index]];
211
>
if (keys[index] !== String(index) || typeof byte !== 'number' || !Number.isInteger(byte) || byte < 0 || byte > 255) {
213
>
}
215
>
}
216
>
return result;
217
>
}
219
>
/**
220
>
* Extract images from a chat request's variable attachments (user-attached images).
221
>
*/
222
>
export function extractImagesFromChatRequest(
224
>
): IChatExtractedImage[] {
225
>
return extractImagesFromChatVariables(request.variables);
226
>
}
228
>
export function extractImagesFromChatVariables(
230
>
): IChatExtractedImage[] {
231
>
const images: IChatExtractedImage[] = [];
232
>
for (const variable of variables) {
233
>
if (!isImageVariableEntry(variable)) {
234
continue;
235
}
237
>
if (!buffer) {
239
>
}
240
>
const mimeType = variable.mimeType ?? getMediaMime(variable.name) ?? 'image/png';
chatImageExtraction.ts ×6
241
>
const uri = variable.references?.[0]?.reference;
242
>
const imageUri = URI.isUri(uri) ? uri : URI.from({ scheme: 'data', path: `${variable.id}/${encodeURIComponent(variable.name)}` });
243
>
images.push({
244
>
id: imageUri.toString(),
245
>
uri: imageUri,
246
>
name: variable.name,
247
>
mimeType,
248
>
data: VSBuffer.wrap(buffer),
249
>
source: localize('chatImageExtraction.userAttachment', "Attachment"),
250
>
caption: undefined,
251
>
});
252
>
}
253
>
return images;
254
>
}