change api shape

Author: justschen

src/vs/workbench/api/common/extHostTypes.ts

@@ -17,12 +17,12 @@ import { nextCharLength } from '../../../base/common/strings.js';
 import { isNumber, isObject, isString, isStringArray } from '../../../base/common/types.js';
 import { URI } from '../../../base/common/uri.js';
 import { generateUuid } from '../../../base/common/uuid.js';
+import { TextEditorSelectionSource } from '../../../platform/editor/common/editor.js';
 import { ExtensionIdentifier, IExtensionDescription } from '../../../platform/extensions/common/extensions.js';
 import { FileSystemProviderErrorCode, markAsFileSystemProviderError } from '../../../platform/files/common/files.js';
 import { RemoteAuthorityResolverErrorCode } from '../../../platform/remote/common/remoteAuthorityResolver.js';
 import { CellEditType, ICellMetadataEdit, IDocumentMetadataEdit, isTextStreamMime } from '../../contrib/notebook/common/notebookCommon.js';
 import { IRelativePatternDto } from './extHost.protocol.js';
-import { TextEditorSelectionSource } from '../../../platform/editor/common/editor.js';
 
 /**
  * @deprecated
@@ -4918,18 +4918,10 @@ export class LanguageModelImagePart implements vscode.LanguageModelImagePart {
 }
 
 export interface ChatImagePart {
-	image_url: {
-		/**
-			 * Either a URL of the image or the base64 encoded image data.
-			 */
-		url: string;
-
-		/**
-		 * Specifies the detail level of the image. Learn more in the
-		 * [Vision guide](https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding).
-		 */
-		detail?: ImageDetailLevel;
-	};
+	mimeType: string;
+	data: Uint8Array;
+	detail?: ImageDetailLevel;
+
 }
 
 export class LanguageModelPromptTsxPart {

src/vs/workbench/contrib/chat/common/languageModels.ts

@@ -35,7 +35,21 @@ export interface IChatMessageImagePart {
 }
 
 export interface IChatImageURLPart {
-	image_url: { url: string; detail?: ImageDetailLevel };
+	/**
+	 * The image's MIME type (e.g., "image/png", "image/jpeg").
+	 */
+	mimeType: string;
+
+	/**
+	 * The raw binary data of the image, encoded as a Uint8Array. Note: do not use base64 encoding. Maximum image size is 5MB.
+	 */
+	data: Uint8Array;
+
+	/**
+	 * Specifies the detail level of the image. Learn more in the
+	 * [Vision guide](https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding).
+	 */
+	detail?: ImageDetailLevel;
 }
 
 /**

src/vscode-dts/vscode.proposed.languageModelImagePart.d.ts

@@ -74,19 +74,22 @@ declare module 'vscode' {
 	}
 
 	export interface ChatImagePart {
-		image_url: {
-			/**
-					 * Either a URL of the image or the base64 encoded image data.
-					 */
-			url: string;
-
-			/**
-			 * Specifies the detail level of the image. Learn more in the
-			 * [Vision guide](https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding).
-			 */
-			detail?: ImageDetailLevel;
-
-		};
+		/**
+		 * The image's MIME type (e.g., "image/png", "image/jpeg").
+		 */
+		mimeType: string;
+
+		/**
+		 * The raw binary data of the image, encoded as a Uint8Array. Note: do not use base64 encoding. Maximum image size is 5MB.
+		 */
+		data: Uint8Array;
+
+		/**
+		 * Specifies the detail level of the image. Learn more in the
+		 * [Vision guide](https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding).
+		 */
+		detail?: ImageDetailLevel;
+
 	}
 
 	/**