Add more telemtry for auto (#1714)

* Add more telemtry for auto

* Update src/extension/byok/node/openAIEndpoint.ts

Co-authored-by: Copilot <[email protected]>

---------

Co-authored-by: Copilot <[email protected]>

Author: lramos15

src/extension/byok/node/openAIEndpoint.ts

@@ -43,6 +43,18 @@ function hydrateBYOKErrorMessages(response: ChatResponse): ChatResponse {
 	return response;
 }
 
+/**
+ * Checks to see if a given endpoint is a BYOK model.
+ * @param endpoint The endpoint to check if it's a BYOK model
+ * @returns 1 if client side byok, 2 if server side byok, -1 if not a byok model
+ */
+export function isBYOKModel(endpoint: IChatEndpoint | undefined): number {
+	if (!endpoint) {
+		return -1;
+	}
+	return endpoint instanceof OpenAIEndpoint ? 1 : (endpoint.customModel ? 2 : -1);
+}
+
 export class OpenAIEndpoint extends ChatEndpoint {
 	// Reserved headers that cannot be overridden for security and functionality reasons
 	// Including forbidden request headers: https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_request_header

src/extension/conversation/vscode-node/languageModelAccess.ts

@@ -15,8 +15,8 @@ import { EmbeddingType, getWellKnownEmbeddingTypeInfo, IEmbeddingsComputer } fro
 import { IEndpointProvider } from '../../../platform/endpoint/common/endpointProvider';
 import { CustomDataPartMimeTypes } from '../../../platform/endpoint/common/endpointTypes';
 import { encodeStatefulMarker } from '../../../platform/endpoint/common/statefulMarkerContainer';
-import { AutoChatEndpoint } from '../../../platform/endpoint/vscode-node/autoChatEndpoint';
-import { IAutomodeService } from '../../../platform/endpoint/vscode-node/automodeService';
+import { AutoChatEndpoint } from '../../../platform/endpoint/node/autoChatEndpoint';
+import { IAutomodeService } from '../../../platform/endpoint/node/automodeService';
 import { IEnvService, isScenarioAutomation } from '../../../platform/env/common/envService';
 import { IVSCodeExtensionContext } from '../../../platform/extContext/common/extensionContext';
 import { ILogService } from '../../../platform/log/common/logService';

src/extension/extension/vscode-node/services.ts

@@ -21,9 +21,9 @@ import { DiffServiceImpl } from '../../../platform/diff/node/diffServiceImpl';
 import { ICAPIClientService } from '../../../platform/endpoint/common/capiClient';
 import { IDomainService } from '../../../platform/endpoint/common/domainService';
 import { IEndpointProvider } from '../../../platform/endpoint/common/endpointProvider';
+import { AutomodeService, IAutomodeService } from '../../../platform/endpoint/node/automodeService';
 import { CAPIClientImpl } from '../../../platform/endpoint/node/capiClientImpl';
 import { DomainService } from '../../../platform/endpoint/node/domainServiceImpl';
-import { AutomodeService, IAutomodeService } from '../../../platform/endpoint/vscode-node/automodeService';
 import { INativeEnvService, isScenarioAutomation } from '../../../platform/env/common/envService';
 import { NativeEnvServiceImpl } from '../../../platform/env/vscode-node/nativeEnvServiceImpl';
 import { IGitCommitMessageService } from '../../../platform/git/common/gitCommitMessageService';

src/extension/prompt/node/chatMLFetcher.ts

@@ -10,6 +10,7 @@ import { ChatFetchError, ChatFetchResponseType, ChatFetchRetriableError, ChatLoc
 import { IConversationOptions } from '../../../platform/chat/common/conversationOptions';
 import { getTextPart, toTextParts } from '../../../platform/chat/common/globalStringUtils';
 import { HARD_TOOL_LIMIT } from '../../../platform/configuration/common/configurationService';
+import { isAutoModel } from '../../../platform/endpoint/node/autoChatEndpoint';
 import { ILogService } from '../../../platform/log/common/logService';
 import { OptionalChatRequestParams } from '../../../platform/networking/common/fetch';
 import { IFetcherService } from '../../../platform/networking/common/fetcherService';
@@ -25,7 +26,7 @@ import { isCancellationError } from '../../../util/vs/base/common/errors';
 import { Emitter } from '../../../util/vs/base/common/event';
 import { generateUuid } from '../../../util/vs/base/common/uuid';
 import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation';
-import { OpenAIEndpoint } from '../../byok/node/openAIEndpoint';
+import { isBYOKModel } from '../../byok/node/openAIEndpoint';
 import { EXTENSION_ID } from '../../common/constants';
 
 export interface IMadeChatRequestEvent {
@@ -250,7 +251,8 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
 						timeToFirstTokenEmitted: (baseTelemetry && streamRecorder.firstTokenEmittedTime) ? streamRecorder.firstTokenEmittedTime - baseTelemetry.issuedTime : -1,
 						timeToCancelled: baseTelemetry ? Date.now() - baseTelemetry.issuedTime : -1,
 						isVisionRequest: this.filterImageMessages(messages) ? 1 : -1,
-						isBYOK: chatEndpoint instanceof OpenAIEndpoint ? 1 : (chatEndpoint.customModel ? 2 : -1)
+						isBYOK: isBYOKModel(chatEndpoint),
+						isAuto: isAutoModel(chatEndpoint)
 					});
 					pendingLoggedChatRequest?.resolveWithCancelation();
 					return this.processCanceledResponse(response, ourRequestId);
@@ -305,7 +307,8 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
 					timeToFirstToken: undefined,
 					timeToCancelled: timeToError,
 					isVisionRequest: this.filterImageMessages(messages) ? 1 : -1,
-					isBYOK: chatEndpoint instanceof OpenAIEndpoint ? 1 : (chatEndpoint.customModel ? 2 : -1)
+					isBYOK: isBYOKModel(chatEndpoint),
+					isAuto: isAutoModel(chatEndpoint)
 				});
 			} else {
 				this._sendResponseErrorTelemetry(processed, telemetryProperties, ourRequestId, chatEndpoint, requestBody, tokenCount, maxResponseTokens, timeToError, this.filterImageMessages(messages));
@@ -337,7 +340,8 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
 			timeToFirstTokenEmitted,
 			timeToCancelled,
 			isVisionRequest,
-			isBYOK
+			isBYOK,
+			isAuto
 		}: {
 			totalTokenMax: number;
 			promptTokenCount: number;
@@ -347,6 +351,7 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
 			timeToCancelled: number;
 			isVisionRequest: number;
 			isBYOK: number;
+			isAuto: number;
 		}
 	) {
 		/* __GDPR__
@@ -366,6 +371,7 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
 				"timeToCancelled": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to first token", "isMeasurement": true },
 				"isVisionRequest": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether the request was for a vision model", "isMeasurement": true },
 				"isBYOK": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was for a BYOK model", "isMeasurement": true },
+				"isAuto": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was for an Auto model", "isMeasurement": true },
 				"retryAfterErrorCategory": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "If the response failed and this is a retry attempt, this contains the error category." },
 				"retryAfterFilterCategory": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "If the response was filtered and this is a retry attempt, this contains the original filtered content category." }
 			}
@@ -384,7 +390,8 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
 			timeToFirstTokenEmitted,
 			timeToCancelled,
 			isVisionRequest,
-			isBYOK
+			isBYOK,
+			isAuto
 		});
 	}
 
@@ -419,6 +426,7 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
 				"timeToFirstTokenEmitted": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to first token emitted (visible text)", "isMeasurement": true },
 				"isVisionRequest": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether the request was for a vision model", "isMeasurement": true },
 				"isBYOK": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was for a BYOK model", "isMeasurement": true },
+				"isAuto": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was for an Auto model", "isMeasurement": true },
 				"retryAfterErrorCategory": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "If the response failed and this is a retry attempt, this contains the error category." },
 				"retryAfterFilterCategory": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "If the response was filtered and this is a retry attempt, this contains the original filtered content category." }
 			}
@@ -441,7 +449,8 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
 			tokenCountMax: maxResponseTokens,
 			timeToFirstToken,
 			isVisionRequest: isVisionRequest ? 1 : -1,
-			isBYOK: chatEndpointInfo instanceof OpenAIEndpoint ? 1 : (chatEndpointInfo.customModel ? 2 : -1)
+			isBYOK: isBYOKModel(chatEndpointInfo),
+			isAuto: isAutoModel(chatEndpointInfo)
 		});
 	}
 
@@ -471,6 +480,7 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
 					"source": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Source of the initial request" },
 					"initiatorType": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was initiated by a user or an agent" },
 					"model": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Model selection for the response" },
+					"modelInvoked": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Actual model invoked for the response" },
 					"apiType": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "API type for the response- chat completions or responses" },
 					"requestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Id of the current turn request" },
 					"associatedRequestId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Another request ID that this request is associated with (eg, the originating request of a summarization request)." },
@@ -491,6 +501,7 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
 					"timeToComplete": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to complete the request", "isMeasurement": true },
 					"isVisionRequest": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether the request was for a vision model", "isMeasurement": true },
 					"isBYOK": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was for a BYOK model", "isMeasurement": true },
+					"isAuto": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was for an Auto model", "isMeasurement": true },
 					"retryAfterErrorCategory": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "If the response failed and this is a retry attempt, this contains the error category." },
 					"retryAfterFilterCategory": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "If the response was filtered and this is a retry attempt, this contains the original filtered content category." }
 				}
@@ -501,6 +512,7 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
 				source: baseTelemetry?.properties.messageSource ?? 'unknown',
 				initiatorType: userInitiatedRequest ? 'user' : 'agent',
 				model: chatEndpointInfo?.model,
+				modelInvoked: chatCompletion.model,
 				apiType: chatEndpointInfo?.apiType,
 				requestId,
 				associatedRequestId: baseTelemetry?.properties.associatedRequestId,
@@ -523,7 +535,8 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher {
 				timeToFirstTokenEmitted: (baseTelemetry && streamRecorder.firstTokenEmittedTime) ? streamRecorder.firstTokenEmittedTime - baseTelemetry.issuedTime : -1,
 				timeToComplete: baseTelemetry ? Date.now() - baseTelemetry.issuedTime : -1,
 				isVisionRequest: this.filterImageMessages(messages) ? 1 : -1,
-				isBYOK: chatEndpointInfo instanceof OpenAIEndpoint ? 1 : (chatEndpointInfo?.customModel ? 2 : -1)
+				isBYOK: isBYOKModel(chatEndpointInfo),
+				isAuto: isAutoModel(chatEndpointInfo)
 			});
 			if (!this.isRepetitive(chatCompletion, baseTelemetry?.properties)) {
 				completions.push(chatCompletion);

src/extension/prompt/vscode-node/endpointProviderImpl.ts

@@ -8,12 +8,12 @@ import { IAuthenticationService } from '../../../platform/authentication/common/
 import { ConfigKey, IConfigurationService } from '../../../platform/configuration/common/configurationService';
 import { ICAPIClientService } from '../../../platform/endpoint/common/capiClient';
 import { ChatEndpointFamily, EmbeddingsEndpointFamily, IChatModelInformation, ICompletionModelInformation, IEmbeddingModelInformation, IEndpointProvider } from '../../../platform/endpoint/common/endpointProvider';
+import { AutoChatEndpoint } from '../../../platform/endpoint/node/autoChatEndpoint';
+import { IAutomodeService } from '../../../platform/endpoint/node/automodeService';
 import { CopilotChatEndpoint } from '../../../platform/endpoint/node/copilotChatEndpoint';
 import { EmbeddingEndpoint } from '../../../platform/endpoint/node/embeddingsEndpoint';
 import { IModelMetadataFetcher, ModelMetadataFetcher } from '../../../platform/endpoint/node/modelMetadataFetcher';
 import { applyExperimentModifications, ExperimentConfig, getCustomDefaultModelExperimentConfig, ProxyExperimentEndpoint } from '../../../platform/endpoint/node/proxyExperimentEndpoint';
-import { AutoChatEndpoint } from '../../../platform/endpoint/vscode-node/autoChatEndpoint';
-import { IAutomodeService } from '../../../platform/endpoint/vscode-node/automodeService';
 import { ExtensionContributedChatEndpoint } from '../../../platform/endpoint/vscode-node/extChatEndpoint';
 import { IEnvService } from '../../../platform/env/common/envService';
 import { ILogService } from '../../../platform/log/common/logService';

src/extension/test/vscode-node/services.ts

@@ -23,10 +23,10 @@ import { RemoteEmbeddingsComputer } from '../../../platform/embeddings/common/re
 import { ICAPIClientService } from '../../../platform/endpoint/common/capiClient';
 import { IDomainService } from '../../../platform/endpoint/common/domainService';
 import { IEndpointProvider } from '../../../platform/endpoint/common/endpointProvider';
+import { AutomodeService, IAutomodeService } from '../../../platform/endpoint/node/automodeService';
 import { CAPIClientImpl } from '../../../platform/endpoint/node/capiClientImpl';
 import { DomainService } from '../../../platform/endpoint/node/domainServiceImpl';
 import { TestEndpointProvider } from '../../../platform/endpoint/test/node/testEndpointProvider';
-import { AutomodeService, IAutomodeService } from '../../../platform/endpoint/vscode-node/automodeService';
 import { IEnvService } from '../../../platform/env/common/envService';
 import { EnvServiceImpl } from '../../../platform/env/vscode/envServiceImpl';
 import { IVSCodeExtensionContext } from '../../../platform/extContext/common/extensionContext';

src/platform/endpoint/vscode-node/autoChatEndpoint.ts renamed to src/platform/endpoint/node/autoChatEndpoint.ts

@@ -16,7 +16,7 @@ import { ITokenizerProvider } from '../../tokenizer/node/tokenizer';
 import { ICAPIClientService } from '../common/capiClient';
 import { IDomainService } from '../common/domainService';
 import { IChatModelInformation } from '../common/endpointProvider';
-import { ChatEndpoint } from '../node/chatEndpoint';
+import { ChatEndpoint } from './chatEndpoint';
 
 /**
  * This endpoint represents the "Auto" model in the model picker.
@@ -111,4 +111,11 @@ function calculateAutoModelInfo(endpoint: IChatEndpoint, sessionToken: string, d
 		}
 	};
 	return newModelInfo;
+}
+
+export function isAutoModel(endpoint: IChatEndpoint | undefined): number {
+	if (!endpoint) {
+		return -1;
+	}
+	return endpoint.model === AutoChatEndpoint.pseudoModelId || (endpoint instanceof AutoChatEndpoint) ? 1 : -1;
 }