From b0342cccb0a38f22547b1169b6a831aeadd1543f Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Fri, 21 Feb 2025 13:04:13 +0100 Subject: [PATCH 01/23] initial draft --- packages/langchain/package.json | 1 + packages/langchain/src/orchestration/chat.ts | 74 +++++ packages/langchain/src/orchestration/index.ts | 0 packages/langchain/src/orchestration/types.ts | 36 +++ packages/langchain/src/orchestration/util.ts | 288 ++++++++++++++++++ pnpm-lock.yaml | 3 + 6 files changed, 402 insertions(+) create mode 100644 packages/langchain/src/orchestration/chat.ts create mode 100644 packages/langchain/src/orchestration/index.ts create mode 100644 packages/langchain/src/orchestration/types.ts create mode 100644 packages/langchain/src/orchestration/util.ts diff --git a/packages/langchain/package.json b/packages/langchain/package.json index 6c70d2513..fa763d451 100644 --- a/packages/langchain/package.json +++ b/packages/langchain/package.json @@ -29,6 +29,7 @@ "@sap-ai-sdk/ai-api": "workspace:^", "@sap-ai-sdk/core": "workspace:^", "@sap-ai-sdk/foundation-models": "workspace:^", + "@sap-ai-sdk/orchestration": "workspace:^", "@sap-cloud-sdk/connectivity": "^3.26.1", "uuid": "^11.1.0", "@langchain/core": "0.3.40", diff --git a/packages/langchain/src/orchestration/chat.ts b/packages/langchain/src/orchestration/chat.ts new file mode 100644 index 000000000..c7df1999e --- /dev/null +++ b/packages/langchain/src/orchestration/chat.ts @@ -0,0 +1,74 @@ +import { BaseChatModel, BaseChatModelParams } from '@langchain/core/language_models/chat_models'; +import { OrchestrationClient as OrchestrationClientBase, OrchestrationModuleConfig } from '@sap-ai-sdk/orchestration'; +import { mapOutputToChatResult } from './util.js'; +import type { ResourceGroupConfig } from '@sap-ai-sdk/ai-api'; +import type { BaseMessage } from '@langchain/core/messages'; +import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager'; +import type { ChatResult } from '@langchain/core/outputs'; +import type { + AzureOpenAiChatCallOptions, +} from './types.js'; +import type { HttpDestinationOrFetchOptions } from '@sap-cloud-sdk/connectivity'; + +/** + * LangChain chat client for Azure OpenAI consumption on SAP BTP. + */ +export class OrchestrationClient extends BaseChatModel { + // Omit streaming until supported + orchestrationConfig: Omit; + langchainOptions?: BaseChatModelParams; + deploymentConfig?: ResourceGroupConfig; + destination?: HttpDestinationOrFetchOptions; + + // initialize with complete config and shit + // allow to prompt + pass model params at invocatio + // initialize a new client on every call, so that .bind() and .bindTools will properly work + // shit is still cached because it's not instance specific + constructor( + // Omit streaming until supported + orchestrationConfig: Omit, + langchainOptions: BaseChatModelParams = {}, + deploymentConfig?: ResourceGroupConfig, + destination?: HttpDestinationOrFetchOptions, + ) { + super(langchainOptions); + this.orchestrationConfig = orchestrationConfig + this.destination = destination + this.deploymentConfig = deploymentConfig; + } + + _llmType(): string { + return 'orchestration'; + } + + override async _generate( + messages: BaseMessage[], + options: typeof this.ParsedCallOptions, + runManager?: CallbackManagerForLLMRun + ): Promise { + const res = await this.caller.callWithOptions( + { + signal: options.signal + }, + () => { + const orchestrationClient = new OrchestrationClientBase(this.orchestrationConfig, this.deploymentConfig, this.destination); + return orchestrationClient.chatCompletion( + mapLangchainToOrchestrationClient(this, messages, options), + } + const orchestrationClient = new OrchestrationClientBase(this.fields, this.deploymentConfig, this.destination); + this.orchestrationClient.chatCompletion( + mapLangchainToOrchestrationClient(this, messages, options), + options.requestConfig + ) + ); + + const content = res.getContent(); + + // we currently do not support streaming + await runManager?.handleLLMNewToken( + typeof content === 'string' ? content : '' + ); + + return mapOutputToChatResult(res.data); + } +} diff --git a/packages/langchain/src/orchestration/index.ts b/packages/langchain/src/orchestration/index.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/langchain/src/orchestration/types.ts b/packages/langchain/src/orchestration/types.ts new file mode 100644 index 000000000..528f7f343 --- /dev/null +++ b/packages/langchain/src/orchestration/types.ts @@ -0,0 +1,36 @@ +import type { BaseLLMParams } from '@langchain/core/language_models/llms'; +import type { + BaseChatModelCallOptions +} from '@langchain/core/language_models/chat_models'; +import type { + AzureOpenAiCreateChatCompletionRequest, + AzureOpenAiEmbeddingModel +} from '@sap-ai-sdk/foundation-models'; +import type { CustomRequestConfig } from '@sap-ai-sdk/core'; +import type { ModelConfig, ResourceGroupConfig } from '@sap-ai-sdk/ai-api'; + +/** + * Call options for the {@link AzureOpenAiChatClient}. + */ +export type AzureOpenAiChatCallOptions = BaseChatModelCallOptions & + Pick< + AzureOpenAiCreateChatCompletionRequest, + | 'data_sources' + | 'n' + | 'seed' + | 'logprobs' + | 'top_logprobs' + | 'response_format' + | 'tool_choice' + | 'functions' + | 'function_call' + | 'tools' + > & { + requestConfig?: CustomRequestConfig; + }; + +/** + * Input type for {@link AzureOpenAiEmbeddingClient} initialization. + */ +export type AzureOpenAiEmbeddingModelParams = + ModelConfig & ResourceGroupConfig & BaseLLMParams; diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts new file mode 100644 index 000000000..67146d385 --- /dev/null +++ b/packages/langchain/src/orchestration/util.ts @@ -0,0 +1,288 @@ +import { AIMessage } from '@langchain/core/messages'; +import { zodToJsonSchema } from 'zod-to-json-schema'; +import { v4 as uuidv4 } from 'uuid'; +import type { ToolCall } from '@langchain/core/messages/tool'; +import type { + AzureOpenAiChatCompletionRequestUserMessage, + AzureOpenAiChatCompletionRequestAssistantMessage, + AzureOpenAiChatCompletionTool, + AzureOpenAiChatCompletionRequestMessage, + AzureOpenAiCreateChatCompletionResponse, + AzureOpenAiCreateChatCompletionRequest, + AzureOpenAiFunctionParameters, + AzureOpenAiChatCompletionMessageToolCalls, + AzureOpenAiChatCompletionRequestToolMessage, + AzureOpenAiChatCompletionRequestFunctionMessage, + AzureOpenAiChatCompletionRequestSystemMessage +} from '@sap-ai-sdk/foundation-models'; +import type { + BaseMessage, + FunctionMessage, + HumanMessage, + SystemMessage, + ToolMessage +} from '@langchain/core/messages'; +import type { ChatResult } from '@langchain/core/outputs'; +import type { StructuredTool } from '@langchain/core/tools'; +import type { AzureOpenAiChatClient } from './chat.js'; +import type { AzureOpenAiChatCallOptions } from './types.js'; + +/** + * Maps a LangChain {@link StructuredTool} to {@link AzureOpenAiChatCompletionFunctions}. + * @param tool - Base class for tools that accept input of any shape defined by a Zod schema. + * @returns The OpenAI chat completion function. + */ +function mapToolToOpenAiFunction(tool: StructuredTool): { + description?: string; + name: string; + parameters: AzureOpenAiFunctionParameters; +} & Record { + return { + name: tool.name, + description: tool.description, + parameters: zodToJsonSchema(tool.schema) + }; +} + +/** + * Maps a LangChain {@link StructuredTool} to {@link AzureOpenAiChatCompletionTool}. + * @param tool - Base class for tools that accept input of any shape defined by a Zod schema. + * @returns The OpenAI chat completion tool. + */ +function mapToolToOpenAiTool( + tool: StructuredTool +): AzureOpenAiChatCompletionTool { + return { + type: 'function', + function: mapToolToOpenAiFunction(tool) + }; +} + +/** + * Maps {@link AzureOpenAiChatCompletionMessageToolCalls} to LangChain's {@link ToolCall}. + * @param toolCalls - The {@link AzureOpenAiChatCompletionMessageToolCalls} response. + * @returns The LangChain {@link ToolCall}. + */ +function mapAzureOpenAiToLangchainToolCall( + toolCalls?: AzureOpenAiChatCompletionMessageToolCalls +): ToolCall[] | undefined { + if (toolCalls) { + return toolCalls.map(toolCall => ({ + id: toolCall.id, + name: toolCall.function.name, + args: JSON.parse(toolCall.function.arguments), + type: 'tool_call' + })); + } +} + +/** + * Maps {@link AzureOpenAiCreateChatCompletionResponse} to LangChain's {@link ChatResult}. + * @param completionResponse - The {@link AzureOpenAiCreateChatCompletionResponse} response. + * @returns The LangChain {@link ChatResult} + * @internal + */ +export function mapOutputToChatResult( + completionResponse: AzureOpenAiCreateChatCompletionResponse +): ChatResult { + return { + generations: completionResponse.choices.map(choice => ({ + text: choice.message.content ?? '', + message: new AIMessage({ + content: choice.message.content ?? '', + tool_calls: mapAzureOpenAiToLangchainToolCall( + choice.message.tool_calls + ), + additional_kwargs: { + finish_reason: choice.finish_reason, + index: choice.index, + function_call: choice.message.function_call, + tool_calls: choice.message.tool_calls + } + }), + generationInfo: { + finish_reason: choice.finish_reason, + index: choice.index, + function_call: choice.message.function_call, + tool_calls: choice.message.tool_calls + } + })), + llmOutput: { + created: completionResponse.created, + id: completionResponse.id, + model: completionResponse.model, + object: completionResponse.object, + tokenUsage: { + completionTokens: completionResponse.usage?.completion_tokens ?? 0, + promptTokens: completionResponse.usage?.prompt_tokens ?? 0, + totalTokens: completionResponse.usage?.total_tokens ?? 0 + } + } + }; +} + +/** + * Maps LangChain's {@link ToolCall} to Azure OpenAI's {@link AzureOpenAiChatCompletionMessageToolCalls}. + * @param toolCalls - The {@link ToolCall} to map. + * @returns The Azure OpenAI {@link AzureOpenAiChatCompletionMessageToolCalls}. + */ +function mapLangchainToolCallToAzureOpenAiToolCall( + toolCalls?: ToolCall[] +): AzureOpenAiChatCompletionMessageToolCalls | undefined { + if (toolCalls) { + return toolCalls.map(toolCall => ({ + id: toolCall.id || uuidv4(), + type: 'function', + function: { + name: toolCall.name, + arguments: JSON.stringify(toolCall.args) + } + })); + } +} + +/** + * Maps LangChain's {@link AIMessage} to Azure OpenAI's {@link AzureOpenAiChatCompletionRequestAssistantMessage}. + * @param message - The {@link AIMessage} to map. + * @returns The Azure OpenAI {@link AzureOpenAiChatCompletionRequestAssistantMessage}. + */ +function mapAiMessageToAzureOpenAiAssistantMessage( + message: AIMessage +): AzureOpenAiChatCompletionRequestAssistantMessage { + const tool_calls = + mapLangchainToolCallToAzureOpenAiToolCall(message.tool_calls) ?? + message.additional_kwargs.tool_calls; + return { + name: message.name, + ...(tool_calls?.length ? { tool_calls } : {}), + function_call: message.additional_kwargs.function_call, + content: + message.content as AzureOpenAiChatCompletionRequestAssistantMessage['content'], + role: 'assistant' + }; +} + +function mapHumanMessageToAzureOpenAiUserMessage( + message: HumanMessage +): AzureOpenAiChatCompletionRequestUserMessage { + return { + role: 'user', + content: + message.content as AzureOpenAiChatCompletionRequestUserMessage['content'], + name: message.name + }; +} + +function mapToolMessageToAzureOpenAiToolMessage( + message: ToolMessage +): AzureOpenAiChatCompletionRequestToolMessage { + return { + role: 'tool', + content: + message.content as AzureOpenAiChatCompletionRequestToolMessage['content'], + tool_call_id: message.tool_call_id + }; +} + +function mapFunctionMessageToAzureOpenAiFunctionMessage( + message: FunctionMessage +): AzureOpenAiChatCompletionRequestFunctionMessage { + return { + role: 'function', + content: + message.content as AzureOpenAiChatCompletionRequestFunctionMessage['content'], + name: message.name || 'default' + }; +} + +function mapSystemMessageToAzureOpenAiSystemMessage( + message: SystemMessage +): AzureOpenAiChatCompletionRequestSystemMessage { + return { + role: 'system', + content: + message.content as AzureOpenAiChatCompletionRequestSystemMessage['content'], + name: message.name + }; +} + +/** + * Maps {@link BaseMessage} to {@link AzureOpenAiChatMessage}. + * @param message - The message to map. + * @returns The {@link AzureOpenAiChatMessage}. + */ +// TODO: Add mapping of refusal property, once LangChain base class supports it natively. +function mapBaseMessageToAzureOpenAiChatMessage( + message: BaseMessage +): AzureOpenAiChatCompletionRequestMessage { + switch (message.getType()) { + case 'ai': + return mapAiMessageToAzureOpenAiAssistantMessage(message); + case 'human': + return mapHumanMessageToAzureOpenAiUserMessage(message); + case 'system': + return mapSystemMessageToAzureOpenAiSystemMessage(message); + case 'function': + return mapFunctionMessageToAzureOpenAiFunctionMessage(message); + case 'tool': + return mapToolMessageToAzureOpenAiToolMessage(message as ToolMessage); + default: + throw new Error(`Unsupported message type: ${message.getType()}`); + } +} + +function isStructuredToolArray(tools?: unknown[]): tools is StructuredTool[] { + return !!tools?.every(tool => + Array.isArray((tool as StructuredTool).lc_namespace) + ); +} + +/** + * Maps LangChain's input interface to the AI SDK client's input interface + * @param client The LangChain Azure OpenAI client + * @param options The {@link AzureOpenAiChatCallOptions} + * @param messages The messages to be send + * @returns An AI SDK compatibile request + * @internal + */ +export function mapLangchainToAiClient( + client: AzureOpenAiChatClient, + messages: BaseMessage[], + options?: AzureOpenAiChatCallOptions & { promptIndex?: number } +): AzureOpenAiCreateChatCompletionRequest { + return removeUndefinedProperties({ + messages: messages.map(mapBaseMessageToAzureOpenAiChatMessage), + max_tokens: client.max_tokens === -1 ? undefined : client.max_tokens, + presence_penalty: client.presence_penalty, + frequency_penalty: client.frequency_penalty, + temperature: client.temperature, + top_p: client.top_p, + logit_bias: client.logit_bias, + user: client.user, + data_sources: options?.data_sources, + n: options?.n, + response_format: options?.response_format, + seed: options?.seed, + logprobs: options?.logprobs, + top_logprobs: options?.top_logprobs, + function_call: options?.function_call, + stop: options?.stop ?? client.stop, + functions: isStructuredToolArray(options?.functions) + ? options?.functions.map(mapToolToOpenAiFunction) + : options?.functions, + tools: isStructuredToolArray(options?.tools) + ? options?.tools.map(mapToolToOpenAiTool) + : options?.tools, + tool_choice: options?.tool_choice + }); +} + +function removeUndefinedProperties(obj: T): T { + const result = { ...obj }; + for (const key in result) { + if (result[key as keyof T] === undefined) { + delete result[key as keyof T]; + } + } + return result; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fc9f2fe9b..d0b9c4ac8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -157,6 +157,9 @@ importers: '@sap-ai-sdk/foundation-models': specifier: workspace:^ version: link:../foundation-models + '@sap-ai-sdk/orchestration': + specifier: workspace:^ + version: link:../orchestration '@sap-cloud-sdk/connectivity': specifier: ^3.26.1 version: 3.26.1 From 388222e983a6769270539ed12cb452bbead09063 Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Fri, 21 Feb 2025 18:03:27 +0100 Subject: [PATCH 02/23] some more cutting --- packages/langchain/src/orchestration/chat.ts | 35 +-- packages/langchain/src/orchestration/types.ts | 9 + packages/langchain/src/orchestration/util.ts | 223 +----------------- 3 files changed, 40 insertions(+), 227 deletions(-) diff --git a/packages/langchain/src/orchestration/chat.ts b/packages/langchain/src/orchestration/chat.ts index c7df1999e..e3d345242 100644 --- a/packages/langchain/src/orchestration/chat.ts +++ b/packages/langchain/src/orchestration/chat.ts @@ -1,19 +1,21 @@ -import { BaseChatModel, BaseChatModelParams } from '@langchain/core/language_models/chat_models'; -import { OrchestrationClient as OrchestrationClientBase, OrchestrationModuleConfig } from '@sap-ai-sdk/orchestration'; -import { mapOutputToChatResult } from './util.js'; +import { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import { OrchestrationClient as OrchestrationClientBase } from '@sap-ai-sdk/orchestration'; +import { mapLangchainMessagesToOrchestrationMessages, mapOutputToChatResult } from './util.js'; +import type { OrchestrationModuleConfig } from '@sap-ai-sdk/orchestration'; +import type { BaseChatModelParams } from '@langchain/core/language_models/chat_models'; import type { ResourceGroupConfig } from '@sap-ai-sdk/ai-api'; import type { BaseMessage } from '@langchain/core/messages'; import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager'; import type { ChatResult } from '@langchain/core/outputs'; import type { - AzureOpenAiChatCallOptions, + OrchestrationCallOptions, } from './types.js'; import type { HttpDestinationOrFetchOptions } from '@sap-cloud-sdk/connectivity'; /** * LangChain chat client for Azure OpenAI consumption on SAP BTP. */ -export class OrchestrationClient extends BaseChatModel { +export class OrchestrationClient extends BaseChatModel { // Omit streaming until supported orchestrationConfig: Omit; langchainOptions?: BaseChatModelParams; @@ -32,8 +34,8 @@ export class OrchestrationClient extends BaseChatModel { @@ -51,15 +59,14 @@ export class OrchestrationClient extends BaseChatModel { + // Initializing a new client every time, as caching is unaffected + // and we support the .bind() flow of langchain this way const orchestrationClient = new OrchestrationClientBase(this.orchestrationConfig, this.deploymentConfig, this.destination); - return orchestrationClient.chatCompletion( - mapLangchainToOrchestrationClient(this, messages, options), + return orchestrationClient.chatCompletion({ + messagesHistory: mapLangchainMessagesToOrchestrationMessages(messages), + inputParams: options.inputParams + }, options.customRequestConfig); } - const orchestrationClient = new OrchestrationClientBase(this.fields, this.deploymentConfig, this.destination); - this.orchestrationClient.chatCompletion( - mapLangchainToOrchestrationClient(this, messages, options), - options.requestConfig - ) ); const content = res.getContent(); diff --git a/packages/langchain/src/orchestration/types.ts b/packages/langchain/src/orchestration/types.ts index 528f7f343..a938aaf83 100644 --- a/packages/langchain/src/orchestration/types.ts +++ b/packages/langchain/src/orchestration/types.ts @@ -1,3 +1,4 @@ +import type { Prompt } from '@sap-ai-sdk/orchestration'; import type { BaseLLMParams } from '@langchain/core/language_models/llms'; import type { BaseChatModelCallOptions @@ -29,6 +30,14 @@ export type AzureOpenAiChatCallOptions = BaseChatModelCallOptions & requestConfig?: CustomRequestConfig; }; +/** + * Options for orchestration calls that combines the base chat model call options with prompt-related settings. + * @typedef {Object} OrchestrationCallOptions + * @augments {BaseChatModelCallOptions} + * @augments {Prompt} + */ +export type OrchestrationCallOptions = BaseChatModelCallOptions & { inputParams: Prompt['inputParams']; customRequestConfig: CustomRequestConfig }; + /** * Input type for {@link AzureOpenAiEmbeddingClient} initialization. */ diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 67146d385..e8c823a4b 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -1,61 +1,24 @@ import { AIMessage } from '@langchain/core/messages'; -import { zodToJsonSchema } from 'zod-to-json-schema'; -import { v4 as uuidv4 } from 'uuid'; +import type { ChatMessages } from '@sap-ai-sdk/orchestration'; import type { ToolCall } from '@langchain/core/messages/tool'; import type { - AzureOpenAiChatCompletionRequestUserMessage, - AzureOpenAiChatCompletionRequestAssistantMessage, - AzureOpenAiChatCompletionTool, - AzureOpenAiChatCompletionRequestMessage, AzureOpenAiCreateChatCompletionResponse, - AzureOpenAiCreateChatCompletionRequest, - AzureOpenAiFunctionParameters, - AzureOpenAiChatCompletionMessageToolCalls, - AzureOpenAiChatCompletionRequestToolMessage, - AzureOpenAiChatCompletionRequestFunctionMessage, - AzureOpenAiChatCompletionRequestSystemMessage + AzureOpenAiChatCompletionMessageToolCalls } from '@sap-ai-sdk/foundation-models'; import type { - BaseMessage, - FunctionMessage, - HumanMessage, - SystemMessage, - ToolMessage + BaseMessage } from '@langchain/core/messages'; import type { ChatResult } from '@langchain/core/outputs'; -import type { StructuredTool } from '@langchain/core/tools'; -import type { AzureOpenAiChatClient } from './chat.js'; -import type { AzureOpenAiChatCallOptions } from './types.js'; /** - * Maps a LangChain {@link StructuredTool} to {@link AzureOpenAiChatCompletionFunctions}. - * @param tool - Base class for tools that accept input of any shape defined by a Zod schema. - * @returns The OpenAI chat completion function. + * Maps LangChain messages to Orchestration messages. + * @param messages - The messages to map. + * @returns The mapped messages. */ -function mapToolToOpenAiFunction(tool: StructuredTool): { - description?: string; - name: string; - parameters: AzureOpenAiFunctionParameters; -} & Record { - return { - name: tool.name, - description: tool.description, - parameters: zodToJsonSchema(tool.schema) - }; -} - -/** - * Maps a LangChain {@link StructuredTool} to {@link AzureOpenAiChatCompletionTool}. - * @param tool - Base class for tools that accept input of any shape defined by a Zod schema. - * @returns The OpenAI chat completion tool. - */ -function mapToolToOpenAiTool( - tool: StructuredTool -): AzureOpenAiChatCompletionTool { - return { - type: 'function', - function: mapToolToOpenAiFunction(tool) - }; +export function mapLangchainMessagesToOrchestrationMessages( + messages: BaseMessage[] +): ChatMessages { + return []; } /** @@ -120,169 +83,3 @@ export function mapOutputToChatResult( } }; } - -/** - * Maps LangChain's {@link ToolCall} to Azure OpenAI's {@link AzureOpenAiChatCompletionMessageToolCalls}. - * @param toolCalls - The {@link ToolCall} to map. - * @returns The Azure OpenAI {@link AzureOpenAiChatCompletionMessageToolCalls}. - */ -function mapLangchainToolCallToAzureOpenAiToolCall( - toolCalls?: ToolCall[] -): AzureOpenAiChatCompletionMessageToolCalls | undefined { - if (toolCalls) { - return toolCalls.map(toolCall => ({ - id: toolCall.id || uuidv4(), - type: 'function', - function: { - name: toolCall.name, - arguments: JSON.stringify(toolCall.args) - } - })); - } -} - -/** - * Maps LangChain's {@link AIMessage} to Azure OpenAI's {@link AzureOpenAiChatCompletionRequestAssistantMessage}. - * @param message - The {@link AIMessage} to map. - * @returns The Azure OpenAI {@link AzureOpenAiChatCompletionRequestAssistantMessage}. - */ -function mapAiMessageToAzureOpenAiAssistantMessage( - message: AIMessage -): AzureOpenAiChatCompletionRequestAssistantMessage { - const tool_calls = - mapLangchainToolCallToAzureOpenAiToolCall(message.tool_calls) ?? - message.additional_kwargs.tool_calls; - return { - name: message.name, - ...(tool_calls?.length ? { tool_calls } : {}), - function_call: message.additional_kwargs.function_call, - content: - message.content as AzureOpenAiChatCompletionRequestAssistantMessage['content'], - role: 'assistant' - }; -} - -function mapHumanMessageToAzureOpenAiUserMessage( - message: HumanMessage -): AzureOpenAiChatCompletionRequestUserMessage { - return { - role: 'user', - content: - message.content as AzureOpenAiChatCompletionRequestUserMessage['content'], - name: message.name - }; -} - -function mapToolMessageToAzureOpenAiToolMessage( - message: ToolMessage -): AzureOpenAiChatCompletionRequestToolMessage { - return { - role: 'tool', - content: - message.content as AzureOpenAiChatCompletionRequestToolMessage['content'], - tool_call_id: message.tool_call_id - }; -} - -function mapFunctionMessageToAzureOpenAiFunctionMessage( - message: FunctionMessage -): AzureOpenAiChatCompletionRequestFunctionMessage { - return { - role: 'function', - content: - message.content as AzureOpenAiChatCompletionRequestFunctionMessage['content'], - name: message.name || 'default' - }; -} - -function mapSystemMessageToAzureOpenAiSystemMessage( - message: SystemMessage -): AzureOpenAiChatCompletionRequestSystemMessage { - return { - role: 'system', - content: - message.content as AzureOpenAiChatCompletionRequestSystemMessage['content'], - name: message.name - }; -} - -/** - * Maps {@link BaseMessage} to {@link AzureOpenAiChatMessage}. - * @param message - The message to map. - * @returns The {@link AzureOpenAiChatMessage}. - */ -// TODO: Add mapping of refusal property, once LangChain base class supports it natively. -function mapBaseMessageToAzureOpenAiChatMessage( - message: BaseMessage -): AzureOpenAiChatCompletionRequestMessage { - switch (message.getType()) { - case 'ai': - return mapAiMessageToAzureOpenAiAssistantMessage(message); - case 'human': - return mapHumanMessageToAzureOpenAiUserMessage(message); - case 'system': - return mapSystemMessageToAzureOpenAiSystemMessage(message); - case 'function': - return mapFunctionMessageToAzureOpenAiFunctionMessage(message); - case 'tool': - return mapToolMessageToAzureOpenAiToolMessage(message as ToolMessage); - default: - throw new Error(`Unsupported message type: ${message.getType()}`); - } -} - -function isStructuredToolArray(tools?: unknown[]): tools is StructuredTool[] { - return !!tools?.every(tool => - Array.isArray((tool as StructuredTool).lc_namespace) - ); -} - -/** - * Maps LangChain's input interface to the AI SDK client's input interface - * @param client The LangChain Azure OpenAI client - * @param options The {@link AzureOpenAiChatCallOptions} - * @param messages The messages to be send - * @returns An AI SDK compatibile request - * @internal - */ -export function mapLangchainToAiClient( - client: AzureOpenAiChatClient, - messages: BaseMessage[], - options?: AzureOpenAiChatCallOptions & { promptIndex?: number } -): AzureOpenAiCreateChatCompletionRequest { - return removeUndefinedProperties({ - messages: messages.map(mapBaseMessageToAzureOpenAiChatMessage), - max_tokens: client.max_tokens === -1 ? undefined : client.max_tokens, - presence_penalty: client.presence_penalty, - frequency_penalty: client.frequency_penalty, - temperature: client.temperature, - top_p: client.top_p, - logit_bias: client.logit_bias, - user: client.user, - data_sources: options?.data_sources, - n: options?.n, - response_format: options?.response_format, - seed: options?.seed, - logprobs: options?.logprobs, - top_logprobs: options?.top_logprobs, - function_call: options?.function_call, - stop: options?.stop ?? client.stop, - functions: isStructuredToolArray(options?.functions) - ? options?.functions.map(mapToolToOpenAiFunction) - : options?.functions, - tools: isStructuredToolArray(options?.tools) - ? options?.tools.map(mapToolToOpenAiTool) - : options?.tools, - tool_choice: options?.tool_choice - }); -} - -function removeUndefinedProperties(obj: T): T { - const result = { ...obj }; - for (const key in result) { - if (result[key as keyof T] === undefined) { - delete result[key as keyof T]; - } - } - return result; -} From a5fff2b2ec1e40f852b62c7136265fa7d7f06e16 Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Fri, 21 Feb 2025 19:09:47 +0100 Subject: [PATCH 03/23] add some comments and incomplete mappings --- packages/langchain/src/orchestration/chat.ts | 6 ++++-- packages/langchain/src/orchestration/types.ts | 18 ++++++------------ packages/langchain/src/orchestration/util.ts | 9 ++++----- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/packages/langchain/src/orchestration/chat.ts b/packages/langchain/src/orchestration/chat.ts index e3d345242..d330d8ffe 100644 --- a/packages/langchain/src/orchestration/chat.ts +++ b/packages/langchain/src/orchestration/chat.ts @@ -6,9 +6,9 @@ import type { BaseChatModelParams } from '@langchain/core/language_models/chat_m import type { ResourceGroupConfig } from '@sap-ai-sdk/ai-api'; import type { BaseMessage } from '@langchain/core/messages'; import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager'; -import type { ChatResult } from '@langchain/core/outputs'; import type { OrchestrationCallOptions, + OrchestrationResponse } from './types.js'; import type { HttpDestinationOrFetchOptions } from '@sap-cloud-sdk/connectivity'; @@ -53,7 +53,7 @@ export class OrchestrationClient extends BaseChatModel // this would AFAIK align more with other langchain clients options: typeof this.ParsedCallOptions, runManager?: CallbackManagerForLLMRun - ): Promise { + ): Promise { const res = await this.caller.callWithOptions( { signal: options.signal @@ -63,6 +63,8 @@ export class OrchestrationClient extends BaseChatModel // and we support the .bind() flow of langchain this way const orchestrationClient = new OrchestrationClientBase(this.orchestrationConfig, this.deploymentConfig, this.destination); return orchestrationClient.chatCompletion({ + // how to handle tools here? doesn't really exist as input in orchestration as message history + // make template a call option, to merge it ?? messagesHistory: mapLangchainMessagesToOrchestrationMessages(messages), inputParams: options.inputParams }, options.customRequestConfig); diff --git a/packages/langchain/src/orchestration/types.ts b/packages/langchain/src/orchestration/types.ts index a938aaf83..7ceb121a8 100644 --- a/packages/langchain/src/orchestration/types.ts +++ b/packages/langchain/src/orchestration/types.ts @@ -1,14 +1,12 @@ -import type { Prompt } from '@sap-ai-sdk/orchestration'; -import type { BaseLLMParams } from '@langchain/core/language_models/llms'; +import type { ChatResult } from '@langchain/core/outputs'; +import type { Prompt, ModuleResults } from '@sap-ai-sdk/orchestration'; import type { BaseChatModelCallOptions } from '@langchain/core/language_models/chat_models'; import type { - AzureOpenAiCreateChatCompletionRequest, - AzureOpenAiEmbeddingModel + AzureOpenAiCreateChatCompletionRequest } from '@sap-ai-sdk/foundation-models'; import type { CustomRequestConfig } from '@sap-ai-sdk/core'; -import type { ModelConfig, ResourceGroupConfig } from '@sap-ai-sdk/ai-api'; /** * Call options for the {@link AzureOpenAiChatClient}. @@ -31,15 +29,11 @@ export type AzureOpenAiChatCallOptions = BaseChatModelCallOptions & }; /** - * Options for orchestration calls that combines the base chat model call options with prompt-related settings. - * @typedef {Object} OrchestrationCallOptions - * @augments {BaseChatModelCallOptions} - * @augments {Prompt} + * TODO: Add docs. */ export type OrchestrationCallOptions = BaseChatModelCallOptions & { inputParams: Prompt['inputParams']; customRequestConfig: CustomRequestConfig }; /** - * Input type for {@link AzureOpenAiEmbeddingClient} initialization. + * TODO: Add docs. */ -export type AzureOpenAiEmbeddingModelParams = - ModelConfig & ResourceGroupConfig & BaseLLMParams; +export type OrchestrationResponse = ChatResult & ModuleResults & { request_id: string }; diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index e8c823a4b..f10108e6a 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -1,14 +1,13 @@ import { AIMessage } from '@langchain/core/messages'; -import type { ChatMessages } from '@sap-ai-sdk/orchestration'; +import type { ChatMessages, CompletionPostResponse } from '@sap-ai-sdk/orchestration'; import type { ToolCall } from '@langchain/core/messages/tool'; import type { - AzureOpenAiCreateChatCompletionResponse, AzureOpenAiChatCompletionMessageToolCalls } from '@sap-ai-sdk/foundation-models'; import type { BaseMessage } from '@langchain/core/messages'; -import type { ChatResult } from '@langchain/core/outputs'; +import type { OrchestrationResponse } from './types.js'; /** * Maps LangChain messages to Orchestration messages. @@ -46,8 +45,8 @@ function mapAzureOpenAiToLangchainToolCall( * @internal */ export function mapOutputToChatResult( - completionResponse: AzureOpenAiCreateChatCompletionResponse -): ChatResult { + completionResponse: CompletionPostResponse +): OrchestrationResponse { return { generations: completionResponse.choices.map(choice => ({ text: choice.message.content ?? '', From b08589c6c406c925e3a5489111049e33557c741a Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Mon, 24 Feb 2025 18:00:21 +0100 Subject: [PATCH 04/23] latest changes --- packages/langchain/src/orchestration/chat.ts | 70 ++++++++++--------- .../orchestration-message-chunk.ts | 16 +++++ .../orchestration/orchestration-message.ts | 16 +++++ packages/langchain/src/orchestration/types.ts | 18 +++-- packages/langchain/src/orchestration/util.ts | 8 ++- 5 files changed, 86 insertions(+), 42 deletions(-) create mode 100644 packages/langchain/src/orchestration/orchestration-message-chunk.ts create mode 100644 packages/langchain/src/orchestration/orchestration-message.ts diff --git a/packages/langchain/src/orchestration/chat.ts b/packages/langchain/src/orchestration/chat.ts index d330d8ffe..55330ee1e 100644 --- a/packages/langchain/src/orchestration/chat.ts +++ b/packages/langchain/src/orchestration/chat.ts @@ -1,72 +1,77 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models'; import { OrchestrationClient as OrchestrationClientBase } from '@sap-ai-sdk/orchestration'; +import { AsyncCaller } from '@langchain/core/utils/async_caller'; import { mapLangchainMessagesToOrchestrationMessages, mapOutputToChatResult } from './util.js'; +import type { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; +import type { ChatResult } from '@langchain/core/outputs'; import type { OrchestrationModuleConfig } from '@sap-ai-sdk/orchestration'; import type { BaseChatModelParams } from '@langchain/core/language_models/chat_models'; import type { ResourceGroupConfig } from '@sap-ai-sdk/ai-api'; import type { BaseMessage } from '@langchain/core/messages'; import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager'; import type { - OrchestrationCallOptions, - OrchestrationResponse + OrchestrationCallOptions } from './types.js'; import type { HttpDestinationOrFetchOptions } from '@sap-cloud-sdk/connectivity'; +// TODO: Update all docs + /** * LangChain chat client for Azure OpenAI consumption on SAP BTP. */ -export class OrchestrationClient extends BaseChatModel { - // Omit streaming until supported - orchestrationConfig: Omit; - langchainOptions?: BaseChatModelParams; - deploymentConfig?: ResourceGroupConfig; - destination?: HttpDestinationOrFetchOptions; - - // initialize with complete config and shit - // allow to prompt + pass model params at invocatio - // initialize a new client on every call, so that .bind() and .bindTools will properly work - // shit is still cached because it's not instance specific +export class OrchestrationClient extends BaseChatModel { constructor( // Omit streaming until supported - orchestrationConfig: Omit, - langchainOptions: BaseChatModelParams = {}, - deploymentConfig?: ResourceGroupConfig, - destination?: HttpDestinationOrFetchOptions, + public orchestrationConfig: Omit, + public langchainOptions: BaseChatModelParams = {}, + public deploymentConfig?: ResourceGroupConfig, + public destination?: HttpDestinationOrFetchOptions, ) { super(langchainOptions); - this.orchestrationConfig = orchestrationConfig; - this.destination = destination; - this.deploymentConfig = deploymentConfig; } _llmType(): string { return 'orchestration'; } + /** + * Decisions: + * bind only supports ParsedCallOptions, we don't support arbitrary LLM options, only tool calls & default BaseLanguageModelCallOptions, e.g. stop + * this aligns with other vendors' client designs (e.g. openai, google) + * top of the array (array[array.length - 1]) contains the current message, everything before then is history. + * Module results are part of our own message type, which extends AI Message to work with all other langchain functionality. + * + * For timeout, we need to apply our own middleware, it is not handled by langchain. + */ + override async _generate( messages: BaseMessage[], - // Ignoring all default options for now, could make sense to initalize a new caller based on the bound properties + options - // that way the request options, e.g. maxRetries etc. will be applied. - // some other options like tool_choice need to be put inside of the llm model_params and merged with existing configs - // if we want to support those options - // we can also make LLM params a call option - // this would AFAIK align more with other langchain clients options: typeof this.ParsedCallOptions, runManager?: CallbackManagerForLLMRun - ): Promise { - const res = await this.caller.callWithOptions( + ): Promise { + let caller = this.caller; + if(options.maxConcurrency) { + const { maxConcurrency, maxRetries, onFailedAttempt } = this.langchainOptions; + caller = new AsyncCaller( + { maxConcurrency: maxConcurrency ?? options.maxConcurrency, + maxRetries, + onFailedAttempt + } + ); + } + const res = await caller.callWithOptions( { signal: options.signal }, () => { - // Initializing a new client every time, as caching is unaffected - // and we support the .bind() flow of langchain this way + // consider this.tools & this.stop property, merge it ith template orchestration config const orchestrationClient = new OrchestrationClientBase(this.orchestrationConfig, this.deploymentConfig, this.destination); + const { messageHistory, inputParams } = mapLangchainMessagesToOrchestrationMessages(messages); return orchestrationClient.chatCompletion({ // how to handle tools here? doesn't really exist as input in orchestration as message history // make template a call option, to merge it ?? - messagesHistory: mapLangchainMessagesToOrchestrationMessages(messages), - inputParams: options.inputParams + messagesHistory, + inputParams }, options.customRequestConfig); } ); @@ -81,3 +86,4 @@ export class OrchestrationClient extends BaseChatModel return mapOutputToChatResult(res.data); } } + diff --git a/packages/langchain/src/orchestration/orchestration-message-chunk.ts b/packages/langchain/src/orchestration/orchestration-message-chunk.ts new file mode 100644 index 000000000..9ff0d6d7f --- /dev/null +++ b/packages/langchain/src/orchestration/orchestration-message-chunk.ts @@ -0,0 +1,16 @@ +import { AIMessageChunk } from '@langchain/core/messages'; +import type { AIMessageChunkFields } from '@langchain/core/messages'; +import type { ModuleResults } from '@sap-ai-sdk/orchestration'; + +/** + * TODO: Add docs. + */ +export class OrchestrationMessageChunk extends AIMessageChunk { + module_results: ModuleResults; + request_id: string; + constructor(fields: string | AIMessageChunkFields, module_results: ModuleResults, request_id: string) { + super(fields); + this.module_results = module_results; + this.request_id = request_id; + } + } diff --git a/packages/langchain/src/orchestration/orchestration-message.ts b/packages/langchain/src/orchestration/orchestration-message.ts new file mode 100644 index 000000000..69fbd1952 --- /dev/null +++ b/packages/langchain/src/orchestration/orchestration-message.ts @@ -0,0 +1,16 @@ +import { AIMessage } from '@langchain/core/messages'; +import type { AIMessageFields } from '@langchain/core/messages'; +import type { ModuleResults } from '@sap-ai-sdk/orchestration'; + +/** + * TODO: Add docs. + */ +export class OrchestrationMessage extends AIMessage { + module_results: ModuleResults; + request_id: string; + constructor(fields: string | AIMessageFields, module_results: ModuleResults, request_id: string) { + super(fields); + this.module_results = module_results; + this.request_id = request_id; + } +} diff --git a/packages/langchain/src/orchestration/types.ts b/packages/langchain/src/orchestration/types.ts index 7ceb121a8..d0d238703 100644 --- a/packages/langchain/src/orchestration/types.ts +++ b/packages/langchain/src/orchestration/types.ts @@ -1,5 +1,4 @@ -import type { ChatResult } from '@langchain/core/outputs'; -import type { Prompt, ModuleResults } from '@sap-ai-sdk/orchestration'; +import type { Template } from '@sap-ai-sdk/orchestration'; import type { BaseChatModelCallOptions } from '@langchain/core/language_models/chat_models'; @@ -31,9 +30,14 @@ export type AzureOpenAiChatCallOptions = BaseChatModelCallOptions & /** * TODO: Add docs. */ -export type OrchestrationCallOptions = BaseChatModelCallOptions & { inputParams: Prompt['inputParams']; customRequestConfig: CustomRequestConfig }; +export type OrchestrationCallOptions = Pick< + BaseChatModelCallOptions, + | 'stop' + | 'signal' + | 'maxConcurrency' + | 'timeout' + > & { + customRequestConfig?: CustomRequestConfig; + tools?: Template['tools']; + }; -/** - * TODO: Add docs. - */ -export type OrchestrationResponse = ChatResult & ModuleResults & { request_id: string }; diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index f10108e6a..8cd49a11a 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -1,4 +1,4 @@ -import { AIMessage } from '@langchain/core/messages'; +import { OrchestrationMessage } from './orchestration-message.js'; import type { ChatMessages, CompletionPostResponse } from '@sap-ai-sdk/orchestration'; import type { ToolCall } from '@langchain/core/messages/tool'; import type { @@ -50,7 +50,7 @@ export function mapOutputToChatResult( return { generations: completionResponse.choices.map(choice => ({ text: choice.message.content ?? '', - message: new AIMessage({ + message: new OrchestrationMessage({ content: choice.message.content ?? '', tool_calls: mapAzureOpenAiToLangchainToolCall( choice.message.tool_calls @@ -61,7 +61,9 @@ export function mapOutputToChatResult( function_call: choice.message.function_call, tool_calls: choice.message.tool_calls } - }), + }, + completionResponse.module_results, + completionResponse.request_id), generationInfo: { finish_reason: choice.finish_reason, index: choice.index, From 71ab5dc04263678f820e4200e0417caa96f85472 Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Tue, 25 Feb 2025 16:02:31 +0100 Subject: [PATCH 05/23] add mapping functions --- packages/langchain/package.json | 1 + packages/langchain/src/orchestration/chat.ts | 106 +++++++++--- .../orchestration-message-chunk.ts | 18 +- .../orchestration/orchestration-message.ts | 18 +- packages/langchain/src/orchestration/types.ts | 44 +---- packages/langchain/src/orchestration/util.ts | 156 ++++++++++++++---- pnpm-lock.yaml | 9 +- 7 files changed, 240 insertions(+), 112 deletions(-) diff --git a/packages/langchain/package.json b/packages/langchain/package.json index fa763d451..605f7bf70 100644 --- a/packages/langchain/package.json +++ b/packages/langchain/package.json @@ -31,6 +31,7 @@ "@sap-ai-sdk/foundation-models": "workspace:^", "@sap-ai-sdk/orchestration": "workspace:^", "@sap-cloud-sdk/connectivity": "^3.26.1", + "@sap-cloud-sdk/resilience": "^3.26.1", "uuid": "^11.1.0", "@langchain/core": "0.3.40", "zod-to-json-schema": "^3.24.1" diff --git a/packages/langchain/src/orchestration/chat.ts b/packages/langchain/src/orchestration/chat.ts index 55330ee1e..f2b598f66 100644 --- a/packages/langchain/src/orchestration/chat.ts +++ b/packages/langchain/src/orchestration/chat.ts @@ -1,7 +1,13 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models'; import { OrchestrationClient as OrchestrationClientBase } from '@sap-ai-sdk/orchestration'; import { AsyncCaller } from '@langchain/core/utils/async_caller'; -import { mapLangchainMessagesToOrchestrationMessages, mapOutputToChatResult } from './util.js'; +import { resilience } from '@sap-cloud-sdk/resilience'; +import { + isTemplate, + mapLangchainMessagesToOrchestrationMessages, + mapOutputToChatResult +} from './util.js'; +import type { CustomRequestConfig } from '@sap-ai-sdk/core'; import type { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; import type { ChatResult } from '@langchain/core/outputs'; import type { OrchestrationModuleConfig } from '@sap-ai-sdk/orchestration'; @@ -9,9 +15,7 @@ import type { BaseChatModelParams } from '@langchain/core/language_models/chat_m import type { ResourceGroupConfig } from '@sap-ai-sdk/ai-api'; import type { BaseMessage } from '@langchain/core/messages'; import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager'; -import type { - OrchestrationCallOptions -} from './types.js'; +import type { OrchestrationCallOptions } from './types.js'; import type { HttpDestinationOrFetchOptions } from '@sap-cloud-sdk/connectivity'; // TODO: Update all docs @@ -19,13 +23,16 @@ import type { HttpDestinationOrFetchOptions } from '@sap-cloud-sdk/connectivity' /** * LangChain chat client for Azure OpenAI consumption on SAP BTP. */ -export class OrchestrationClient extends BaseChatModel { +export class OrchestrationClient extends BaseChatModel< + OrchestrationCallOptions, + OrchestrationMessageChunk +> { constructor( // Omit streaming until supported public orchestrationConfig: Omit, public langchainOptions: BaseChatModelParams = {}, public deploymentConfig?: ResourceGroupConfig, - public destination?: HttpDestinationOrFetchOptions, + public destination?: HttpDestinationOrFetchOptions ) { super(langchainOptions); } @@ -36,12 +43,12 @@ export class OrchestrationClient extends BaseChatModel { let caller = this.caller; - if(options.maxConcurrency) { - const { maxConcurrency, maxRetries, onFailedAttempt } = this.langchainOptions; - caller = new AsyncCaller( - { maxConcurrency: maxConcurrency ?? options.maxConcurrency, - maxRetries, - onFailedAttempt - } - ); + if (options.maxConcurrency) { + const { maxConcurrency, maxRetries, onFailedAttempt } = + this.langchainOptions; + caller = new AsyncCaller({ + maxConcurrency: maxConcurrency ?? options.maxConcurrency, + maxRetries, + onFailedAttempt + }); } const res = await caller.callWithOptions( { signal: options.signal }, () => { - // consider this.tools & this.stop property, merge it ith template orchestration config - const orchestrationClient = new OrchestrationClientBase(this.orchestrationConfig, this.deploymentConfig, this.destination); - const { messageHistory, inputParams } = mapLangchainMessagesToOrchestrationMessages(messages); - return orchestrationClient.chatCompletion({ - // how to handle tools here? doesn't really exist as input in orchestration as message history - // make template a call option, to merge it ?? - messagesHistory, - inputParams - }, options.customRequestConfig); + // consider this.tools & this.stop property, merge it with template orchestration config + const { inputParams } = options; + const mergedOrchestrationConfig = + this.mergeOrchestrationConfig(options); + const orchestrationClient = new OrchestrationClientBase( + mergedOrchestrationConfig, + this.deploymentConfig, + this.destination + ); + const messagesHistory = + mapLangchainMessagesToOrchestrationMessages(messages); + const customRequestConfig: CustomRequestConfig = { + ...options.customRequestConfig, + middleware: resilience({ timeout: options.timeout }) + }; + return orchestrationClient.chatCompletion( + { + // how to handle tools here? doesn't really exist as input in orchestration as message history + // make template a call option, to merge it ?? + messagesHistory, + inputParams + }, + customRequestConfig + ); } ); @@ -85,5 +107,35 @@ export class OrchestrationClient extends BaseChatModel & { - requestConfig?: CustomRequestConfig; - }; - /** * TODO: Add docs. */ export type OrchestrationCallOptions = Pick< BaseChatModelCallOptions, - | 'stop' - | 'signal' - | 'maxConcurrency' - | 'timeout' - > & { - customRequestConfig?: CustomRequestConfig; - tools?: Template['tools']; - }; - + 'stop' | 'signal' | 'maxConcurrency' | 'timeout' +> & { + customRequestConfig?: CustomRequestConfig; + tools?: Template['tools']; + inputParams?: Prompt['inputParams']; +}; diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 8cd49a11a..5022f94b0 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -1,23 +1,111 @@ import { OrchestrationMessage } from './orchestration-message.js'; -import type { ChatMessages, CompletionPostResponse } from '@sap-ai-sdk/orchestration'; -import type { ToolCall } from '@langchain/core/messages/tool'; +import type { ChatResult } from '@langchain/core/outputs'; import type { - AzureOpenAiChatCompletionMessageToolCalls -} from '@sap-ai-sdk/foundation-models'; + ChatMessage, + CompletionPostResponse, + Template, + TemplatingModuleConfig +} from '@sap-ai-sdk/orchestration'; +import type { ToolCall } from '@langchain/core/messages/tool'; +import type { AzureOpenAiChatCompletionMessageToolCalls } from '@sap-ai-sdk/foundation-models'; import type { - BaseMessage + AIMessage, + BaseMessage, + HumanMessage, + SystemMessage } from '@langchain/core/messages'; -import type { OrchestrationResponse } from './types.js'; /** - * Maps LangChain messages to Orchestration messages. - * @param messages - The messages to map. - * @returns The mapped messages. + * Checks if the object is a {@link Template}. + * @param object - The object to check. + * @returns True if the object is a {@link Template}. + */ +export function isTemplate(object: TemplatingModuleConfig): object is Template { + return 'template' in object; +} + +/** + * Maps {@link BaseMessage} to {@link AzureOpenAiChatMessage}. + * @param message - The message to map. + * @returns The {@link AzureOpenAiChatMessage}. + */ +// TODO: Add mapping of refusal property, once LangChain base class supports it natively. +function mapBaseMessageToAzureOpenAiChatMessage( + message: BaseMessage +): ChatMessage { + switch (message.getType()) { + case 'ai': + return mapAiMessageToAzureOpenAiAssistantMessage(message); + case 'human': + return mapHumanMessageToChatMessage(message); + case 'system': + return mapSystemMessageToAzureOpenAiSystemMessage(message); + // TODO: As soon as tool messages are supported by orchestration, create mapping function similar to our azure mapping function. + case 'function': + case 'tool': + default: + throw new Error(`Unsupported message type: ${message.getType()}`); + } +} + +/** + * Maps LangChain's {@link AIMessage} to Azure OpenAI's {@link AzureOpenAiChatCompletionRequestAssistantMessage}. + * @param message - The {@link AIMessage} to map. + * @returns The Azure OpenAI {@link AzureOpenAiChatCompletionRequestAssistantMessage}. + */ +function mapAiMessageToAzureOpenAiAssistantMessage( + message: AIMessage +): ChatMessage { + /* TODO: Tool calls are currently bugged in orchestration, pass these fields as soon as orchestration supports it. + const tool_calls = + mapLangchainToolCallToAzureOpenAiToolCall(message.tool_calls) ?? + message.additional_kwargs.tool_calls; + */ + return { + /* TODO: Tool calls are currently bugged in orchestration, pass these fields as soon as orchestration supports it. + ...(tool_calls?.length ? { tool_calls } : {}), + function_call: message.additional_kwargs.function_call, + */ + content: message.content, + role: 'assistant' + } as ChatMessage; +} + +function mapHumanMessageToChatMessage(message: HumanMessage): ChatMessage { + return { + role: 'user', + content: message.content + } as ChatMessage; +} + +function mapSystemMessageToAzureOpenAiSystemMessage( + message: SystemMessage +): ChatMessage { + // TODO: Remove as soon as image_url is a supported inputed for system messages in orchestration. + if ( + typeof message.content !== 'string' && + message.content.some(content => content.type === 'image_url') + ) { + throw new Error( + 'System messages with image URLs are not supported by the Orchestration Client.' + ); + } + return { + role: 'system', + content: message.content + } as ChatMessage; +} + +/** + * TODO: adjust + * Maps LangChain messages to orchestration messages. + * @param messages - The LangChain messages to map. + * @returns The orchestration messages mapped from LangChain messages. */ export function mapLangchainMessagesToOrchestrationMessages( messages: BaseMessage[] -): ChatMessages { - return []; +): ChatMessage[] { + return messages.map(mapBaseMessageToAzureOpenAiChatMessage); } /** @@ -39,38 +127,42 @@ function mapAzureOpenAiToLangchainToolCall( } /** - * Maps {@link AzureOpenAiCreateChatCompletionResponse} to LangChain's {@link ChatResult}. - * @param completionResponse - The {@link AzureOpenAiCreateChatCompletionResponse} response. - * @returns The LangChain {@link ChatResult} + * Maps the completion response to a {@link ChatResult}. + * @param completionResponse - The completion response to map. + * @returns The mapped {@link ChatResult}. * @internal */ export function mapOutputToChatResult( completionResponse: CompletionPostResponse -): OrchestrationResponse { +): ChatResult { return { - generations: completionResponse.choices.map(choice => ({ - text: choice.message.content ?? '', - message: new OrchestrationMessage({ - content: choice.message.content ?? '', - tool_calls: mapAzureOpenAiToLangchainToolCall( - choice.message.tool_calls + generations: completionResponse.orchestration_result.choices.map( + choice => ({ + text: choice.message.content ?? '', + message: new OrchestrationMessage( + { + content: choice.message.content ?? '', + tool_calls: mapAzureOpenAiToLangchainToolCall( + choice.message.tool_calls + ), + additional_kwargs: { + finish_reason: choice.finish_reason, + index: choice.index, + function_call: choice.message.function_call, + tool_calls: choice.message.tool_calls + } + }, + completionResponse.module_results, + completionResponse.request_id ), - additional_kwargs: { + generationInfo: { finish_reason: choice.finish_reason, index: choice.index, function_call: choice.message.function_call, tool_calls: choice.message.tool_calls } - }, - completionResponse.module_results, - completionResponse.request_id), - generationInfo: { - finish_reason: choice.finish_reason, - index: choice.index, - function_call: choice.message.function_call, - tool_calls: choice.message.tool_calls - } - })), + }) + ), llmOutput: { created: completionResponse.created, id: completionResponse.id, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d0b9c4ac8..8c8c59c73 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -163,6 +163,9 @@ importers: '@sap-cloud-sdk/connectivity': specifier: ^3.26.1 version: 3.26.1 + '@sap-cloud-sdk/resilience': + specifier: ^3.26.1 + version: 3.26.1 uuid: specifier: ^11.1.0 version: 11.1.0 @@ -6127,7 +6130,7 @@ snapshots: eslint: 9.20.1 eslint-config-prettier: 10.0.1(eslint@9.20.1) eslint-import-resolver-typescript: 3.8.1(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1)(typescript@5.7.3))(eslint@9.20.1))(eslint@9.20.1) - eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1)(typescript@5.7.3))(eslint-import-resolver-typescript@3.8.1)(eslint@9.20.1) + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1)(typescript@5.7.3))(eslint-import-resolver-typescript@3.8.1(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1)(typescript@5.7.3))(eslint@9.20.1))(eslint@9.20.1))(eslint@9.20.1) eslint-plugin-jsdoc: 50.6.3(eslint@9.20.1) eslint-plugin-prettier: 5.2.3(@types/eslint@8.56.10)(eslint-config-prettier@10.0.1(eslint@9.20.1))(eslint@9.20.1)(prettier@3.5.1) eslint-plugin-regex: 1.10.0(eslint@9.20.1) @@ -7567,7 +7570,7 @@ snapshots: stable-hash: 0.0.4 tinyglobby: 0.2.11 optionalDependencies: - eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1)(typescript@5.7.3))(eslint-import-resolver-typescript@3.8.1)(eslint@9.20.1) + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1)(typescript@5.7.3))(eslint-import-resolver-typescript@3.8.1(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1)(typescript@5.7.3))(eslint@9.20.1))(eslint@9.20.1))(eslint@9.20.1) transitivePeerDependencies: - supports-color @@ -7582,7 +7585,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1)(typescript@5.7.3))(eslint-import-resolver-typescript@3.8.1)(eslint@9.20.1): + eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1)(typescript@5.7.3))(eslint-import-resolver-typescript@3.8.1(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1)(typescript@5.7.3))(eslint@9.20.1))(eslint@9.20.1))(eslint@9.20.1): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.8 From 87ac9add308444ca09821546b6430b248e6b7cc7 Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Tue, 25 Feb 2025 17:25:10 +0100 Subject: [PATCH 06/23] add sample code --- packages/langchain/src/index.ts | 2 + packages/langchain/src/orchestration/chat.ts | 15 +++---- packages/langchain/src/orchestration/index.ts | 5 +++ .../orchestration-message-chunk.ts | 3 +- .../orchestration/orchestration-message.ts | 3 +- packages/langchain/src/orchestration/types.ts | 2 +- packages/langchain/src/orchestration/util.ts | 13 +++--- sample-code/src/langchain-orchestration.ts | 42 +++++++++++++++++++ sample-code/src/server.ts | 14 +++++++ 9 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 sample-code/src/langchain-orchestration.ts diff --git a/packages/langchain/src/index.ts b/packages/langchain/src/index.ts index 5e1fcb6ce..a5aa75dec 100644 --- a/packages/langchain/src/index.ts +++ b/packages/langchain/src/index.ts @@ -7,3 +7,5 @@ export type { AzureOpenAiEmbeddingModelParams, AzureOpenAiChatCallOptions } from './openai/index.js'; +export { OrchestrationClient } from './orchestration/index.js'; +export type { OrchestrationCallOptions } from './orchestration/index.js'; diff --git a/packages/langchain/src/orchestration/chat.ts b/packages/langchain/src/orchestration/chat.ts index f2b598f66..663e09fb4 100644 --- a/packages/langchain/src/orchestration/chat.ts +++ b/packages/langchain/src/orchestration/chat.ts @@ -18,17 +18,15 @@ import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager import type { OrchestrationCallOptions } from './types.js'; import type { HttpDestinationOrFetchOptions } from '@sap-cloud-sdk/connectivity'; -// TODO: Update all docs - /** - * LangChain chat client for Azure OpenAI consumption on SAP BTP. + * The Orchestration client. */ export class OrchestrationClient extends BaseChatModel< OrchestrationCallOptions, OrchestrationMessageChunk > { constructor( - // Omit streaming until supported + // TODO: Omit streaming until supported public orchestrationConfig: Omit, public langchainOptions: BaseChatModelParams = {}, public deploymentConfig?: ResourceGroupConfig, @@ -45,8 +43,8 @@ export class OrchestrationClient extends BaseChatModel< * Decisions: * bind only supports ParsedCallOptions, we don't support arbitrary LLM options, only tool calls & default BaseLanguageModelCallOptions, e.g. stop ✅ * this aligns with other vendors' client designs (e.g. openai, google) ✅ - * top of the array (array[array.length - 1]) contains the current message, everything before then is history. - * Module results are part of our own message type, which extends AI Message to work with all other langchain functionality. + * inputParams are a seperate call option, history = history ✅ + * Module results are part of our own message type, which extends AI Message to work with all other langchain functionality. ✅. * * For timeout, we need to apply our own middleware, it is not handled by langchain. ✅. */ @@ -71,7 +69,6 @@ export class OrchestrationClient extends BaseChatModel< signal: options.signal }, () => { - // consider this.tools & this.stop property, merge it with template orchestration config const { inputParams } = options; const mergedOrchestrationConfig = this.mergeOrchestrationConfig(options); @@ -88,8 +85,6 @@ export class OrchestrationClient extends BaseChatModel< }; return orchestrationClient.chatCompletion( { - // how to handle tools here? doesn't really exist as input in orchestration as message history - // make template a call option, to merge it ?? messagesHistory, inputParams }, @@ -100,7 +95,7 @@ export class OrchestrationClient extends BaseChatModel< const content = res.getContent(); - // we currently do not support streaming + // TODO: Add streaming as soon as we support it await runManager?.handleLLMNewToken( typeof content === 'string' ? content : '' ); diff --git a/packages/langchain/src/orchestration/index.ts b/packages/langchain/src/orchestration/index.ts index e69de29bb..b69f80553 100644 --- a/packages/langchain/src/orchestration/index.ts +++ b/packages/langchain/src/orchestration/index.ts @@ -0,0 +1,5 @@ +export * from './chat.js'; +export * from './orchestration-message.js'; +export * from './orchestration-message-chunk.js'; +export * from './types.js'; +export * from './util.js'; diff --git a/packages/langchain/src/orchestration/orchestration-message-chunk.ts b/packages/langchain/src/orchestration/orchestration-message-chunk.ts index 6a934b3fe..67236b7e7 100644 --- a/packages/langchain/src/orchestration/orchestration-message-chunk.ts +++ b/packages/langchain/src/orchestration/orchestration-message-chunk.ts @@ -3,7 +3,8 @@ import type { AIMessageChunkFields } from '@langchain/core/messages'; import type { ModuleResults } from '@sap-ai-sdk/orchestration'; /** - * TODO: Add docs. + * An AI Message Chunk containing module results and request ID. + * @internal */ export class OrchestrationMessageChunk extends AIMessageChunk { module_results: ModuleResults; diff --git a/packages/langchain/src/orchestration/orchestration-message.ts b/packages/langchain/src/orchestration/orchestration-message.ts index 50ef9c7fb..c84fa0320 100644 --- a/packages/langchain/src/orchestration/orchestration-message.ts +++ b/packages/langchain/src/orchestration/orchestration-message.ts @@ -3,7 +3,8 @@ import type { AIMessageFields } from '@langchain/core/messages'; import type { ModuleResults } from '@sap-ai-sdk/orchestration'; /** - * TODO: Add docs. + * An AI Message containing module results and request ID. + * @internal */ export class OrchestrationMessage extends AIMessage { module_results: ModuleResults; diff --git a/packages/langchain/src/orchestration/types.ts b/packages/langchain/src/orchestration/types.ts index fb00ef065..ece952e81 100644 --- a/packages/langchain/src/orchestration/types.ts +++ b/packages/langchain/src/orchestration/types.ts @@ -3,7 +3,7 @@ import type { BaseChatModelCallOptions } from '@langchain/core/language_models/c import type { CustomRequestConfig } from '@sap-ai-sdk/core'; /** - * TODO: Add docs. + * Options for an orchestration call. */ export type OrchestrationCallOptions = Pick< BaseChatModelCallOptions, diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 5022f94b0..660d282f8 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -19,20 +19,19 @@ import type { * Checks if the object is a {@link Template}. * @param object - The object to check. * @returns True if the object is a {@link Template}. + * @internal */ export function isTemplate(object: TemplatingModuleConfig): object is Template { return 'template' in object; } /** - * Maps {@link BaseMessage} to {@link AzureOpenAiChatMessage}. + * Maps {@link BaseMessage} to {@link ChatMessage}. * @param message - The message to map. - * @returns The {@link AzureOpenAiChatMessage}. + * @returns The {@link ChatMessage}. */ // TODO: Add mapping of refusal property, once LangChain base class supports it natively. -function mapBaseMessageToAzureOpenAiChatMessage( - message: BaseMessage -): ChatMessage { +function mapBaseMessageToChatMessage(message: BaseMessage): ChatMessage { switch (message.getType()) { case 'ai': return mapAiMessageToAzureOpenAiAssistantMessage(message); @@ -97,15 +96,15 @@ function mapSystemMessageToAzureOpenAiSystemMessage( } /** - * TODO: adjust * Maps LangChain messages to orchestration messages. * @param messages - The LangChain messages to map. * @returns The orchestration messages mapped from LangChain messages. + * @internal */ export function mapLangchainMessagesToOrchestrationMessages( messages: BaseMessage[] ): ChatMessage[] { - return messages.map(mapBaseMessageToAzureOpenAiChatMessage); + return messages.map(mapBaseMessageToChatMessage); } /** diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts new file mode 100644 index 000000000..35f9945e5 --- /dev/null +++ b/sample-code/src/langchain-orchestration.ts @@ -0,0 +1,42 @@ +import { StringOutputParser } from '@langchain/core/output_parsers'; +import { OrchestrationClient } from '@sap-ai-sdk/langchain'; +import type { BaseLanguageModelInput } from '@langchain/core/language_models/base'; +import type { Runnable } from '@langchain/core/runnables'; +import type { OrchestrationCallOptions } from '@sap-ai-sdk/langchain'; + +/** + * Ask GPT about the capital of France, as part of a chain. + * @returns The answer from ChatGPT. + */ +export async function invokeChain(): Promise { + const orchestrationConfig = { + // define the language model to be used + llm: { + model_name: 'gpt-35-turbo', + model_params: {} + }, + // define the template + templating: { + template: [ + { + role: 'user', + content: 'Give me a long introduction of {{?input}}' + } + ] + } + }; + + const callOptions = { inputParams: { input: 'SAP Cloud SDK' } }; + + // initialize the client + const client = new OrchestrationClient(orchestrationConfig); + + // create an output parser + const parser = new StringOutputParser(); + + // chain together template, client, and parser + const llmChain = client.pipe(parser) as Runnable; + + // invoke the chain + return llmChain.invoke('My Message History', callOptions); +} diff --git a/sample-code/src/server.ts b/sample-code/src/server.ts index 0c5e3f5f8..ab74b6157 100644 --- a/sample-code/src/server.ts +++ b/sample-code/src/server.ts @@ -42,6 +42,9 @@ import { invoke, invokeToolChain } from './langchain-azure-openai.js'; +import { + invokeChain as invokeChainOrchestration +} from './langchain-orchestration.js'; import { createCollection, createDocumentsWithTimestamp, @@ -416,6 +419,17 @@ app.get('/langchain/invoke-chain', async (req, res) => { } }); +app.get('/langchain/invoke-chain-orchestration', async (req, res) => { + try { + res.send(await invokeChainOrchestration()); + } catch (error: any) { + console.error(error); + res + .status(500) + .send('Yikes, vibes are off apparently 😬 -> ' + error.request.data); + } +}); + app.get('/langchain/invoke-rag-chain', async (req, res) => { try { res.send(await invokeRagChain()); From 6692700700d89d4162f9263955fa6f0332246bbc Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Thu, 27 Feb 2025 13:45:20 +0100 Subject: [PATCH 07/23] override super method, remove cast --- packages/langchain/src/orchestration/chat.ts | 23 ++++++++++++++------ sample-code/src/langchain-orchestration.ts | 5 +---- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/langchain/src/orchestration/chat.ts b/packages/langchain/src/orchestration/chat.ts index 663e09fb4..d478adbd0 100644 --- a/packages/langchain/src/orchestration/chat.ts +++ b/packages/langchain/src/orchestration/chat.ts @@ -7,6 +7,8 @@ import { mapLangchainMessagesToOrchestrationMessages, mapOutputToChatResult } from './util.js'; +import type { BaseLanguageModelInput } from '@langchain/core/language_models/base'; +import type { Runnable, RunnableLike } from '@langchain/core/runnables'; import type { CustomRequestConfig } from '@sap-ai-sdk/core'; import type { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; import type { ChatResult } from '@langchain/core/outputs'; @@ -40,14 +42,21 @@ export class OrchestrationClient extends BaseChatModel< } /** - * Decisions: - * bind only supports ParsedCallOptions, we don't support arbitrary LLM options, only tool calls & default BaseLanguageModelCallOptions, e.g. stop ✅ - * this aligns with other vendors' client designs (e.g. openai, google) ✅ - * inputParams are a seperate call option, history = history ✅ - * Module results are part of our own message type, which extends AI Message to work with all other langchain functionality. ✅. - * - * For timeout, we need to apply our own middleware, it is not handled by langchain. ✅. + * Create a new runnable sequence that runs each individual runnable in series, + * piping the output of one runnable into another runnable or runnable-like. + * @param coerceable - A runnable, function, or object whose values are functions or runnables. + * @returns A new runnable sequence. */ + override pipe( + coerceable: RunnableLike + ): Runnable, OrchestrationCallOptions> { + // Delegate to the superclass pipe method and narrow the type. + return super.pipe(coerceable) as Runnable< + BaseLanguageModelInput, + Exclude, + OrchestrationCallOptions + >; + } override async _generate( messages: BaseMessage[], diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts index 35f9945e5..68fd747f2 100644 --- a/sample-code/src/langchain-orchestration.ts +++ b/sample-code/src/langchain-orchestration.ts @@ -1,8 +1,5 @@ import { StringOutputParser } from '@langchain/core/output_parsers'; import { OrchestrationClient } from '@sap-ai-sdk/langchain'; -import type { BaseLanguageModelInput } from '@langchain/core/language_models/base'; -import type { Runnable } from '@langchain/core/runnables'; -import type { OrchestrationCallOptions } from '@sap-ai-sdk/langchain'; /** * Ask GPT about the capital of France, as part of a chain. @@ -35,7 +32,7 @@ export async function invokeChain(): Promise { const parser = new StringOutputParser(); // chain together template, client, and parser - const llmChain = client.pipe(parser) as Runnable; + const llmChain = client.pipe(parser); // invoke the chain return llmChain.invoke('My Message History', callOptions); From c97d4a8a4af091a044c5e17126a99965d4882482 Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Thu, 27 Feb 2025 13:47:05 +0100 Subject: [PATCH 08/23] lint --- packages/langchain/src/orchestration/chat.ts | 6 +++++- sample-code/src/server.ts | 4 +--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/langchain/src/orchestration/chat.ts b/packages/langchain/src/orchestration/chat.ts index d478adbd0..479640b22 100644 --- a/packages/langchain/src/orchestration/chat.ts +++ b/packages/langchain/src/orchestration/chat.ts @@ -49,7 +49,11 @@ export class OrchestrationClient extends BaseChatModel< */ override pipe( coerceable: RunnableLike - ): Runnable, OrchestrationCallOptions> { + ): Runnable< + BaseLanguageModelInput, + Exclude, + OrchestrationCallOptions + > { // Delegate to the superclass pipe method and narrow the type. return super.pipe(coerceable) as Runnable< BaseLanguageModelInput, diff --git a/sample-code/src/server.ts b/sample-code/src/server.ts index ab74b6157..c81aae100 100644 --- a/sample-code/src/server.ts +++ b/sample-code/src/server.ts @@ -42,9 +42,7 @@ import { invoke, invokeToolChain } from './langchain-azure-openai.js'; -import { - invokeChain as invokeChainOrchestration -} from './langchain-orchestration.js'; +import { invokeChain as invokeChainOrchestration } from './langchain-orchestration.js'; import { createCollection, createDocumentsWithTimestamp, From 2ebb41270878171142f45a857524558efa208ca1 Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Thu, 27 Feb 2025 14:44:28 +0100 Subject: [PATCH 09/23] add util tests --- .../src/orchestration/client.test.ts | 1 + .../src/orchestration/{chat.ts => client.ts} | 0 packages/langchain/src/orchestration/index.ts | 2 +- .../langchain/src/orchestration/util.test.ts | 202 ++++++++++++++++++ packages/langchain/src/orchestration/util.ts | 76 +++---- 5 files changed, 243 insertions(+), 38 deletions(-) create mode 100644 packages/langchain/src/orchestration/client.test.ts rename packages/langchain/src/orchestration/{chat.ts => client.ts} (100%) create mode 100644 packages/langchain/src/orchestration/util.test.ts diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts new file mode 100644 index 000000000..426adbc97 --- /dev/null +++ b/packages/langchain/src/orchestration/client.test.ts @@ -0,0 +1 @@ +// test timeout & maxRetries with mocked target diff --git a/packages/langchain/src/orchestration/chat.ts b/packages/langchain/src/orchestration/client.ts similarity index 100% rename from packages/langchain/src/orchestration/chat.ts rename to packages/langchain/src/orchestration/client.ts diff --git a/packages/langchain/src/orchestration/index.ts b/packages/langchain/src/orchestration/index.ts index b69f80553..fac385586 100644 --- a/packages/langchain/src/orchestration/index.ts +++ b/packages/langchain/src/orchestration/index.ts @@ -1,4 +1,4 @@ -export * from './chat.js'; +export * from './client.js'; export * from './orchestration-message.js'; export * from './orchestration-message-chunk.js'; export * from './types.js'; diff --git a/packages/langchain/src/orchestration/util.test.ts b/packages/langchain/src/orchestration/util.test.ts new file mode 100644 index 000000000..f871db8d1 --- /dev/null +++ b/packages/langchain/src/orchestration/util.test.ts @@ -0,0 +1,202 @@ +import { + AIMessage, + HumanMessage, + SystemMessage, + ToolMessage +} from '@langchain/core/messages'; +import { + mapLangchainMessagesToOrchestrationMessages, + mapOutputToChatResult +} from './util.js'; +import type { OrchestrationMessage } from './orchestration-message.js'; +import type { + CompletionPostResponse, + ResponseMessageToolCall +} from '@sap-ai-sdk/orchestration'; + +describe('mapLangchainMessagesToOrchestrationMessages', () => { + it('should map an array of LangChain messages to Orchestration messages', () => { + const langchainMessages = [ + new SystemMessage('System message content'), + new HumanMessage('Human message content'), + new AIMessage('AI message content') + ]; + + const result = + mapLangchainMessagesToOrchestrationMessages(langchainMessages); + + expect(result).toEqual([ + { role: 'system', content: 'System message content' }, + { role: 'user', content: 'Human message content' }, + { role: 'assistant', content: 'AI message content' } + ]); + }); + + it('should throw error for unsupported message types', () => { + const langchainMessages = [ + new ToolMessage('Tool message content', 'tool-id') + ]; + + expect(() => + mapLangchainMessagesToOrchestrationMessages(langchainMessages) + ).toThrow('Unsupported message type: tool'); + }); +}); + +describe('mapBaseMessageToChatMessage', () => { + it('should map HumanMessage to ChatMessage with user role', () => { + const humanMessage = new HumanMessage('Human message content'); + + // Since mapBaseMessageToChatMessage is internal, we'll test it through mapLangchainMessagesToOrchestrationMessages + const result = mapLangchainMessagesToOrchestrationMessages([humanMessage]); + + expect(result[0]).toEqual({ + role: 'user', + content: 'Human message content' + }); + }); + + it('should map SystemMessage to ChatMessage with system role', () => { + const systemMessage = new SystemMessage('System message content'); + + const result = mapLangchainMessagesToOrchestrationMessages([systemMessage]); + + expect(result[0]).toEqual({ + role: 'system', + content: 'System message content' + }); + }); + + it('should map AIMessage to ChatMessage with assistant role', () => { + const aiMessage = new AIMessage('AI message content'); + + const result = mapLangchainMessagesToOrchestrationMessages([aiMessage]); + + expect(result[0]).toEqual({ + role: 'assistant', + content: 'AI message content' + }); + }); + + it('should throw error when mapping SystemMessage with image_url content', () => { + const systemMessage = new SystemMessage({ + content: [ + { type: 'text', text: 'System text' }, + { + type: 'image_url', + image_url: { url: 'https://example.com/image.jpg' } + } + ] + }); + + expect(() => + mapLangchainMessagesToOrchestrationMessages([systemMessage]) + ).toThrow( + 'System messages with image URLs are not supported by the Orchestration Client.' + ); + }); +}); + +describe('mapOutputToChatResult', () => { + it('should map CompletionPostResponse to ChatResult', () => { + const completionResponse: CompletionPostResponse = { + orchestration_result: { + id: 'test-id', + object: 'chat.completion', + created: 1634840000, + model: 'test-model', + choices: [ + { + message: { + content: 'Test content', + role: 'assistant' + }, + finish_reason: 'stop', + index: 0 + } + ], + usage: { + completion_tokens: 10, + prompt_tokens: 20, + total_tokens: 30 + } + }, + request_id: 'req-123', + module_results: {} + }; + + const result = mapOutputToChatResult(completionResponse); + + expect(result.generations).toHaveLength(1); + expect(result.generations[0].text).toBe('Test content'); + expect(result.generations[0].message.content).toBe('Test content'); + expect(result.generations[0].generationInfo).toEqual({ + finish_reason: 'stop', + index: 0, + function_call: undefined, + tool_calls: undefined + }); + expect(result.llmOutput).toEqual({ + created: 1634840000, + id: 'test-id', + model: 'test-model', + object: 'chat.completion', + tokenUsage: { + completionTokens: 10, + promptTokens: 20, + totalTokens: 30 + } + }); + }); + + it('should map tool_calls correctly', () => { + const toolCallData: ResponseMessageToolCall = { + id: 'call-123', + type: 'function', + function: { + name: 'test_function', + arguments: '{"arg1":"value1"}' + } + }; + + const completionResponse: CompletionPostResponse = { + orchestration_result: { + id: 'test-id', + object: 'chat.completion', + created: 1634840000, + model: 'test-model', + choices: [ + { + index: 0, + message: { + content: 'Test content', + role: 'assistant', + tool_calls: [toolCallData] + }, + finish_reason: 'tool_calls' + } + ], + usage: { + completion_tokens: 10, + prompt_tokens: 20, + total_tokens: 30 + } + }, + request_id: 'req-123', + module_results: {} + }; + + const result = mapOutputToChatResult(completionResponse); + + expect( + (result.generations[0].message as OrchestrationMessage).tool_calls + ).toEqual([ + { + id: 'call-123', + name: 'test_function', + args: { arg1: 'value1' }, + type: 'tool_call' + } + ]); + }); +}); diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 660d282f8..f0790eab7 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -3,8 +3,7 @@ import type { ChatResult } from '@langchain/core/outputs'; import type { ChatMessage, CompletionPostResponse, - Template, - TemplatingModuleConfig + Template } from '@sap-ai-sdk/orchestration'; import type { ToolCall } from '@langchain/core/messages/tool'; import type { AzureOpenAiChatCompletionMessageToolCalls } from '@sap-ai-sdk/foundation-models'; @@ -21,7 +20,7 @@ import type { * @returns True if the object is a {@link Template}. * @internal */ -export function isTemplate(object: TemplatingModuleConfig): object is Template { +export function isTemplate(object: Record): object is Template { return 'template' in object; } @@ -134,43 +133,46 @@ function mapAzureOpenAiToLangchainToolCall( export function mapOutputToChatResult( completionResponse: CompletionPostResponse ): ChatResult { + const { orchestration_result, module_results, request_id } = + completionResponse; + const { choices, created, id, model, object, usage, system_fingerprint } = + orchestration_result; return { - generations: completionResponse.orchestration_result.choices.map( - choice => ({ - text: choice.message.content ?? '', - message: new OrchestrationMessage( - { - content: choice.message.content ?? '', - tool_calls: mapAzureOpenAiToLangchainToolCall( - choice.message.tool_calls - ), - additional_kwargs: { - finish_reason: choice.finish_reason, - index: choice.index, - function_call: choice.message.function_call, - tool_calls: choice.message.tool_calls - } - }, - completionResponse.module_results, - completionResponse.request_id - ), - generationInfo: { - finish_reason: choice.finish_reason, - index: choice.index, - function_call: choice.message.function_call, - tool_calls: choice.message.tool_calls - } - }) - ), + generations: choices.map(choice => ({ + text: choice.message.content ?? '', + message: new OrchestrationMessage( + { + content: choice.message.content ?? '', + tool_calls: mapAzureOpenAiToLangchainToolCall( + choice.message.tool_calls + ), + additional_kwargs: { + finish_reason: choice.finish_reason, + index: choice.index, + function_call: choice.message.function_call, + tool_calls: choice.message.tool_calls + } + }, + module_results, + request_id + ), + generationInfo: { + finish_reason: choice.finish_reason, + index: choice.index, + function_call: choice.message.function_call, + tool_calls: choice.message.tool_calls + } + })), llmOutput: { - created: completionResponse.created, - id: completionResponse.id, - model: completionResponse.model, - object: completionResponse.object, + created, + id, + model, + object, + system_fingerprint, tokenUsage: { - completionTokens: completionResponse.usage?.completion_tokens ?? 0, - promptTokens: completionResponse.usage?.prompt_tokens ?? 0, - totalTokens: completionResponse.usage?.total_tokens ?? 0 + completionTokens: usage?.completion_tokens ?? 0, + promptTokens: usage?.prompt_tokens ?? 0, + totalTokens: usage?.total_tokens ?? 0 } } }; From fc865a6dfeb4bbaf352e8c8b9f58ac47a5624c94 Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Thu, 27 Feb 2025 16:42:06 +0100 Subject: [PATCH 10/23] add resilience tests --- .../src/orchestration/client.test.ts | 99 ++++++++++++++++++- .../langchain/src/orchestration/client.ts | 1 - test-util/mock-http.ts | 38 ++++--- 3 files changed, 124 insertions(+), 14 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 426adbc97..577e7b041 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -1 +1,98 @@ -// test timeout & maxRetries with mocked target +import nock from 'nock'; +import { constructCompletionPostRequest } from '@sap-ai-sdk/orchestration/internal.js'; +import { + mockClientCredentialsGrantCall, + mockDeploymentsList, + mockInference, + parseMockResponse +} from '../../../../test-util/mock-http.js'; +import { OrchestrationClient } from './client.js'; +import type { + CompletionPostResponse, + OrchestrationModuleConfig +} from '@sap-ai-sdk/orchestration'; + +describe('orchestration service client', () => { + beforeEach(() => { + mockClientCredentialsGrantCall(); + mockDeploymentsList({ scenarioId: 'orchestration' }, { id: '1234' }); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + it('calls chatCompletion with minimum configuration', async () => { + const config: OrchestrationModuleConfig = { + llm: { + model_name: 'gpt-35-turbo-16k', + model_params: { max_tokens: 50, temperature: 0.1 } + }, + templating: { + template: [{ role: 'user', content: 'Hello!' }] + } + }; + + const mockResponse = await parseMockResponse( + 'orchestration', + 'orchestration-chat-completion-success-response.json' + ); + + mockInference( + { + data: constructCompletionPostRequest(config, { messagesHistory: [] }) + }, + { + data: mockResponse, + status: 200 + }, + { + url: 'inference/deployments/1234/completion' + }, + { retry: 3 } + ); + const response = await new OrchestrationClient(config, { + maxRetries: 5 + }).invoke([], { timeout: 1 }); + + // expect(nock.isDone()).toBe(true); + expect(response).toMatchInlineSnapshot(` + { + "id": [ + "langchain_core", + "messages", + "OrchestrationMessage", + ], + "kwargs": { + "additional_kwargs": { + "finish_reason": "stop", + "function_call": undefined, + "index": 0, + "tool_calls": undefined, + }, + "content": "Hello! How can I assist you today?", + "invalid_tool_calls": [], + "response_metadata": { + "created": 172, + "finish_reason": "stop", + "function_call": undefined, + "id": "orchestration-id", + "index": 0, + "model": "gpt-35-turbo", + "object": "chat.completion", + "system_fingerprint": undefined, + "tokenUsage": { + "completionTokens": 9, + "promptTokens": 9, + "totalTokens": 18, + }, + "tool_calls": undefined, + }, + "tool_calls": [], + }, + "lc": 1, + "type": "constructor", + } + `); + }); +}); diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 479640b22..678e14a80 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -54,7 +54,6 @@ export class OrchestrationClient extends BaseChatModel< Exclude, OrchestrationCallOptions > { - // Delegate to the superclass pipe method and narrow the type. return super.pipe(coerceable) as Runnable< BaseLanguageModelInput, Exclude, diff --git a/test-util/mock-http.ts b/test-util/mock-http.ts index ff01e4a73..ce8d40ac5 100644 --- a/test-util/mock-http.ts +++ b/test-util/mock-http.ts @@ -2,18 +2,18 @@ import { readFile } from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import nock from 'nock'; -import { type EndpointOptions } from '@sap-ai-sdk/core'; -import { - type FoundationModel, - type DeploymentResolutionOptions -} from '@sap-ai-sdk/ai-api/internal.js'; -import { dummyToken } from './mock-jwt.js'; import { registerDestination, type DestinationAuthToken, type HttpDestination, type ServiceCredentials } from '@sap-cloud-sdk/connectivity'; +import { type EndpointOptions } from '@sap-ai-sdk/core'; +import { + type FoundationModel, + type DeploymentResolutionOptions +} from '@sap-ai-sdk/ai-api/internal.js'; +import { dummyToken } from './mock-jwt.js'; // Get the directory of this file const __filename = fileURLToPath(import.meta.url); @@ -98,19 +98,33 @@ export function mockInference( data: any; status?: number; }, - endpoint: EndpointOptions = mockEndpoint + endpoint: EndpointOptions = mockEndpoint, + resilienceOptions?: { + delay?: number; + retry?: number; + } ): nock.Scope { const { url, apiVersion, resourceGroup = 'default' } = endpoint; const destination = getMockedAiCoreDestination(); - return nock(destination.url, { + const scope = nock(destination.url, { reqheaders: { 'ai-resource-group': resourceGroup, authorization: `Bearer ${destination.authTokens?.[0].value}` } - }) - .post(`/v2/${url}`, request.data) - .query(apiVersion ? { 'api-version': apiVersion } : {}) - .reply(response.status, response.data); + }); + + let interceptor = scope.post(`/v2/${url}`, request.data).query(apiVersion ? { 'api-version': apiVersion } : {}); + + if (resilienceOptions?.retry) { + interceptor.times(resilienceOptions.retry).reply(500); + interceptor = scope.post(`/v2/${url}`, request.data).query(apiVersion ? { 'api-version': apiVersion } : {}); + } + + if (resilienceOptions?.delay) { + interceptor = interceptor.delay(resilienceOptions.delay); + } + + return interceptor.reply(response.status, response.data); } /** From 2c451b1494e8912f3fba42e972b13108ebab7de8 Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Fri, 28 Feb 2025 15:58:07 +0100 Subject: [PATCH 11/23] add remaining resilience functioanlity --- packages/langchain/README.md | 146 +----------- packages/langchain/src/README.md | 206 ++++++++++++++++ packages/langchain/src/openai/README.md | 219 ++++++++++++++++++ .../langchain/src/orchestration/README.md | 219 ++++++++++++++++++ .../__snapshots__/client.test.ts.snap | 81 +++++++ .../src/orchestration/client.test.ts | 132 ++++++----- .../langchain/src/orchestration/client.ts | 21 +- packages/langchain/src/orchestration/types.ts | 2 +- test-util/mock-http.ts | 8 +- 9 files changed, 806 insertions(+), 228 deletions(-) create mode 100644 packages/langchain/src/README.md create mode 100644 packages/langchain/src/openai/README.md create mode 100644 packages/langchain/src/orchestration/README.md create mode 100644 packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap diff --git a/packages/langchain/README.md b/packages/langchain/README.md index 869a99ce0..e3ad02198 100644 --- a/packages/langchain/README.md +++ b/packages/langchain/README.md @@ -56,152 +56,10 @@ Consequently, each deployment ID and resource group uniquely map to a combinatio ## Usage This package offers both chat and embedding clients, currently supporting Azure OpenAI. +Also supports the SAP Orchestration service. All clients comply with [LangChain's interface](https://js.langchain.com/docs/introduction). -### Client Initialization - -To initialize a client, provide the model name: - -```ts -import { - AzureOpenAiChatClient, - AzureOpenAiEmbeddingClient -} from '@sap-ai-sdk/langchain'; - -// For a chat client -const chatClient = new AzureOpenAiChatClient({ modelName: 'gpt-4o' }); -// For an embedding client -const embeddingClient = new AzureOpenAiEmbeddingClient({ modelName: 'gpt-4o' }); -``` - -In addition to the default parameters of the model vendor (e.g., OpenAI) and LangChain, additional parameters can be used to help narrow down the search for the desired model: - -```ts -const chatClient = new AzureOpenAiChatClient({ - modelName: 'gpt-4o', - modelVersion: '24-07-2021', - resourceGroup: 'my-resource-group' -}); -``` - -**Do not pass a `deployment ID` to initialize the client.** -For the LangChain model clients, initialization is done using the model name, model version and resource group. - -An important note is that LangChain clients by default attempt 6 retries with exponential backoff in case of a failure. -Especially in testing environments you might want to reduce this number to speed up the process: - -```ts -const embeddingClient = new AzureOpenAiEmbeddingClient({ - modelName: 'gpt-4o', - maxRetries: 0 -}); -``` - -#### Custom Destination - -When initializing the `AzureOpenAiChatClient` and `AzureOpenAiEmbeddingClient` clients, it is possible to provide a custom destination. -For example, when targeting a destination with the name `my-destination`, the following code can be used: - -```ts -const chatClient = new AzureOpenAiChatClient( - { - modelName: 'gpt-4o', - modelVersion: '24-07-2021', - resourceGroup: 'my-resource-group' - }, - { - destinationName: 'my-destination' - } -); -``` - -By default, the fetched destination is cached. -To disable caching, set the `useCache` parameter to `false` together with the `destinationName` parameter. - -### Chat Client - -The chat client allows you to interact with Azure OpenAI chat models, accessible via the generative AI hub of SAP AI Core. -To invoke the client, pass a prompt: - -```ts -const response = await chatClient.invoke("What's the capital of France?"); -``` - -#### Advanced Example with Templating and Output Parsing - -```ts -import { AzureOpenAiChatClient } from '@sap-ai-sdk/langchain'; -import { StringOutputParser } from '@langchain/core/output_parsers'; -import { ChatPromptTemplate } from '@langchain/core/prompts'; - -// initialize the client -const client = new AzureOpenAiChatClient({ modelName: 'gpt-35-turbo' }); - -// create a prompt template -const promptTemplate = ChatPromptTemplate.fromMessages([ - ['system', 'Answer the following in {language}:'], - ['user', '{text}'] -]); -// create an output parser -const parser = new StringOutputParser(); - -// chain together template, client, and parser -const llmChain = promptTemplate.pipe(client).pipe(parser); - -// invoke the chain -return llmChain.invoke({ - language: 'german', - text: 'What is the capital of France?' -}); -``` - -### Embedding Client - -Embedding clients allow embedding either text or document chunks (represented as arrays of strings). -While you can use them standalone, they are usually used in combination with other LangChain utilities, like a text splitter for preprocessing and a vector store for storage and retrieval of the relevant embeddings. -For a complete example how to implement RAG with our LangChain client, take a look at our [sample code](https://github.com/SAP/ai-sdk-js/blob/main/sample-code/src/langchain-azure-openai.ts). - -#### Embed Text - -```ts -const embeddedText = await embeddingClient.embedQuery( - 'Paris is the capital of France.' -); -``` - -#### Embed Document Chunks - -```ts -const embeddedDocuments = await embeddingClient.embedDocuments([ - 'Page 1: Paris is the capital of France.', - 'Page 2: It is a beautiful city.' -]); -``` - -#### Preprocess, embed, and store documents - -```ts -// Create a text splitter and split the document -const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 2000, - chunkOverlap: 200 -}); -const splits = await textSplitter.splitDocuments(docs); - -// Initialize the embedding client -const embeddingClient = new AzureOpenAiEmbeddingClient({ - modelName: 'text-embedding-ada-002' -}); - -// Create a vector store from the document -const vectorStore = await MemoryVectorStore.fromDocuments( - splits, - embeddingClient -); - -// Create a retriever for the vector store -const retriever = vectorStore.asRetriever(); -``` +### SAP Orchestration Service ## Local Testing diff --git a/packages/langchain/src/README.md b/packages/langchain/src/README.md new file mode 100644 index 000000000..624e0731d --- /dev/null +++ b/packages/langchain/src/README.md @@ -0,0 +1,206 @@ +# @sap-ai-sdk/langchain + +SAP Cloud SDK for AI is the official Software Development Kit (SDK) for **SAP AI Core**, **SAP Generative AI Hub**, and **Orchestration Service**. + +This package provides LangChain model clients built on top of the foundation model clients of the SAP Cloud SDK for AI. + +### Table of Contents + +- [Installation](#installation) +- [Prerequisites](#prerequisites) +- [Relationship between Models and Deployment ID](#relationship-between-models-and-deployment-id) +- [Usage](#usage) + - [Client Initialization](#client-initialization) + - [Chat Client](#chat-client) + - [Embedding Client](#embedding-client) +- [Local Testing](#local-testing) + +## Installation + +``` +$ npm install @sap-ai-sdk/langchain +``` + +## Prerequisites + +- [Enable the AI Core service in SAP BTP](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/initial-setup). +- Use the same `@langchain/core` version as the `@sap-ai-sdk/langchain` package, to see which langchain version this package is currently using, check our [package.json](./package.json). +- Configure the project with **Node.js v20 or higher** and **native ESM** support. +- Ensure a deployed OpenAI model is available in the SAP Generative AI Hub. + - Use the [`DeploymentApi`](https://github.com/SAP/ai-sdk-js/blob/main/packages/ai-api/README.md#create-a-deployment) from `@sap-ai-sdk/ai-api` [to deploy a model](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). + Alternatively, you can also create deployments using the [SAP AI Launchpad](https://help.sap.com/docs/sap-ai-core/generative-ai-hub/activate-generative-ai-hub-for-sap-ai-launchpad?locale=en-US&q=launchpad). + - Once deployment is complete, access the model via the `deploymentUrl`. + +> **Accessing the AI Core Service via the SDK** +> +> The SDK automatically retrieves the `AI Core` service credentials and resolves the access token needed for authentication. +> +> - In Cloud Foundry, it's accessed from the `VCAP_SERVICES` environment variable. +> - In Kubernetes / Kyma environments, you have to mount the service binding as a secret instead, for more information refer to [this documentation](https://www.npmjs.com/package/@sap/xsenv#usage-in-kubernetes). + +## Relationship between Models and Deployment ID + +SAP AI Core manages access to generative AI models through the global AI scenario `foundation-models`. +Creating a deployment for a model requires access to this scenario. + +Each model, model version, and resource group allows for a one-time deployment. +After deployment completion, the response includes a `deploymentUrl` and an `id`, which is the deployment ID. +For more information, see [here](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). + +[Resource groups](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/resource-groups?q=resource+group) represent a virtual collection of related resources within the scope of one SAP AI Core tenant. + +Consequently, each deployment ID and resource group uniquely map to a combination of model and model version within the `foundation-models` scenario. + +## Usage + +This sub-package offers a Azure OpenAI chat and embedding client. +Both clients comply with [LangChain's interface](https://js.langchain.com/docs/introduction). + +### Client Initialization + +To initialize a client, provide the model name: + +```ts +import { + AzureOpenAiChatClient, + AzureOpenAiEmbeddingClient +} from '@sap-ai-sdk/langchain'; + +// For a chat client +const chatClient = new AzureOpenAiChatClient({ modelName: 'gpt-4o' }); +// For an embedding client +const embeddingClient = new AzureOpenAiEmbeddingClient({ modelName: 'gpt-4o' }); +``` + +In addition to the default parameters of the model vendor (e.g., OpenAI) and LangChain, additional parameters can be used to help narrow down the search for the desired model: + +```ts +const chatClient = new AzureOpenAiChatClient({ + modelName: 'gpt-4o', + modelVersion: '24-07-2021', + resourceGroup: 'my-resource-group' +}); +``` + +**Do not pass a `deployment ID` to initialize the client.** +For the LangChain model clients, initialization is done using the model name, model version and resource group. + +An important note is that LangChain clients by default attempt 6 retries with exponential backoff in case of a failure. +Especially in testing environments you might want to reduce this number to speed up the process: + +```ts +const embeddingClient = new AzureOpenAiEmbeddingClient({ + modelName: 'gpt-4o', + maxRetries: 0 +}); +``` + +#### Custom Destination + +When initializing the `AzureOpenAiChatClient` and `AzureOpenAiEmbeddingClient` clients, it is possible to provide a custom destination. +For example, when targeting a destination with the name `my-destination`, the following code can be used: + +```ts +const chatClient = new AzureOpenAiChatClient( + { + modelName: 'gpt-4o', + modelVersion: '24-07-2021', + resourceGroup: 'my-resource-group' + }, + { + destinationName: 'my-destination' + } +); +``` + +By default, the fetched destination is cached. +To disable caching, set the `useCache` parameter to `false` together with the `destinationName` parameter. + +### Chat Client + +The chat client allows you to interact with Azure OpenAI chat models, accessible via the generative AI hub of SAP AI Core. +To invoke the client, pass a prompt: + +```ts +const response = await chatClient.invoke("What's the capital of France?"); +``` + +#### Advanced Example with Templating and Output Parsing + +```ts +import { AzureOpenAiChatClient } from '@sap-ai-sdk/langchain'; +import { StringOutputParser } from '@langchain/core/output_parsers'; +import { ChatPromptTemplate } from '@langchain/core/prompts'; + +// initialize the client +const client = new AzureOpenAiChatClient({ modelName: 'gpt-35-turbo' }); + +// create a prompt template +const promptTemplate = ChatPromptTemplate.fromMessages([ + ['system', 'Answer the following in {language}:'], + ['user', '{text}'] +]); +// create an output parser +const parser = new StringOutputParser(); + +// chain together template, client, and parser +const llmChain = promptTemplate.pipe(client).pipe(parser); + +// invoke the chain +return llmChain.invoke({ + language: 'german', + text: 'What is the capital of France?' +}); +``` + +### Embedding Client + +Embedding clients allow embedding either text or document chunks (represented as arrays of strings). +While you can use them standalone, they are usually used in combination with other LangChain utilities, like a text splitter for preprocessing and a vector store for storage and retrieval of the relevant embeddings. +For a complete example how to implement RAG with our LangChain client, take a look at our [sample code](https://github.com/SAP/ai-sdk-js/blob/main/sample-code/src/langchain-azure-openai.ts). + +#### Embed Text + +```ts +const embeddedText = await embeddingClient.embedQuery( + 'Paris is the capital of France.' +); +``` + +#### Embed Document Chunks + +```ts +const embeddedDocuments = await embeddingClient.embedDocuments([ + 'Page 1: Paris is the capital of France.', + 'Page 2: It is a beautiful city.' +]); +``` + +#### Preprocess, embed, and store documents + +```ts +// Create a text splitter and split the document +const textSplitter = new RecursiveCharacterTextSplitter({ + chunkSize: 2000, + chunkOverlap: 200 +}); +const splits = await textSplitter.splitDocuments(docs); + +// Initialize the embedding client +const embeddingClient = new AzureOpenAiEmbeddingClient({ + modelName: 'text-embedding-ada-002' +}); + +// Create a vector store from the document +const vectorStore = await MemoryVectorStore.fromDocuments( + splits, + embeddingClient +); + +// Create a retriever for the vector store +const retriever = vectorStore.asRetriever(); +``` + +## Local Testing + +For local testing instructions, refer to this [section](https://github.com/SAP/ai-sdk-js/blob/main/README.md#local-testing). diff --git a/packages/langchain/src/openai/README.md b/packages/langchain/src/openai/README.md new file mode 100644 index 000000000..869a99ce0 --- /dev/null +++ b/packages/langchain/src/openai/README.md @@ -0,0 +1,219 @@ +# @sap-ai-sdk/langchain + +SAP Cloud SDK for AI is the official Software Development Kit (SDK) for **SAP AI Core**, **SAP Generative AI Hub**, and **Orchestration Service**. + +This package provides LangChain model clients built on top of the foundation model clients of the SAP Cloud SDK for AI. + +### Table of Contents + +- [Installation](#installation) +- [Prerequisites](#prerequisites) +- [Relationship between Models and Deployment ID](#relationship-between-models-and-deployment-id) +- [Usage](#usage) + - [Client Initialization](#client-initialization) + - [Chat Client](#chat-client) + - [Embedding Client](#embedding-client) +- [Local Testing](#local-testing) +- [Support, Feedback, Contribution](#support-feedback-contribution) +- [License](#license) + +## Installation + +``` +$ npm install @sap-ai-sdk/langchain +``` + +## Prerequisites + +- [Enable the AI Core service in SAP BTP](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/initial-setup). +- Use the same `@langchain/core` version as the `@sap-ai-sdk/langchain` package, to see which langchain version this package is currently using, check our [package.json](./package.json). +- Configure the project with **Node.js v20 or higher** and **native ESM** support. +- Ensure a deployed OpenAI model is available in the SAP Generative AI Hub. + - Use the [`DeploymentApi`](https://github.com/SAP/ai-sdk-js/blob/main/packages/ai-api/README.md#create-a-deployment) from `@sap-ai-sdk/ai-api` [to deploy a model](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). + Alternatively, you can also create deployments using the [SAP AI Launchpad](https://help.sap.com/docs/sap-ai-core/generative-ai-hub/activate-generative-ai-hub-for-sap-ai-launchpad?locale=en-US&q=launchpad). + - Once deployment is complete, access the model via the `deploymentUrl`. + +> **Accessing the AI Core Service via the SDK** +> +> The SDK automatically retrieves the `AI Core` service credentials and resolves the access token needed for authentication. +> +> - In Cloud Foundry, it's accessed from the `VCAP_SERVICES` environment variable. +> - In Kubernetes / Kyma environments, you have to mount the service binding as a secret instead, for more information refer to [this documentation](https://www.npmjs.com/package/@sap/xsenv#usage-in-kubernetes). + +## Relationship between Models and Deployment ID + +SAP AI Core manages access to generative AI models through the global AI scenario `foundation-models`. +Creating a deployment for a model requires access to this scenario. + +Each model, model version, and resource group allows for a one-time deployment. +After deployment completion, the response includes a `deploymentUrl` and an `id`, which is the deployment ID. +For more information, see [here](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). + +[Resource groups](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/resource-groups?q=resource+group) represent a virtual collection of related resources within the scope of one SAP AI Core tenant. + +Consequently, each deployment ID and resource group uniquely map to a combination of model and model version within the `foundation-models` scenario. + +## Usage + +This package offers both chat and embedding clients, currently supporting Azure OpenAI. +All clients comply with [LangChain's interface](https://js.langchain.com/docs/introduction). + +### Client Initialization + +To initialize a client, provide the model name: + +```ts +import { + AzureOpenAiChatClient, + AzureOpenAiEmbeddingClient +} from '@sap-ai-sdk/langchain'; + +// For a chat client +const chatClient = new AzureOpenAiChatClient({ modelName: 'gpt-4o' }); +// For an embedding client +const embeddingClient = new AzureOpenAiEmbeddingClient({ modelName: 'gpt-4o' }); +``` + +In addition to the default parameters of the model vendor (e.g., OpenAI) and LangChain, additional parameters can be used to help narrow down the search for the desired model: + +```ts +const chatClient = new AzureOpenAiChatClient({ + modelName: 'gpt-4o', + modelVersion: '24-07-2021', + resourceGroup: 'my-resource-group' +}); +``` + +**Do not pass a `deployment ID` to initialize the client.** +For the LangChain model clients, initialization is done using the model name, model version and resource group. + +An important note is that LangChain clients by default attempt 6 retries with exponential backoff in case of a failure. +Especially in testing environments you might want to reduce this number to speed up the process: + +```ts +const embeddingClient = new AzureOpenAiEmbeddingClient({ + modelName: 'gpt-4o', + maxRetries: 0 +}); +``` + +#### Custom Destination + +When initializing the `AzureOpenAiChatClient` and `AzureOpenAiEmbeddingClient` clients, it is possible to provide a custom destination. +For example, when targeting a destination with the name `my-destination`, the following code can be used: + +```ts +const chatClient = new AzureOpenAiChatClient( + { + modelName: 'gpt-4o', + modelVersion: '24-07-2021', + resourceGroup: 'my-resource-group' + }, + { + destinationName: 'my-destination' + } +); +``` + +By default, the fetched destination is cached. +To disable caching, set the `useCache` parameter to `false` together with the `destinationName` parameter. + +### Chat Client + +The chat client allows you to interact with Azure OpenAI chat models, accessible via the generative AI hub of SAP AI Core. +To invoke the client, pass a prompt: + +```ts +const response = await chatClient.invoke("What's the capital of France?"); +``` + +#### Advanced Example with Templating and Output Parsing + +```ts +import { AzureOpenAiChatClient } from '@sap-ai-sdk/langchain'; +import { StringOutputParser } from '@langchain/core/output_parsers'; +import { ChatPromptTemplate } from '@langchain/core/prompts'; + +// initialize the client +const client = new AzureOpenAiChatClient({ modelName: 'gpt-35-turbo' }); + +// create a prompt template +const promptTemplate = ChatPromptTemplate.fromMessages([ + ['system', 'Answer the following in {language}:'], + ['user', '{text}'] +]); +// create an output parser +const parser = new StringOutputParser(); + +// chain together template, client, and parser +const llmChain = promptTemplate.pipe(client).pipe(parser); + +// invoke the chain +return llmChain.invoke({ + language: 'german', + text: 'What is the capital of France?' +}); +``` + +### Embedding Client + +Embedding clients allow embedding either text or document chunks (represented as arrays of strings). +While you can use them standalone, they are usually used in combination with other LangChain utilities, like a text splitter for preprocessing and a vector store for storage and retrieval of the relevant embeddings. +For a complete example how to implement RAG with our LangChain client, take a look at our [sample code](https://github.com/SAP/ai-sdk-js/blob/main/sample-code/src/langchain-azure-openai.ts). + +#### Embed Text + +```ts +const embeddedText = await embeddingClient.embedQuery( + 'Paris is the capital of France.' +); +``` + +#### Embed Document Chunks + +```ts +const embeddedDocuments = await embeddingClient.embedDocuments([ + 'Page 1: Paris is the capital of France.', + 'Page 2: It is a beautiful city.' +]); +``` + +#### Preprocess, embed, and store documents + +```ts +// Create a text splitter and split the document +const textSplitter = new RecursiveCharacterTextSplitter({ + chunkSize: 2000, + chunkOverlap: 200 +}); +const splits = await textSplitter.splitDocuments(docs); + +// Initialize the embedding client +const embeddingClient = new AzureOpenAiEmbeddingClient({ + modelName: 'text-embedding-ada-002' +}); + +// Create a vector store from the document +const vectorStore = await MemoryVectorStore.fromDocuments( + splits, + embeddingClient +); + +// Create a retriever for the vector store +const retriever = vectorStore.asRetriever(); +``` + +## Local Testing + +For local testing instructions, refer to this [section](https://github.com/SAP/ai-sdk-js/blob/main/README.md#local-testing). + +## Support, Feedback, Contribution + +This project is open to feature requests, bug reports and questions via [GitHub issues](https://github.com/SAP/ai-sdk-js/issues). + +Contribution and feedback are encouraged and always welcome. +For more information about how to contribute, the project structure, as well as additional contribution information, see our [Contribution Guidelines](https://github.com/SAP/ai-sdk-js/blob/main/CONTRIBUTING.md). + +## License + +The SAP Cloud SDK for AI is released under the [Apache License Version 2.0.](http://www.apache.org/licenses/). diff --git a/packages/langchain/src/orchestration/README.md b/packages/langchain/src/orchestration/README.md new file mode 100644 index 000000000..f7b2650b7 --- /dev/null +++ b/packages/langchain/src/orchestration/README.md @@ -0,0 +1,219 @@ +# @sap-ai-sdk/langchain + +SAP Cloud SDK for AI is the official Software Development Kit (SDK) for **SAP AI Core**, **SAP Generative AI Hub**, and **Orchestration Service**. + +This package provides LangChain model clients built on top of the foundation model clients of the SAP Cloud SDK for AI. + +### Table of Contents + +- [Installation](#installation) +- [Prerequisites](#prerequisites) +- [Relationship between Models and Deployment ID](#relationship-between-models-and-deployment-id) +- [Usage](#usage) + - [Client Initialization](#client-initialization) + - [Chat Client](#chat-client) + - [Embedding Client](#embedding-client) +- [Local Testing](#local-testing) +- [Support, Feedback, Contribution](#support-feedback-contribution) +- [License](#license) + +## Installation + +``` +$ npm install @sap-ai-sdk/langchain +``` + +## Prerequisites + +- [Enable the AI Core service in SAP BTP](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/initial-setup). +- Use the same `@langchain/core` version as the `@sap-ai-sdk/langchain` package, to see which langchain version this package is currently using, check our [package.json](./package.json). +- Configure the project with **Node.js v20 or higher** and **native ESM** support. +- Ensure a deployed OpenAI model is available in the SAP Generative AI Hub. + - Use the [`DeploymentApi`](https://github.com/SAP/ai-sdk-js/blob/main/packages/ai-api/README.md#create-a-deployment) from `@sap-ai-sdk/ai-api` [to deploy a model](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). + Alternatively, you can also create deployments using the [SAP AI Launchpad](https://help.sap.com/docs/sap-ai-core/generative-ai-hub/activate-generative-ai-hub-for-sap-ai-launchpad?locale=en-US&q=launchpad). + - Once deployment is complete, access the model via the `deploymentUrl`. + +> **Accessing the AI Core Service via the SDK** +> +> The SDK automatically retrieves the `AI Core` service credentials and resolves the access token needed for authentication. +> +> - In Cloud Foundry, it's accessed from the `VCAP_SERVICES` environment variable. +> - In Kubernetes / Kyma environments, you have to mount the service binding as a secret instead, for more information refer to [this documentation](https://www.npmjs.com/package/@sap/xsenv#usage-in-kubernetes). + +## Relationship between Models and Deployment ID + +SAP AI Core manages access to generative AI models through the global AI scenario `foundation-models`. +Creating a deployment for a model requires access to this scenario. + +Each model, model version, and resource group allows for a one-time deployment. +After deployment completion, the response includes a `deploymentUrl` and an `id`, which is the deployment ID. +For more information, see [here](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). + +[Resource groups](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/resource-groups?q=resource+group) represent a virtual collection of related resources within the scope of one SAP AI Core tenant. + +Consequently, each deployment ID and resource group uniquely map to a combination of model and model version within the `foundation-models` scenario. + +## Usage + +This package offers both chat and embedding clients, currently supporting Azure OpenAI. +All clients comply with [LangChain's interface](https://js.langchain.com/docs/introduction). + +### Client Initialization + +To initialize a client, provide the model name: + +```ts +import { + AzureOpenAiChatClient, + AzureOpenAiEmbeddingClient +} from '@sap-ai-sdk/langchain'; + +// For a chat client +const chatClient = new AzureOpenAiChatClient({ modelName: 'gpt-4o' }); +// For an embedding client +const embeddingClient = new AzureOpenAiEmbeddingClient({ modelName: 'gpt-4o' }); +``` + +In addition to the default parameters of the model vendor (e.g., OpenAI) and LangChain, additional parameters can be used to help narrow down the search for the desired model: + +```ts +const chatClient = new AzureOpenAiChatClient({ + modelName: 'gpt-4o', + modelVersion: '24-07-2021', + resourceGroup: 'my-resource-group' +}); +``` + +**Do not pass a `deployment ID` to initialize the client.** +For the LangChain model clients, initialization is done using the model name, model version and resource group. + +An important note is that LangChain clients by default attempt 6 retries with exponential backoff in case of a failure. +Especially in testing environments you might want to reduce this number to speed up the process: + +```ts +const embeddingClient = new OrchestrationClient({ + modelName: 'gpt-4o', + maxRetries: 0 +}); +``` + +#### Custom Destination + +When initializing the `AzureOpenAiChatClient` and `AzureOpenAiEmbeddingClient` clients, it is possible to provide a custom destination. +For example, when targeting a destination with the name `my-destination`, the following code can be used: + +```ts +const chatClient = new AzureOpenAiChatClient( + { + modelName: 'gpt-4o', + modelVersion: '24-07-2021', + resourceGroup: 'my-resource-group' + }, + { + destinationName: 'my-destination' + } +); +``` + +By default, the fetched destination is cached. +To disable caching, set the `useCache` parameter to `false` together with the `destinationName` parameter. + +### Chat Client + +The chat client allows you to interact with Azure OpenAI chat models, accessible via the generative AI hub of SAP AI Core. +To invoke the client, pass a prompt: + +```ts +const response = await chatClient.invoke("What's the capital of France?"); +``` + +#### Advanced Example with Templating and Output Parsing + +```ts +import { AzureOpenAiChatClient } from '@sap-ai-sdk/langchain'; +import { StringOutputParser } from '@langchain/core/output_parsers'; +import { ChatPromptTemplate } from '@langchain/core/prompts'; + +// initialize the client +const client = new AzureOpenAiChatClient({ modelName: 'gpt-35-turbo' }); + +// create a prompt template +const promptTemplate = ChatPromptTemplate.fromMessages([ + ['system', 'Answer the following in {language}:'], + ['user', '{text}'] +]); +// create an output parser +const parser = new StringOutputParser(); + +// chain together template, client, and parser +const llmChain = promptTemplate.pipe(client).pipe(parser); + +// invoke the chain +return llmChain.invoke({ + language: 'german', + text: 'What is the capital of France?' +}); +``` + +### Embedding Client + +Embedding clients allow embedding either text or document chunks (represented as arrays of strings). +While you can use them standalone, they are usually used in combination with other LangChain utilities, like a text splitter for preprocessing and a vector store for storage and retrieval of the relevant embeddings. +For a complete example how to implement RAG with our LangChain client, take a look at our [sample code](https://github.com/SAP/ai-sdk-js/blob/main/sample-code/src/langchain-azure-openai.ts). + +#### Embed Text + +```ts +const embeddedText = await embeddingClient.embedQuery( + 'Paris is the capital of France.' +); +``` + +#### Embed Document Chunks + +```ts +const embeddedDocuments = await embeddingClient.embedDocuments([ + 'Page 1: Paris is the capital of France.', + 'Page 2: It is a beautiful city.' +]); +``` + +#### Preprocess, embed, and store documents + +```ts +// Create a text splitter and split the document +const textSplitter = new RecursiveCharacterTextSplitter({ + chunkSize: 2000, + chunkOverlap: 200 +}); +const splits = await textSplitter.splitDocuments(docs); + +// Initialize the embedding client +const embeddingClient = new AzureOpenAiEmbeddingClient({ + modelName: 'text-embedding-ada-002' +}); + +// Create a vector store from the document +const vectorStore = await MemoryVectorStore.fromDocuments( + splits, + embeddingClient +); + +// Create a retriever for the vector store +const retriever = vectorStore.asRetriever(); +``` + +## Local Testing + +For local testing instructions, refer to this [section](https://github.com/SAP/ai-sdk-js/blob/main/README.md#local-testing). + +## Support, Feedback, Contribution + +This project is open to feature requests, bug reports and questions via [GitHub issues](https://github.com/SAP/ai-sdk-js/issues). + +Contribution and feedback are encouraged and always welcome. +For more information about how to contribute, the project structure, as well as additional contribution information, see our [Contribution Guidelines](https://github.com/SAP/ai-sdk-js/blob/main/CONTRIBUTING.md). + +## License + +The SAP Cloud SDK for AI is released under the [Apache License Version 2.0.](http://www.apache.org/licenses/). diff --git a/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap b/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap new file mode 100644 index 000000000..79e6e9b2a --- /dev/null +++ b/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap @@ -0,0 +1,81 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`orchestration service client handles error response when maxRetries is smaller than retry configuration 1`] = `"Request failed with status code 500"`; + +exports[`orchestration service client handles requests with longer timeout value 1`] = ` +{ + "id": [ + "langchain_core", + "messages", + "OrchestrationMessage", + ], + "kwargs": { + "additional_kwargs": { + "finish_reason": "stop", + "function_call": undefined, + "index": 0, + "tool_calls": undefined, + }, + "content": "Hello! How can I assist you today?", + "invalid_tool_calls": [], + "response_metadata": { + "created": 172, + "finish_reason": "stop", + "function_call": undefined, + "id": "orchestration-id", + "index": 0, + "model": "gpt-35-turbo", + "object": "chat.completion", + "system_fingerprint": undefined, + "tokenUsage": { + "completionTokens": 9, + "promptTokens": 9, + "totalTokens": 18, + }, + "tool_calls": undefined, + }, + "tool_calls": [], + }, + "lc": 1, + "type": "constructor", +} +`; + +exports[`orchestration service client handles successful response when maxRetries equals retry configuration 1`] = ` +{ + "id": [ + "langchain_core", + "messages", + "OrchestrationMessage", + ], + "kwargs": { + "additional_kwargs": { + "finish_reason": "stop", + "function_call": undefined, + "index": 0, + "tool_calls": undefined, + }, + "content": "Hello! How can I assist you today?", + "invalid_tool_calls": [], + "response_metadata": { + "created": 172, + "finish_reason": "stop", + "function_call": undefined, + "id": "orchestration-id", + "index": 0, + "model": "gpt-35-turbo", + "object": "chat.completion", + "system_fingerprint": undefined, + "tokenUsage": { + "completionTokens": 9, + "promptTokens": 9, + "totalTokens": 18, + }, + "tool_calls": undefined, + }, + "tool_calls": [], + }, + "lc": 1, + "type": "constructor", +} +`; diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 577e7b041..791e19ead 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -1,5 +1,6 @@ -import nock from 'nock'; import { constructCompletionPostRequest } from '@sap-ai-sdk/orchestration/internal.js'; +import { jest } from '@jest/globals'; +import nock from 'nock'; import { mockClientCredentialsGrantCall, mockDeploymentsList, @@ -12,32 +13,27 @@ import type { OrchestrationModuleConfig } from '@sap-ai-sdk/orchestration'; +jest.setTimeout(30000); + describe('orchestration service client', () => { - beforeEach(() => { + let mockResponse: CompletionPostResponse; + beforeEach(async () => { mockClientCredentialsGrantCall(); mockDeploymentsList({ scenarioId: 'orchestration' }, { id: '1234' }); + mockResponse = await parseMockResponse( + 'orchestration', + 'orchestration-chat-completion-success-response.json' + ); }); afterEach(() => { nock.cleanAll(); }); - it('calls chatCompletion with minimum configuration', async () => { - const config: OrchestrationModuleConfig = { - llm: { - model_name: 'gpt-35-turbo-16k', - model_params: { max_tokens: 50, temperature: 0.1 } - }, - templating: { - template: [{ role: 'user', content: 'Hello!' }] - } - }; - - const mockResponse = await parseMockResponse( - 'orchestration', - 'orchestration-chat-completion-success-response.json' - ); - + function mockInferenceWithResilience(resilience: { + retry?: number; + delay?: number; + }) { mockInference( { data: constructCompletionPostRequest(config, { messagesHistory: [] }) @@ -49,50 +45,62 @@ describe('orchestration service client', () => { { url: 'inference/deployments/1234/completion' }, - { retry: 3 } + resilience ); - const response = await new OrchestrationClient(config, { - maxRetries: 5 - }).invoke([], { timeout: 1 }); - - // expect(nock.isDone()).toBe(true); - expect(response).toMatchInlineSnapshot(` - { - "id": [ - "langchain_core", - "messages", - "OrchestrationMessage", - ], - "kwargs": { - "additional_kwargs": { - "finish_reason": "stop", - "function_call": undefined, - "index": 0, - "tool_calls": undefined, - }, - "content": "Hello! How can I assist you today?", - "invalid_tool_calls": [], - "response_metadata": { - "created": 172, - "finish_reason": "stop", - "function_call": undefined, - "id": "orchestration-id", - "index": 0, - "model": "gpt-35-turbo", - "object": "chat.completion", - "system_fingerprint": undefined, - "tokenUsage": { - "completionTokens": 9, - "promptTokens": 9, - "totalTokens": 18, - }, - "tool_calls": undefined, - }, - "tool_calls": [], - }, - "lc": 1, - "type": "constructor", - } - `); + } + + const config: OrchestrationModuleConfig = { + llm: { + model_name: 'gpt-35-turbo-16k', + model_params: { max_tokens: 50, temperature: 0.1 } + }, + templating: { + template: [{ role: 'user', content: 'Hello!' }] + } + }; + + it('returns successful response when maxRetries equals retry configuration', async () => { + mockInferenceWithResilience({ retry: 2 }); + + const client = new OrchestrationClient(config, { + maxRetries: 2 + }); + + expect(await client.invoke([])).toMatchSnapshot(); + }); + + it('throws error response when maxRetries is smaller than required retries', async () => { + mockInferenceWithResilience({ retry: 2 }); + + const client = new OrchestrationClient(config, { + maxRetries: 1 + }); + + await expect(client.invoke([])).rejects.toThrowErrorMatchingInlineSnapshot( + '"Request failed with status code 500"' + ); + }); + + it('throws when delay exceeds timeout', async () => { + mockInferenceWithResilience({ delay: 2000 }); + + const client = new OrchestrationClient(config); + + const response = client.invoke([], { timeout: 1000 }); + + await expect(response).rejects.toThrow( + expect.objectContaining({ + stack: expect.stringMatching(/Timeout/) + }) + ); + }); + + it('returns successful response when timeout is bigger than delay', async () => { + mockInferenceWithResilience({ delay: 2000 }); + + const client = new OrchestrationClient(config); + + const response = await client.invoke([], { timeout: 3000 }); + expect(response).toMatchSnapshot(); }); }); diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 678e14a80..81ec87fdb 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -1,7 +1,5 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models'; import { OrchestrationClient as OrchestrationClientBase } from '@sap-ai-sdk/orchestration'; -import { AsyncCaller } from '@langchain/core/utils/async_caller'; -import { resilience } from '@sap-cloud-sdk/resilience'; import { isTemplate, mapLangchainMessagesToOrchestrationMessages, @@ -9,7 +7,6 @@ import { } from './util.js'; import type { BaseLanguageModelInput } from '@langchain/core/language_models/base'; import type { Runnable, RunnableLike } from '@langchain/core/runnables'; -import type { CustomRequestConfig } from '@sap-ai-sdk/core'; import type { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; import type { ChatResult } from '@langchain/core/outputs'; import type { OrchestrationModuleConfig } from '@sap-ai-sdk/orchestration'; @@ -66,22 +63,12 @@ export class OrchestrationClient extends BaseChatModel< options: typeof this.ParsedCallOptions, runManager?: CallbackManagerForLLMRun ): Promise { - let caller = this.caller; - if (options.maxConcurrency) { - const { maxConcurrency, maxRetries, onFailedAttempt } = - this.langchainOptions; - caller = new AsyncCaller({ - maxConcurrency: maxConcurrency ?? options.maxConcurrency, - maxRetries, - onFailedAttempt - }); - } - const res = await caller.callWithOptions( + const res = await this.caller.callWithOptions( { signal: options.signal }, () => { - const { inputParams } = options; + const { inputParams, customRequestConfig } = options; const mergedOrchestrationConfig = this.mergeOrchestrationConfig(options); const orchestrationClient = new OrchestrationClientBase( @@ -91,10 +78,6 @@ export class OrchestrationClient extends BaseChatModel< ); const messagesHistory = mapLangchainMessagesToOrchestrationMessages(messages); - const customRequestConfig: CustomRequestConfig = { - ...options.customRequestConfig, - middleware: resilience({ timeout: options.timeout }) - }; return orchestrationClient.chatCompletion( { messagesHistory, diff --git a/packages/langchain/src/orchestration/types.ts b/packages/langchain/src/orchestration/types.ts index ece952e81..17347ecc2 100644 --- a/packages/langchain/src/orchestration/types.ts +++ b/packages/langchain/src/orchestration/types.ts @@ -7,7 +7,7 @@ import type { CustomRequestConfig } from '@sap-ai-sdk/core'; */ export type OrchestrationCallOptions = Pick< BaseChatModelCallOptions, - 'stop' | 'signal' | 'maxConcurrency' | 'timeout' + 'stop' | 'signal' | 'timeout' > & { customRequestConfig?: CustomRequestConfig; tools?: Template['tools']; diff --git a/test-util/mock-http.ts b/test-util/mock-http.ts index ce8d40ac5..77409057e 100644 --- a/test-util/mock-http.ts +++ b/test-util/mock-http.ts @@ -116,11 +116,15 @@ export function mockInference( let interceptor = scope.post(`/v2/${url}`, request.data).query(apiVersion ? { 'api-version': apiVersion } : {}); if (resilienceOptions?.retry) { - interceptor.times(resilienceOptions.retry).reply(500); + interceptor = interceptor.times(resilienceOptions.retry); + if(resilienceOptions.delay) { + interceptor = interceptor.delay(resilienceOptions.delay); + } + interceptor.reply(500); interceptor = scope.post(`/v2/${url}`, request.data).query(apiVersion ? { 'api-version': apiVersion } : {}); } - if (resilienceOptions?.delay) { + if (!resilienceOptions?.retry && resilienceOptions?.delay) { interceptor = interceptor.delay(resilienceOptions.delay); } From 2e811be3ea8360f5ef9e8c484ed82d0bec51172f Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Fri, 28 Feb 2025 15:59:16 +0100 Subject: [PATCH 12/23] update snapshot fiel --- .../src/orchestration/__snapshots__/client.test.ts.snap | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap b/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap index 79e6e9b2a..a2e3efc8d 100644 --- a/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap +++ b/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap @@ -1,8 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`orchestration service client handles error response when maxRetries is smaller than retry configuration 1`] = `"Request failed with status code 500"`; - -exports[`orchestration service client handles requests with longer timeout value 1`] = ` +exports[`orchestration service client returns successful response when maxRetries equals retry configuration 1`] = ` { "id": [ "langchain_core", @@ -41,7 +39,7 @@ exports[`orchestration service client handles requests with longer timeout value } `; -exports[`orchestration service client handles successful response when maxRetries equals retry configuration 1`] = ` +exports[`orchestration service client returns successful response when timeout is bigger than delay 1`] = ` { "id": [ "langchain_core", From 144242629b085e67558e078ef4600531b9caf972 Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Fri, 28 Feb 2025 16:11:12 +0100 Subject: [PATCH 13/23] add changeset --- .changeset/clear-suns-sleep.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changeset/clear-suns-sleep.md diff --git a/.changeset/clear-suns-sleep.md b/.changeset/clear-suns-sleep.md new file mode 100644 index 000000000..163ccdb56 --- /dev/null +++ b/.changeset/clear-suns-sleep.md @@ -0,0 +1,10 @@ +--- +'@sap-ai-sdk/orchestration': minor +'@sap-ai-sdk/langchain': minor +'@sap-ai-sdk/smoke-tests': minor +'@sap-ai-sdk/e2e-tests': minor +'@sap-ai-sdk/sample-code': minor +'@sap-ai-sdk/sample-cap': minor +--- + +[New Functionality] Add LangChain Orchestration client. From b3bce6d28bd7c04a63646fa8e3a358eb9a9804fd Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Fri, 28 Feb 2025 16:52:09 +0100 Subject: [PATCH 14/23] update readmes and types --- .changeset/clear-suns-sleep.md | 5 - packages/langchain/README.md | 14 +- packages/langchain/src/README.md | 206 ------------------ packages/langchain/src/openai/README.md | 4 +- .../langchain/src/orchestration/README.md | 48 ---- packages/langchain/src/orchestration/types.ts | 2 +- sample-code/README.md | 4 +- 7 files changed, 14 insertions(+), 269 deletions(-) delete mode 100644 packages/langchain/src/README.md diff --git a/.changeset/clear-suns-sleep.md b/.changeset/clear-suns-sleep.md index 163ccdb56..ad2cee685 100644 --- a/.changeset/clear-suns-sleep.md +++ b/.changeset/clear-suns-sleep.md @@ -1,10 +1,5 @@ --- -'@sap-ai-sdk/orchestration': minor '@sap-ai-sdk/langchain': minor -'@sap-ai-sdk/smoke-tests': minor -'@sap-ai-sdk/e2e-tests': minor -'@sap-ai-sdk/sample-code': minor -'@sap-ai-sdk/sample-cap': minor --- [New Functionality] Add LangChain Orchestration client. diff --git a/packages/langchain/README.md b/packages/langchain/README.md index e3ad02198..429ff5479 100644 --- a/packages/langchain/README.md +++ b/packages/langchain/README.md @@ -10,9 +10,8 @@ This package provides LangChain model clients built on top of the foundation mod - [Prerequisites](#prerequisites) - [Relationship between Models and Deployment ID](#relationship-between-models-and-deployment-id) - [Usage](#usage) - - [Client Initialization](#client-initialization) - - [Chat Client](#chat-client) - - [Embedding Client](#embedding-client) + - [SAP Orchestration Service](#sap-orchestration-service) + - [Azure OpenAI](#azure-openai) - [Local Testing](#local-testing) - [Support, Feedback, Contribution](#support-feedback-contribution) - [License](#license) @@ -55,12 +54,17 @@ Consequently, each deployment ID and resource group uniquely map to a combinatio ## Usage -This package offers both chat and embedding clients, currently supporting Azure OpenAI. -Also supports the SAP Orchestration service. +This package offers LangChain clients for Azure OpenAI as well as the SAP orchestration service All clients comply with [LangChain's interface](https://js.langchain.com/docs/introduction). ### SAP Orchestration Service +For details on the orchestration client, refer to this [document](./src/orchestration/README.md). + +### Azure OpenAI + +For details on the Azure OpenAI clients, refer to this [document](./src/openai/README.md). + ## Local Testing For local testing instructions, refer to this [section](https://github.com/SAP/ai-sdk-js/blob/main/README.md#local-testing). diff --git a/packages/langchain/src/README.md b/packages/langchain/src/README.md deleted file mode 100644 index 624e0731d..000000000 --- a/packages/langchain/src/README.md +++ /dev/null @@ -1,206 +0,0 @@ -# @sap-ai-sdk/langchain - -SAP Cloud SDK for AI is the official Software Development Kit (SDK) for **SAP AI Core**, **SAP Generative AI Hub**, and **Orchestration Service**. - -This package provides LangChain model clients built on top of the foundation model clients of the SAP Cloud SDK for AI. - -### Table of Contents - -- [Installation](#installation) -- [Prerequisites](#prerequisites) -- [Relationship between Models and Deployment ID](#relationship-between-models-and-deployment-id) -- [Usage](#usage) - - [Client Initialization](#client-initialization) - - [Chat Client](#chat-client) - - [Embedding Client](#embedding-client) -- [Local Testing](#local-testing) - -## Installation - -``` -$ npm install @sap-ai-sdk/langchain -``` - -## Prerequisites - -- [Enable the AI Core service in SAP BTP](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/initial-setup). -- Use the same `@langchain/core` version as the `@sap-ai-sdk/langchain` package, to see which langchain version this package is currently using, check our [package.json](./package.json). -- Configure the project with **Node.js v20 or higher** and **native ESM** support. -- Ensure a deployed OpenAI model is available in the SAP Generative AI Hub. - - Use the [`DeploymentApi`](https://github.com/SAP/ai-sdk-js/blob/main/packages/ai-api/README.md#create-a-deployment) from `@sap-ai-sdk/ai-api` [to deploy a model](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). - Alternatively, you can also create deployments using the [SAP AI Launchpad](https://help.sap.com/docs/sap-ai-core/generative-ai-hub/activate-generative-ai-hub-for-sap-ai-launchpad?locale=en-US&q=launchpad). - - Once deployment is complete, access the model via the `deploymentUrl`. - -> **Accessing the AI Core Service via the SDK** -> -> The SDK automatically retrieves the `AI Core` service credentials and resolves the access token needed for authentication. -> -> - In Cloud Foundry, it's accessed from the `VCAP_SERVICES` environment variable. -> - In Kubernetes / Kyma environments, you have to mount the service binding as a secret instead, for more information refer to [this documentation](https://www.npmjs.com/package/@sap/xsenv#usage-in-kubernetes). - -## Relationship between Models and Deployment ID - -SAP AI Core manages access to generative AI models through the global AI scenario `foundation-models`. -Creating a deployment for a model requires access to this scenario. - -Each model, model version, and resource group allows for a one-time deployment. -After deployment completion, the response includes a `deploymentUrl` and an `id`, which is the deployment ID. -For more information, see [here](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). - -[Resource groups](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/resource-groups?q=resource+group) represent a virtual collection of related resources within the scope of one SAP AI Core tenant. - -Consequently, each deployment ID and resource group uniquely map to a combination of model and model version within the `foundation-models` scenario. - -## Usage - -This sub-package offers a Azure OpenAI chat and embedding client. -Both clients comply with [LangChain's interface](https://js.langchain.com/docs/introduction). - -### Client Initialization - -To initialize a client, provide the model name: - -```ts -import { - AzureOpenAiChatClient, - AzureOpenAiEmbeddingClient -} from '@sap-ai-sdk/langchain'; - -// For a chat client -const chatClient = new AzureOpenAiChatClient({ modelName: 'gpt-4o' }); -// For an embedding client -const embeddingClient = new AzureOpenAiEmbeddingClient({ modelName: 'gpt-4o' }); -``` - -In addition to the default parameters of the model vendor (e.g., OpenAI) and LangChain, additional parameters can be used to help narrow down the search for the desired model: - -```ts -const chatClient = new AzureOpenAiChatClient({ - modelName: 'gpt-4o', - modelVersion: '24-07-2021', - resourceGroup: 'my-resource-group' -}); -``` - -**Do not pass a `deployment ID` to initialize the client.** -For the LangChain model clients, initialization is done using the model name, model version and resource group. - -An important note is that LangChain clients by default attempt 6 retries with exponential backoff in case of a failure. -Especially in testing environments you might want to reduce this number to speed up the process: - -```ts -const embeddingClient = new AzureOpenAiEmbeddingClient({ - modelName: 'gpt-4o', - maxRetries: 0 -}); -``` - -#### Custom Destination - -When initializing the `AzureOpenAiChatClient` and `AzureOpenAiEmbeddingClient` clients, it is possible to provide a custom destination. -For example, when targeting a destination with the name `my-destination`, the following code can be used: - -```ts -const chatClient = new AzureOpenAiChatClient( - { - modelName: 'gpt-4o', - modelVersion: '24-07-2021', - resourceGroup: 'my-resource-group' - }, - { - destinationName: 'my-destination' - } -); -``` - -By default, the fetched destination is cached. -To disable caching, set the `useCache` parameter to `false` together with the `destinationName` parameter. - -### Chat Client - -The chat client allows you to interact with Azure OpenAI chat models, accessible via the generative AI hub of SAP AI Core. -To invoke the client, pass a prompt: - -```ts -const response = await chatClient.invoke("What's the capital of France?"); -``` - -#### Advanced Example with Templating and Output Parsing - -```ts -import { AzureOpenAiChatClient } from '@sap-ai-sdk/langchain'; -import { StringOutputParser } from '@langchain/core/output_parsers'; -import { ChatPromptTemplate } from '@langchain/core/prompts'; - -// initialize the client -const client = new AzureOpenAiChatClient({ modelName: 'gpt-35-turbo' }); - -// create a prompt template -const promptTemplate = ChatPromptTemplate.fromMessages([ - ['system', 'Answer the following in {language}:'], - ['user', '{text}'] -]); -// create an output parser -const parser = new StringOutputParser(); - -// chain together template, client, and parser -const llmChain = promptTemplate.pipe(client).pipe(parser); - -// invoke the chain -return llmChain.invoke({ - language: 'german', - text: 'What is the capital of France?' -}); -``` - -### Embedding Client - -Embedding clients allow embedding either text or document chunks (represented as arrays of strings). -While you can use them standalone, they are usually used in combination with other LangChain utilities, like a text splitter for preprocessing and a vector store for storage and retrieval of the relevant embeddings. -For a complete example how to implement RAG with our LangChain client, take a look at our [sample code](https://github.com/SAP/ai-sdk-js/blob/main/sample-code/src/langchain-azure-openai.ts). - -#### Embed Text - -```ts -const embeddedText = await embeddingClient.embedQuery( - 'Paris is the capital of France.' -); -``` - -#### Embed Document Chunks - -```ts -const embeddedDocuments = await embeddingClient.embedDocuments([ - 'Page 1: Paris is the capital of France.', - 'Page 2: It is a beautiful city.' -]); -``` - -#### Preprocess, embed, and store documents - -```ts -// Create a text splitter and split the document -const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 2000, - chunkOverlap: 200 -}); -const splits = await textSplitter.splitDocuments(docs); - -// Initialize the embedding client -const embeddingClient = new AzureOpenAiEmbeddingClient({ - modelName: 'text-embedding-ada-002' -}); - -// Create a vector store from the document -const vectorStore = await MemoryVectorStore.fromDocuments( - splits, - embeddingClient -); - -// Create a retriever for the vector store -const retriever = vectorStore.asRetriever(); -``` - -## Local Testing - -For local testing instructions, refer to this [section](https://github.com/SAP/ai-sdk-js/blob/main/README.md#local-testing). diff --git a/packages/langchain/src/openai/README.md b/packages/langchain/src/openai/README.md index 869a99ce0..cc686c109 100644 --- a/packages/langchain/src/openai/README.md +++ b/packages/langchain/src/openai/README.md @@ -55,7 +55,7 @@ Consequently, each deployment ID and resource group uniquely map to a combinatio ## Usage -This package offers both chat and embedding clients, currently supporting Azure OpenAI. +This package offers both chat and embedding clients for Azure OpenAI. All clients comply with [LangChain's interface](https://js.langchain.com/docs/introduction). ### Client Initialization @@ -74,7 +74,7 @@ const chatClient = new AzureOpenAiChatClient({ modelName: 'gpt-4o' }); const embeddingClient = new AzureOpenAiEmbeddingClient({ modelName: 'gpt-4o' }); ``` -In addition to the default parameters of the model vendor (e.g., OpenAI) and LangChain, additional parameters can be used to help narrow down the search for the desired model: +In addition to the default parameters of Azure OpenAI and LangChain, additional parameters can be used to help narrow down the search for the desired model: ```ts const chatClient = new AzureOpenAiChatClient({ diff --git a/packages/langchain/src/orchestration/README.md b/packages/langchain/src/orchestration/README.md index f7b2650b7..0a94c9f73 100644 --- a/packages/langchain/src/orchestration/README.md +++ b/packages/langchain/src/orchestration/README.md @@ -155,54 +155,6 @@ return llmChain.invoke({ }); ``` -### Embedding Client - -Embedding clients allow embedding either text or document chunks (represented as arrays of strings). -While you can use them standalone, they are usually used in combination with other LangChain utilities, like a text splitter for preprocessing and a vector store for storage and retrieval of the relevant embeddings. -For a complete example how to implement RAG with our LangChain client, take a look at our [sample code](https://github.com/SAP/ai-sdk-js/blob/main/sample-code/src/langchain-azure-openai.ts). - -#### Embed Text - -```ts -const embeddedText = await embeddingClient.embedQuery( - 'Paris is the capital of France.' -); -``` - -#### Embed Document Chunks - -```ts -const embeddedDocuments = await embeddingClient.embedDocuments([ - 'Page 1: Paris is the capital of France.', - 'Page 2: It is a beautiful city.' -]); -``` - -#### Preprocess, embed, and store documents - -```ts -// Create a text splitter and split the document -const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 2000, - chunkOverlap: 200 -}); -const splits = await textSplitter.splitDocuments(docs); - -// Initialize the embedding client -const embeddingClient = new AzureOpenAiEmbeddingClient({ - modelName: 'text-embedding-ada-002' -}); - -// Create a vector store from the document -const vectorStore = await MemoryVectorStore.fromDocuments( - splits, - embeddingClient -); - -// Create a retriever for the vector store -const retriever = vectorStore.asRetriever(); -``` - ## Local Testing For local testing instructions, refer to this [section](https://github.com/SAP/ai-sdk-js/blob/main/README.md#local-testing). diff --git a/packages/langchain/src/orchestration/types.ts b/packages/langchain/src/orchestration/types.ts index 17347ecc2..7acaac6dc 100644 --- a/packages/langchain/src/orchestration/types.ts +++ b/packages/langchain/src/orchestration/types.ts @@ -7,7 +7,7 @@ import type { CustomRequestConfig } from '@sap-ai-sdk/core'; */ export type OrchestrationCallOptions = Pick< BaseChatModelCallOptions, - 'stop' | 'signal' | 'timeout' + 'stop' | 'signal' | 'timeout' | 'callbacks' | 'metadata' | 'runId' | 'runName' | 'tags' > & { customRequestConfig?: CustomRequestConfig; tools?: Template['tools']; diff --git a/sample-code/README.md b/sample-code/README.md index 6188bb66b..8ecd5eda4 100644 --- a/sample-code/README.md +++ b/sample-code/README.md @@ -204,13 +204,13 @@ The `toContentStream()` method is called to extract the content of the chunk for Once the streaming is done, finish reason and token usage are printed out. -### Langchain +### LangChain #### Invoke with a Simple Input `GET /langchain/invoke` -Invoke langchain Azure OpenAI client with a simple input to get chat completion response. +Invoke LangChain Azure OpenAI client with a simple input to get chat completion response. #### Invoke a Chain for Templating From 873f4d97606fbf93c67473784b528afec8510d6e Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Fri, 28 Feb 2025 18:16:22 +0100 Subject: [PATCH 15/23] add more docs --- packages/langchain/README.md | 4 +- packages/langchain/src/openai/README.md | 13 -- .../langchain/src/orchestration/README.md | 135 ++++++++---------- packages/langchain/src/orchestration/types.ts | 9 +- packages/langchain/src/orchestration/util.ts | 2 +- sample-code/README.md | 11 +- sample-code/src/index.ts | 1 + .../src/orchestration-langchain.test.ts | 11 ++ 8 files changed, 89 insertions(+), 97 deletions(-) create mode 100644 tests/e2e-tests/src/orchestration-langchain.test.ts diff --git a/packages/langchain/README.md b/packages/langchain/README.md index 429ff5479..47f77da35 100644 --- a/packages/langchain/README.md +++ b/packages/langchain/README.md @@ -10,8 +10,8 @@ This package provides LangChain model clients built on top of the foundation mod - [Prerequisites](#prerequisites) - [Relationship between Models and Deployment ID](#relationship-between-models-and-deployment-id) - [Usage](#usage) - - [SAP Orchestration Service](#sap-orchestration-service) - - [Azure OpenAI](#azure-openai) + - [SAP Orchestration Service](#sap-orchestration-service) + - [Azure OpenAI](#azure-openai) - [Local Testing](#local-testing) - [Support, Feedback, Contribution](#support-feedback-contribution) - [License](#license) diff --git a/packages/langchain/src/openai/README.md b/packages/langchain/src/openai/README.md index cc686c109..63af57a50 100644 --- a/packages/langchain/src/openai/README.md +++ b/packages/langchain/src/openai/README.md @@ -14,8 +14,6 @@ This package provides LangChain model clients built on top of the foundation mod - [Chat Client](#chat-client) - [Embedding Client](#embedding-client) - [Local Testing](#local-testing) -- [Support, Feedback, Contribution](#support-feedback-contribution) -- [License](#license) ## Installation @@ -206,14 +204,3 @@ const retriever = vectorStore.asRetriever(); ## Local Testing For local testing instructions, refer to this [section](https://github.com/SAP/ai-sdk-js/blob/main/README.md#local-testing). - -## Support, Feedback, Contribution - -This project is open to feature requests, bug reports and questions via [GitHub issues](https://github.com/SAP/ai-sdk-js/issues). - -Contribution and feedback are encouraged and always welcome. -For more information about how to contribute, the project structure, as well as additional contribution information, see our [Contribution Guidelines](https://github.com/SAP/ai-sdk-js/blob/main/CONTRIBUTING.md). - -## License - -The SAP Cloud SDK for AI is released under the [Apache License Version 2.0.](http://www.apache.org/licenses/). diff --git a/packages/langchain/src/orchestration/README.md b/packages/langchain/src/orchestration/README.md index 0a94c9f73..10bee06f4 100644 --- a/packages/langchain/src/orchestration/README.md +++ b/packages/langchain/src/orchestration/README.md @@ -11,11 +11,10 @@ This package provides LangChain model clients built on top of the foundation mod - [Relationship between Models and Deployment ID](#relationship-between-models-and-deployment-id) - [Usage](#usage) - [Client Initialization](#client-initialization) - - [Chat Client](#chat-client) - - [Embedding Client](#embedding-client) + - [Client](#client) + - [Resilience](#resilience) - [Local Testing](#local-testing) -- [Support, Feedback, Contribution](#support-feedback-contribution) -- [License](#license) +- [Limitations](#limitations) ## Installation @@ -55,60 +54,46 @@ Consequently, each deployment ID and resource group uniquely map to a combinatio ## Usage -This package offers both chat and embedding clients, currently supporting Azure OpenAI. -All clients comply with [LangChain's interface](https://js.langchain.com/docs/introduction). +This package offers a LangChain orchestration service client. +It does not support streaming yet. +The client complies with the [LangChain's interface](https://js.langchain.com/docs/introduction). ### Client Initialization -To initialize a client, provide the model name: +To initialize the client, you can provide 4 different configurations. +The only required one is the [orchestration configuration](), however, you can also set: -```ts -import { - AzureOpenAiChatClient, - AzureOpenAiEmbeddingClient -} from '@sap-ai-sdk/langchain'; - -// For a chat client -const chatClient = new AzureOpenAiChatClient({ modelName: 'gpt-4o' }); -// For an embedding client -const embeddingClient = new AzureOpenAiEmbeddingClient({ modelName: 'gpt-4o' }); -``` +- public langchainOptions: BaseChatModelParams = {}, + public deploymentConfig?: ResourceGroupConfig, + public destination?: HttpDestinationOrFetchOptions -In addition to the default parameters of the model vendor (e.g., OpenAI) and LangChain, additional parameters can be used to help narrow down the search for the desired model: +A minimal example to instantiate the orchestration client uses a template and model name: ```ts -const chatClient = new AzureOpenAiChatClient({ - modelName: 'gpt-4o', - modelVersion: '24-07-2021', - resourceGroup: 'my-resource-group' -}); -``` - -**Do not pass a `deployment ID` to initialize the client.** -For the LangChain model clients, initialization is done using the model name, model version and resource group. - -An important note is that LangChain clients by default attempt 6 retries with exponential backoff in case of a failure. -Especially in testing environments you might want to reduce this number to speed up the process: +const config: OrchestrationModuleConfig = { + llm: { + model_name: 'gpt-35-turbo' + }, + templating: { + template: [ + { role: 'user', content: 'Give me a long introduction of {{?subject}}' } + ] + } +}; -```ts -const embeddingClient = new OrchestrationClient({ - modelName: 'gpt-4o', - maxRetries: 0 -}); +const client = new OrchestratioClient(config); ``` #### Custom Destination -When initializing the `AzureOpenAiChatClient` and `AzureOpenAiEmbeddingClient` clients, it is possible to provide a custom destination. +When initializing the `OrchestrationClient`, it is possible to provide a custom destination. For example, when targeting a destination with the name `my-destination`, the following code can be used: ```ts -const chatClient = new AzureOpenAiChatClient( - { - modelName: 'gpt-4o', - modelVersion: '24-07-2021', - resourceGroup: 'my-resource-group' - }, +const client = new OrchestrationClient( + orchestrationConfig, + langchainOptions, + deploymentConfig, { destinationName: 'my-destination' } @@ -118,40 +103,38 @@ const chatClient = new AzureOpenAiChatClient( By default, the fetched destination is cached. To disable caching, set the `useCache` parameter to `false` together with the `destinationName` parameter. -### Chat Client +### Client Invocation -The chat client allows you to interact with Azure OpenAI chat models, accessible via the generative AI hub of SAP AI Core. -To invoke the client, pass a prompt: +When invoking the client, you only have to pass a message history and most of the time input parameters for the template module. ```ts -const response = await chatClient.invoke("What's the capital of France?"); +const systemMessage = new SystemMessage('Be a helpful assisstant!'); +const history = [systemMessage]; +const response = await client.invoke(history, { + inputParams: { subject: 'paris' } +}); +``` + +#### Resilience + +If you need to add resilience to your client, you can make use of the default options available in LangChain, most importantly `timeout` and `maxRetry`. + +##### Timeout + +By default, there is no timeout set in the client, if you want to limit the maximum duration the entire request, including retries, should take, +you can set a timeout duration in ms when using the invoke method: + +```ts +const response = await client.invoke(messageHistory, { timeout: 10000 }); ``` -#### Advanced Example with Templating and Output Parsing +##### Retry + +By default, LangChain clients retry 6 times, if you want to adjust this behavior, you need to do so at client initialization: ```ts -import { AzureOpenAiChatClient } from '@sap-ai-sdk/langchain'; -import { StringOutputParser } from '@langchain/core/output_parsers'; -import { ChatPromptTemplate } from '@langchain/core/prompts'; - -// initialize the client -const client = new AzureOpenAiChatClient({ modelName: 'gpt-35-turbo' }); - -// create a prompt template -const promptTemplate = ChatPromptTemplate.fromMessages([ - ['system', 'Answer the following in {language}:'], - ['user', '{text}'] -]); -// create an output parser -const parser = new StringOutputParser(); - -// chain together template, client, and parser -const llmChain = promptTemplate.pipe(client).pipe(parser); - -// invoke the chain -return llmChain.invoke({ - language: 'german', - text: 'What is the capital of France?' +const client = new OrchestrationClient(orchestrationConfig, { + maxRetries: 0 }); ``` @@ -159,13 +142,9 @@ return llmChain.invoke({ For local testing instructions, refer to this [section](https://github.com/SAP/ai-sdk-js/blob/main/README.md#local-testing). -## Support, Feedback, Contribution - -This project is open to feature requests, bug reports and questions via [GitHub issues](https://github.com/SAP/ai-sdk-js/issues). - -Contribution and feedback are encouraged and always welcome. -For more information about how to contribute, the project structure, as well as additional contribution information, see our [Contribution Guidelines](https://github.com/SAP/ai-sdk-js/blob/main/CONTRIBUTING.md). +## Limitations -## License +Currently unsupported features are: -The SAP Cloud SDK for AI is released under the [Apache License Version 2.0.](http://www.apache.org/licenses/). +- Streaming +- Tool Calling diff --git a/packages/langchain/src/orchestration/types.ts b/packages/langchain/src/orchestration/types.ts index 7acaac6dc..09f4f3806 100644 --- a/packages/langchain/src/orchestration/types.ts +++ b/packages/langchain/src/orchestration/types.ts @@ -7,7 +7,14 @@ import type { CustomRequestConfig } from '@sap-ai-sdk/core'; */ export type OrchestrationCallOptions = Pick< BaseChatModelCallOptions, - 'stop' | 'signal' | 'timeout' | 'callbacks' | 'metadata' | 'runId' | 'runName' | 'tags' + | 'stop' + | 'signal' + | 'timeout' + | 'callbacks' + | 'metadata' + | 'runId' + | 'runName' + | 'tags' > & { customRequestConfig?: CustomRequestConfig; tools?: Template['tools']; diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index f0790eab7..2785e5c7c 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -79,7 +79,7 @@ function mapHumanMessageToChatMessage(message: HumanMessage): ChatMessage { function mapSystemMessageToAzureOpenAiSystemMessage( message: SystemMessage ): ChatMessage { - // TODO: Remove as soon as image_url is a supported inputed for system messages in orchestration. + // TODO: Remove as soon as image_url is a supported input for system messages in orchestration. if ( typeof message.content !== 'string' && message.content.some(content => content.type === 'image_url') diff --git a/sample-code/README.md b/sample-code/README.md index 8ecd5eda4..88705c59d 100644 --- a/sample-code/README.md +++ b/sample-code/README.md @@ -212,14 +212,21 @@ Once the streaming is done, finish reason and token usage are printed out. Invoke LangChain Azure OpenAI client with a simple input to get chat completion response. -#### Invoke a Chain for Templating +#### Invoke a Chain with Templating `GET /langchain/invoke-chain` Invoke chain to get chat completion response from Azure OpenAI. The chain contains a template and a string parser. -#### Invoke a Chain for Retrieval-Augmented Generation (RAG) +#### Invoke a Chain with Templating and Orchestration Client + +`GET /langchain/invoke-chain-orchestration` + +Invoke a chain to get a orchestration response from the orchestration service. +The chain has a built-in template and is chained with a string parser. + +#### Invoke a Chain with Retrieval-Augmented Generation (RAG) `GET /langchain/invoke-rag-chain` diff --git a/sample-code/src/index.ts b/sample-code/src/index.ts index a6a5938e0..609157de4 100644 --- a/sample-code/src/index.ts +++ b/sample-code/src/index.ts @@ -28,6 +28,7 @@ export { invokeChain, invokeRagChain } from './langchain-azure-openai.js'; +export { invokeChain as orchestrationInvokeChain } from './langchain-orchestration.js'; export { getDeployments, getDeploymentsWithDestination, diff --git a/tests/e2e-tests/src/orchestration-langchain.test.ts b/tests/e2e-tests/src/orchestration-langchain.test.ts new file mode 100644 index 000000000..31e9c4429 --- /dev/null +++ b/tests/e2e-tests/src/orchestration-langchain.test.ts @@ -0,0 +1,11 @@ +import { orchestrationInvokeChain } from '@sap-ai-sdk/sample-code'; +import { loadEnv } from './utils/load-env.js'; + +loadEnv(); + +describe('Langchain OpenAI Access', () => { + it('executes invoke as part of a chain ', async () => { + const result = await orchestrationInvokeChain(); + expect(result).toContain('SAP Cloud SDK'); + }); +}); From 8ba3284f1080cefa74a3d5812a5b77d69205187d Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Fri, 28 Feb 2025 18:20:31 +0100 Subject: [PATCH 16/23] remove unused dependency --- packages/langchain/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/langchain/package.json b/packages/langchain/package.json index 654167af9..1a08af10b 100644 --- a/packages/langchain/package.json +++ b/packages/langchain/package.json @@ -31,7 +31,6 @@ "@sap-ai-sdk/foundation-models": "workspace:^", "@sap-ai-sdk/orchestration": "workspace:^", "@sap-cloud-sdk/connectivity": "^3.26.1", - "@sap-cloud-sdk/resilience": "^3.26.1", "uuid": "^11.1.0", "@langchain/core": "0.3.40", "zod-to-json-schema": "^3.24.3" From 2979a398e4e1f198366f8c3f0e1a9f31a7f09baf Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Fri, 28 Feb 2025 18:21:39 +0100 Subject: [PATCH 17/23] update lockfile --- pnpm-lock.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 958b9cf0e..78561e492 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -163,9 +163,6 @@ importers: '@sap-cloud-sdk/connectivity': specifier: ^3.26.1 version: 3.26.1 - '@sap-cloud-sdk/resilience': - specifier: ^3.26.1 - version: 3.26.1 uuid: specifier: ^11.1.0 version: 11.1.0 From 98c20e7fda8c081f1a4ab9a4bf4568de9129e36a Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Fri, 28 Feb 2025 18:35:56 +0100 Subject: [PATCH 18/23] add more docs --- packages/langchain/src/orchestration/README.md | 9 +++------ sample-code/src/langchain-orchestration.ts | 2 +- tests/e2e-tests/src/orchestration-langchain.test.ts | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/langchain/src/orchestration/README.md b/packages/langchain/src/orchestration/README.md index 10bee06f4..5d859f0ba 100644 --- a/packages/langchain/src/orchestration/README.md +++ b/packages/langchain/src/orchestration/README.md @@ -61,11 +61,8 @@ The client complies with the [LangChain's interface](https://js.langchain.com/do ### Client Initialization To initialize the client, you can provide 4 different configurations. -The only required one is the [orchestration configuration](), however, you can also set: - -- public langchainOptions: BaseChatModelParams = {}, - public deploymentConfig?: ResourceGroupConfig, - public destination?: HttpDestinationOrFetchOptions +The only required one is the orchestration configuration which we describe in-depth in our [orchestration foundation client](https://github.com/SAP/ai-sdk-js/blob/main/packages/orchestration/README.md). +You can also set the [default LangChain options](https://v03.api.js.langchain.com/types/_langchain_core.language_models_chat_models.BaseChatModelParams.html), as well as a custom resource group and destination. A minimal example to instantiate the orchestration client uses a template and model name: @@ -111,7 +108,7 @@ When invoking the client, you only have to pass a message history and most of th const systemMessage = new SystemMessage('Be a helpful assisstant!'); const history = [systemMessage]; const response = await client.invoke(history, { - inputParams: { subject: 'paris' } + inputParams: { subject: 'Paris' } }); ``` diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts index 68fd747f2..970bc7610 100644 --- a/sample-code/src/langchain-orchestration.ts +++ b/sample-code/src/langchain-orchestration.ts @@ -2,7 +2,7 @@ import { StringOutputParser } from '@langchain/core/output_parsers'; import { OrchestrationClient } from '@sap-ai-sdk/langchain'; /** - * Ask GPT about the capital of France, as part of a chain. + * ASk GPT about an introduction to SAP Cloud SDK. * @returns The answer from ChatGPT. */ export async function invokeChain(): Promise { diff --git a/tests/e2e-tests/src/orchestration-langchain.test.ts b/tests/e2e-tests/src/orchestration-langchain.test.ts index 31e9c4429..7c39ac38e 100644 --- a/tests/e2e-tests/src/orchestration-langchain.test.ts +++ b/tests/e2e-tests/src/orchestration-langchain.test.ts @@ -3,7 +3,7 @@ import { loadEnv } from './utils/load-env.js'; loadEnv(); -describe('Langchain OpenAI Access', () => { +describe('Orchestration LangChain client', () => { it('executes invoke as part of a chain ', async () => { const result = await orchestrationInvokeChain(); expect(result).toContain('SAP Cloud SDK'); From d8a6256be77e7219c0b233958803a9ba0e6af93e Mon Sep 17 00:00:00 2001 From: deeksha sinha Date: Mon, 3 Mar 2025 11:29:47 +0100 Subject: [PATCH 19/23] adjust docs --- packages/langchain/README.md | 39 ++++------ packages/langchain/src/openai/README.md | 48 ++++-------- .../langchain/src/orchestration/README.md | 73 ++++++------------- 3 files changed, 49 insertions(+), 111 deletions(-) diff --git a/packages/langchain/README.md b/packages/langchain/README.md index 47f77da35..98c3f7cfc 100644 --- a/packages/langchain/README.md +++ b/packages/langchain/README.md @@ -2,16 +2,15 @@ SAP Cloud SDK for AI is the official Software Development Kit (SDK) for **SAP AI Core**, **SAP Generative AI Hub**, and **Orchestration Service**. -This package provides LangChain model clients built on top of the foundation model clients of the SAP Cloud SDK for AI. +This package provides LangChain clients built on top of the foundation model and orchestration clients of the SAP Cloud SDK for AI. ### Table of Contents - [Installation](#installation) - [Prerequisites](#prerequisites) -- [Relationship between Models and Deployment ID](#relationship-between-models-and-deployment-id) - [Usage](#usage) - - [SAP Orchestration Service](#sap-orchestration-service) - - [Azure OpenAI](#azure-openai) + - [SAP Orchestration Client](#sap-orchestration-client) + - [Azure OpenAI Client](#azure-openai-client) - [Local Testing](#local-testing) - [Support, Feedback, Contribution](#support-feedback-contribution) - [License](#license) @@ -27,10 +26,11 @@ $ npm install @sap-ai-sdk/langchain - [Enable the AI Core service in SAP BTP](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/initial-setup). - Use the same `@langchain/core` version as the `@sap-ai-sdk/langchain` package, to see which langchain version this package is currently using, check our [package.json](./package.json). - Configure the project with **Node.js v20 or higher** and **native ESM** support. -- Ensure a deployed OpenAI model is available in the SAP Generative AI Hub. - - Use the [`DeploymentApi`](https://github.com/SAP/ai-sdk-js/blob/main/packages/ai-api/README.md#create-a-deployment) from `@sap-ai-sdk/ai-api` [to deploy a model](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). - Alternatively, you can also create deployments using the [SAP AI Launchpad](https://help.sap.com/docs/sap-ai-core/generative-ai-hub/activate-generative-ai-hub-for-sap-ai-launchpad?locale=en-US&q=launchpad). - - Once deployment is complete, access the model via the `deploymentUrl`. +- Ensure that a relevant deployment is available in the SAP Generative AI Hub: + - Use the [`DeploymentApi`](https://github.com/SAP/ai-sdk-js/blob/main/packages/ai-api/README.md#create-a-deployment) from `@sap-ai-sdk/ai-api` or the [SAP AI Launchpad](https://help.sap.com/docs/sap-ai-core/generative-ai-hub/activate-generative-ai-hub-for-sap-ai-launchpad?locale=en-US&q=launchpad) to create a deployment. + - For **OpenAI models**, follow [this guide](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). + - For **orchestration services**, follow [this guide](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-orchestration). + - Once deployed, access the service via the `deploymentUrl`. > **Accessing the AI Core Service via the SDK** > @@ -39,31 +39,18 @@ $ npm install @sap-ai-sdk/langchain > - In Cloud Foundry, it's accessed from the `VCAP_SERVICES` environment variable. > - In Kubernetes / Kyma environments, you have to mount the service binding as a secret instead, for more information refer to [this documentation](https://www.npmjs.com/package/@sap/xsenv#usage-in-kubernetes). -## Relationship between Models and Deployment ID - -SAP AI Core manages access to generative AI models through the global AI scenario `foundation-models`. -Creating a deployment for a model requires access to this scenario. - -Each model, model version, and resource group allows for a one-time deployment. -After deployment completion, the response includes a `deploymentUrl` and an `id`, which is the deployment ID. -For more information, see [here](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). - -[Resource groups](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/resource-groups?q=resource+group) represent a virtual collection of related resources within the scope of one SAP AI Core tenant. - -Consequently, each deployment ID and resource group uniquely map to a combination of model and model version within the `foundation-models` scenario. - ## Usage -This package offers LangChain clients for Azure OpenAI as well as the SAP orchestration service +This package offers LangChain clients for Azure OpenAI and SAP Orchestration service. All clients comply with [LangChain's interface](https://js.langchain.com/docs/introduction). -### SAP Orchestration Service +### Orchestration Client -For details on the orchestration client, refer to this [document](./src/orchestration/README.md). +For more information about the Orchestration client, refer to the [documentation](https://github.com/SAP/ai-sdk-js/tree/main/packages/langchain/src/orchestration/README.md). -### Azure OpenAI +### Azure OpenAI Client -For details on the Azure OpenAI clients, refer to this [document](./src/openai/README.md). +For more information about Azure OpenAI client, refer to the [documentation](https://github.com/SAP/ai-sdk-js/tree/main/packages/langchain/src/openai/README.md). ## Local Testing diff --git a/packages/langchain/src/openai/README.md b/packages/langchain/src/openai/README.md index 63af57a50..d44ae394e 100644 --- a/packages/langchain/src/openai/README.md +++ b/packages/langchain/src/openai/README.md @@ -4,10 +4,10 @@ SAP Cloud SDK for AI is the official Software Development Kit (SDK) for **SAP AI This package provides LangChain model clients built on top of the foundation model clients of the SAP Cloud SDK for AI. +> **Note**: For installation and prerequisites, refer to the [README](../../README.md). + ### Table of Contents -- [Installation](#installation) -- [Prerequisites](#prerequisites) - [Relationship between Models and Deployment ID](#relationship-between-models-and-deployment-id) - [Usage](#usage) - [Client Initialization](#client-initialization) @@ -15,28 +15,6 @@ This package provides LangChain model clients built on top of the foundation mod - [Embedding Client](#embedding-client) - [Local Testing](#local-testing) -## Installation - -``` -$ npm install @sap-ai-sdk/langchain -``` - -## Prerequisites - -- [Enable the AI Core service in SAP BTP](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/initial-setup). -- Use the same `@langchain/core` version as the `@sap-ai-sdk/langchain` package, to see which langchain version this package is currently using, check our [package.json](./package.json). -- Configure the project with **Node.js v20 or higher** and **native ESM** support. -- Ensure a deployed OpenAI model is available in the SAP Generative AI Hub. - - Use the [`DeploymentApi`](https://github.com/SAP/ai-sdk-js/blob/main/packages/ai-api/README.md#create-a-deployment) from `@sap-ai-sdk/ai-api` [to deploy a model](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). - Alternatively, you can also create deployments using the [SAP AI Launchpad](https://help.sap.com/docs/sap-ai-core/generative-ai-hub/activate-generative-ai-hub-for-sap-ai-launchpad?locale=en-US&q=launchpad). - - Once deployment is complete, access the model via the `deploymentUrl`. - -> **Accessing the AI Core Service via the SDK** -> -> The SDK automatically retrieves the `AI Core` service credentials and resolves the access token needed for authentication. -> -> - In Cloud Foundry, it's accessed from the `VCAP_SERVICES` environment variable. -> - In Kubernetes / Kyma environments, you have to mount the service binding as a secret instead, for more information refer to [this documentation](https://www.npmjs.com/package/@sap/xsenv#usage-in-kubernetes). ## Relationship between Models and Deployment ID @@ -54,7 +32,7 @@ Consequently, each deployment ID and resource group uniquely map to a combinatio ## Usage This package offers both chat and embedding clients for Azure OpenAI. -All clients comply with [LangChain's interface](https://js.langchain.com/docs/introduction). +The client complies with [LangChain's interface](https://js.langchain.com/docs/introduction). ### Client Initialization @@ -83,10 +61,10 @@ const chatClient = new AzureOpenAiChatClient({ ``` **Do not pass a `deployment ID` to initialize the client.** -For the LangChain model clients, initialization is done using the model name, model version and resource group. +For LangChain model clients, initialization requires specifying the model name, model version, and resource group. -An important note is that LangChain clients by default attempt 6 retries with exponential backoff in case of a failure. -Especially in testing environments you might want to reduce this number to speed up the process: +By default, LangChain clients retry up to 6 times with exponential backoff in case of failure. +In testing environments, reducing this number can speed up the process: ```ts const embeddingClient = new AzureOpenAiEmbeddingClient({ @@ -97,8 +75,8 @@ const embeddingClient = new AzureOpenAiEmbeddingClient({ #### Custom Destination -When initializing the `AzureOpenAiChatClient` and `AzureOpenAiEmbeddingClient` clients, it is possible to provide a custom destination. -For example, when targeting a destination with the name `my-destination`, the following code can be used: +When initializing the `AzureOpenAiChatClient` and `AzureOpenAiEmbeddingClient`, a custom destination can be specified. +For example, to target `my-destination`, use the following code: ```ts const chatClient = new AzureOpenAiChatClient( @@ -118,8 +96,8 @@ To disable caching, set the `useCache` parameter to `false` together with the `d ### Chat Client -The chat client allows you to interact with Azure OpenAI chat models, accessible via the generative AI hub of SAP AI Core. -To invoke the client, pass a prompt: +The `AzureOpenAiChatClient` allows interaction with Azure OpenAI chat models, accessible through the Generative AI Hub of SAP AI Core. +To invoke the client, pass a prompt as shown below: ```ts const response = await chatClient.invoke("What's the capital of France?"); @@ -155,9 +133,9 @@ return llmChain.invoke({ ### Embedding Client -Embedding clients allow embedding either text or document chunks (represented as arrays of strings). -While you can use them standalone, they are usually used in combination with other LangChain utilities, like a text splitter for preprocessing and a vector store for storage and retrieval of the relevant embeddings. -For a complete example how to implement RAG with our LangChain client, take a look at our [sample code](https://github.com/SAP/ai-sdk-js/blob/main/sample-code/src/langchain-azure-openai.ts). +The `AzureOpenAiEmbeddingClient` allows embedding of text or document chunks (represented as arrays of strings). +While it can be used standalone, it is typically combined with other LangChain utilities, such as a text splitter for preprocessing and a vector store for storing and retrieving relevant embeddings. +For a complete example of how to implement RAG with the LangChain client, refer to the [sample code](https://github.com/SAP/ai-sdk-js/blob/main/sample-code/src/langchain-azure-openai.ts). #### Embed Text diff --git a/packages/langchain/src/orchestration/README.md b/packages/langchain/src/orchestration/README.md index 5d859f0ba..ff38ac3d0 100644 --- a/packages/langchain/src/orchestration/README.md +++ b/packages/langchain/src/orchestration/README.md @@ -4,11 +4,11 @@ SAP Cloud SDK for AI is the official Software Development Kit (SDK) for **SAP AI This package provides LangChain model clients built on top of the foundation model clients of the SAP Cloud SDK for AI. +> **Note**: For installation and prerequisites, refer to the [README](../../README.md). + ### Table of Contents -- [Installation](#installation) -- [Prerequisites](#prerequisites) -- [Relationship between Models and Deployment ID](#relationship-between-models-and-deployment-id) +- [Relationship between Orchestration and Resource Groups](#relationship-between-orchestration-and-resource-groups) - [Usage](#usage) - [Client Initialization](#client-initialization) - [Client](#client) @@ -16,55 +16,29 @@ This package provides LangChain model clients built on top of the foundation mod - [Local Testing](#local-testing) - [Limitations](#limitations) -## Installation - -``` -$ npm install @sap-ai-sdk/langchain -``` - -## Prerequisites +## Relationship between Orchestration and Resource Groups -- [Enable the AI Core service in SAP BTP](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/initial-setup). -- Use the same `@langchain/core` version as the `@sap-ai-sdk/langchain` package, to see which langchain version this package is currently using, check our [package.json](./package.json). -- Configure the project with **Node.js v20 or higher** and **native ESM** support. -- Ensure a deployed OpenAI model is available in the SAP Generative AI Hub. - - Use the [`DeploymentApi`](https://github.com/SAP/ai-sdk-js/blob/main/packages/ai-api/README.md#create-a-deployment) from `@sap-ai-sdk/ai-api` [to deploy a model](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). - Alternatively, you can also create deployments using the [SAP AI Launchpad](https://help.sap.com/docs/sap-ai-core/generative-ai-hub/activate-generative-ai-hub-for-sap-ai-launchpad?locale=en-US&q=launchpad). - - Once deployment is complete, access the model via the `deploymentUrl`. - -> **Accessing the AI Core Service via the SDK** -> -> The SDK automatically retrieves the `AI Core` service credentials and resolves the access token needed for authentication. -> -> - In Cloud Foundry, it's accessed from the `VCAP_SERVICES` environment variable. -> - In Kubernetes / Kyma environments, you have to mount the service binding as a secret instead, for more information refer to [this documentation](https://www.npmjs.com/package/@sap/xsenv#usage-in-kubernetes). - -## Relationship between Models and Deployment ID - -SAP AI Core manages access to generative AI models through the global AI scenario `foundation-models`. -Creating a deployment for a model requires access to this scenario. - -Each model, model version, and resource group allows for a one-time deployment. -After deployment completion, the response includes a `deploymentUrl` and an `id`, which is the deployment ID. -For more information, see [here](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). +SAP AI Core manages access to orchestration of generative AI models through the global AI scenario `orchestration`. +Creating a deployment for enabling orchestration capabilities requires access to this scenario. [Resource groups](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/resource-groups?q=resource+group) represent a virtual collection of related resources within the scope of one SAP AI Core tenant. +Each resource group allows for a one-time orchestration deployment. -Consequently, each deployment ID and resource group uniquely map to a combination of model and model version within the `foundation-models` scenario. +Consequently, each orchestration deployment uniquely maps to a resource group within the `orchestration` scenario. ## Usage This package offers a LangChain orchestration service client. -It does not support streaming yet. -The client complies with the [LangChain's interface](https://js.langchain.com/docs/introduction). +Streaming is not supported. +The client complies with [LangChain's interface](https://js.langchain.com/docs/introduction). ### Client Initialization -To initialize the client, you can provide 4 different configurations. -The only required one is the orchestration configuration which we describe in-depth in our [orchestration foundation client](https://github.com/SAP/ai-sdk-js/blob/main/packages/orchestration/README.md). -You can also set the [default LangChain options](https://v03.api.js.langchain.com/types/_langchain_core.language_models_chat_models.BaseChatModelParams.html), as well as a custom resource group and destination. +To initialize the client, four different configurations can be provided. +The only required configuration is the orchestration configuration, explained in detail in the [orchestration foundation client](https://github.com/SAP/ai-sdk-js/blob/main/packages/orchestration/README.md). +Additionally, it is possible to set [default LangChain options](https://v03.api.js.langchain.com/types/_langchain_core.language_models_chat_models.BaseChatModelParams.html), a custom resource group, and a destination. -A minimal example to instantiate the orchestration client uses a template and model name: +A minimal example for instantiating the orchestration client uses a template and model name: ```ts const config: OrchestrationModuleConfig = { @@ -83,17 +57,15 @@ const client = new OrchestratioClient(config); #### Custom Destination -When initializing the `OrchestrationClient`, it is possible to provide a custom destination. -For example, when targeting a destination with the name `my-destination`, the following code can be used: +The `OrchestrationClient` can be initialized with a custom destination. +For example, to target `my-destination`, use the following code: ```ts const client = new OrchestrationClient( orchestrationConfig, langchainOptions, deploymentConfig, - { - destinationName: 'my-destination' - } + { destinationName: 'my-destination' } ); ``` @@ -102,7 +74,7 @@ To disable caching, set the `useCache` parameter to `false` together with the `d ### Client Invocation -When invoking the client, you only have to pass a message history and most of the time input parameters for the template module. +When invoking the client, pass a message history and, in most cases, input parameters for the template module. ```ts const systemMessage = new SystemMessage('Be a helpful assisstant!'); @@ -114,12 +86,12 @@ const response = await client.invoke(history, { #### Resilience -If you need to add resilience to your client, you can make use of the default options available in LangChain, most importantly `timeout` and `maxRetry`. +To add resilience to the client, use LangChain's default options, especially `timeout` and `maxRetry`. ##### Timeout -By default, there is no timeout set in the client, if you want to limit the maximum duration the entire request, including retries, should take, -you can set a timeout duration in ms when using the invoke method: +By default, no timeout is set in the client. +To limit the maximum duration for the entire request, including retries, specify a timeout in milliseconds when using the `invoke` method: ```ts const response = await client.invoke(messageHistory, { timeout: 10000 }); @@ -127,7 +99,8 @@ const response = await client.invoke(messageHistory, { timeout: 10000 }); ##### Retry -By default, LangChain clients retry 6 times, if you want to adjust this behavior, you need to do so at client initialization: +LangChain clients retry up to 6 times by default. +To modify this behavior, set the `maxRetries` option during client initialization: ```ts const client = new OrchestrationClient(orchestrationConfig, { From 6050f26ee7a3b5c761e203bee0bbf09dd1620e65 Mon Sep 17 00:00:00 2001 From: cloud-sdk-js Date: Mon, 3 Mar 2025 10:30:47 +0000 Subject: [PATCH 20/23] fix: Changes from lint --- packages/langchain/README.md | 8 ++++---- packages/langchain/src/openai/README.md | 1 - packages/langchain/src/orchestration/README.md | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/langchain/README.md b/packages/langchain/README.md index 98c3f7cfc..a394e7e97 100644 --- a/packages/langchain/README.md +++ b/packages/langchain/README.md @@ -27,10 +27,10 @@ $ npm install @sap-ai-sdk/langchain - Use the same `@langchain/core` version as the `@sap-ai-sdk/langchain` package, to see which langchain version this package is currently using, check our [package.json](./package.json). - Configure the project with **Node.js v20 or higher** and **native ESM** support. - Ensure that a relevant deployment is available in the SAP Generative AI Hub: - - Use the [`DeploymentApi`](https://github.com/SAP/ai-sdk-js/blob/main/packages/ai-api/README.md#create-a-deployment) from `@sap-ai-sdk/ai-api` or the [SAP AI Launchpad](https://help.sap.com/docs/sap-ai-core/generative-ai-hub/activate-generative-ai-hub-for-sap-ai-launchpad?locale=en-US&q=launchpad) to create a deployment. - - For **OpenAI models**, follow [this guide](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). - - For **orchestration services**, follow [this guide](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-orchestration). - - Once deployed, access the service via the `deploymentUrl`. + - Use the [`DeploymentApi`](https://github.com/SAP/ai-sdk-js/blob/main/packages/ai-api/README.md#create-a-deployment) from `@sap-ai-sdk/ai-api` or the [SAP AI Launchpad](https://help.sap.com/docs/sap-ai-core/generative-ai-hub/activate-generative-ai-hub-for-sap-ai-launchpad?locale=en-US&q=launchpad) to create a deployment. + - For **OpenAI models**, follow [this guide](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). + - For **orchestration services**, follow [this guide](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-orchestration). + - Once deployed, access the service via the `deploymentUrl`. > **Accessing the AI Core Service via the SDK** > diff --git a/packages/langchain/src/openai/README.md b/packages/langchain/src/openai/README.md index d44ae394e..5136fdc50 100644 --- a/packages/langchain/src/openai/README.md +++ b/packages/langchain/src/openai/README.md @@ -15,7 +15,6 @@ This package provides LangChain model clients built on top of the foundation mod - [Embedding Client](#embedding-client) - [Local Testing](#local-testing) - ## Relationship between Models and Deployment ID SAP AI Core manages access to generative AI models through the global AI scenario `foundation-models`. diff --git a/packages/langchain/src/orchestration/README.md b/packages/langchain/src/orchestration/README.md index ff38ac3d0..194bbb608 100644 --- a/packages/langchain/src/orchestration/README.md +++ b/packages/langchain/src/orchestration/README.md @@ -36,7 +36,7 @@ The client complies with [LangChain's interface](https://js.langchain.com/docs/i To initialize the client, four different configurations can be provided. The only required configuration is the orchestration configuration, explained in detail in the [orchestration foundation client](https://github.com/SAP/ai-sdk-js/blob/main/packages/orchestration/README.md). -Additionally, it is possible to set [default LangChain options](https://v03.api.js.langchain.com/types/_langchain_core.language_models_chat_models.BaseChatModelParams.html), a custom resource group, and a destination. +Additionally, it is possible to set [default LangChain options](https://v03.api.js.langchain.com/types/_langchain_core.language_models_chat_models.BaseChatModelParams.html), a custom resource group, and a destination. A minimal example for instantiating the orchestration client uses a template and model name: From f61dd765641ff1b07b49f4c59f9aeec56d319e00 Mon Sep 17 00:00:00 2001 From: deeksha sinha Date: Mon, 3 Mar 2025 12:17:37 +0100 Subject: [PATCH 21/23] lint --- packages/langchain/src/openai/README.md | 8 ++++---- packages/langchain/src/orchestration/README.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/langchain/src/openai/README.md b/packages/langchain/src/openai/README.md index 5136fdc50..43f5a9334 100644 --- a/packages/langchain/src/openai/README.md +++ b/packages/langchain/src/openai/README.md @@ -74,7 +74,7 @@ const embeddingClient = new AzureOpenAiEmbeddingClient({ #### Custom Destination -When initializing the `AzureOpenAiChatClient` and `AzureOpenAiEmbeddingClient`, a custom destination can be specified. +When initializing the `AzureOpenAiChatClient` and `AzureOpenAiEmbeddingClient`, a custom destination can be specified. For example, to target `my-destination`, use the following code: ```ts @@ -95,7 +95,7 @@ To disable caching, set the `useCache` parameter to `false` together with the `d ### Chat Client -The `AzureOpenAiChatClient` allows interaction with Azure OpenAI chat models, accessible through the Generative AI Hub of SAP AI Core. +The `AzureOpenAiChatClient` allows interaction with Azure OpenAI chat models, accessible through the Generative AI Hub of SAP AI Core. To invoke the client, pass a prompt as shown below: ```ts @@ -132,8 +132,8 @@ return llmChain.invoke({ ### Embedding Client -The `AzureOpenAiEmbeddingClient` allows embedding of text or document chunks (represented as arrays of strings). -While it can be used standalone, it is typically combined with other LangChain utilities, such as a text splitter for preprocessing and a vector store for storing and retrieving relevant embeddings. +The `AzureOpenAiEmbeddingClient` allows embedding of text or document chunks (represented as arrays of strings). +While it can be used standalone, it is typically combined with other LangChain utilities, such as a text splitter for preprocessing and a vector store for storing and retrieving relevant embeddings. For a complete example of how to implement RAG with the LangChain client, refer to the [sample code](https://github.com/SAP/ai-sdk-js/blob/main/sample-code/src/langchain-azure-openai.ts). #### Embed Text diff --git a/packages/langchain/src/orchestration/README.md b/packages/langchain/src/orchestration/README.md index 194bbb608..e39bb4f64 100644 --- a/packages/langchain/src/orchestration/README.md +++ b/packages/langchain/src/orchestration/README.md @@ -34,7 +34,7 @@ The client complies with [LangChain's interface](https://js.langchain.com/docs/i ### Client Initialization -To initialize the client, four different configurations can be provided. +To initialize the client, four different configurations can be provided. The only required configuration is the orchestration configuration, explained in detail in the [orchestration foundation client](https://github.com/SAP/ai-sdk-js/blob/main/packages/orchestration/README.md). Additionally, it is possible to set [default LangChain options](https://v03.api.js.langchain.com/types/_langchain_core.language_models_chat_models.BaseChatModelParams.html), a custom resource group, and a destination. @@ -57,7 +57,7 @@ const client = new OrchestratioClient(config); #### Custom Destination -The `OrchestrationClient` can be initialized with a custom destination. +The `OrchestrationClient` can be initialized with a custom destination. For example, to target `my-destination`, use the following code: ```ts From ffaf5d284cd25d8ed969c55f12620f6cf72793d3 Mon Sep 17 00:00:00 2001 From: cloud-sdk-js Date: Mon, 3 Mar 2025 11:18:32 +0000 Subject: [PATCH 22/23] fix: Changes from lint --- packages/langchain/src/openai/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/langchain/src/openai/README.md b/packages/langchain/src/openai/README.md index 43f5a9334..a101d948b 100644 --- a/packages/langchain/src/openai/README.md +++ b/packages/langchain/src/openai/README.md @@ -74,7 +74,7 @@ const embeddingClient = new AzureOpenAiEmbeddingClient({ #### Custom Destination -When initializing the `AzureOpenAiChatClient` and `AzureOpenAiEmbeddingClient`, a custom destination can be specified. +When initializing the `AzureOpenAiChatClient` and `AzureOpenAiEmbeddingClient`, a custom destination can be specified. For example, to target `my-destination`, use the following code: ```ts From 8b93796ac342644436713b4ddeaafbfc339a1aa3 Mon Sep 17 00:00:00 2001 From: Deeksha Sinha <88374536+deekshas8@users.noreply.github.com> Date: Mon, 3 Mar 2025 12:30:44 +0100 Subject: [PATCH 23/23] Apply suggestions from code review --- packages/langchain/README.md | 6 +++--- packages/langchain/src/orchestration/README.md | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/langchain/README.md b/packages/langchain/README.md index a394e7e97..376f2e776 100644 --- a/packages/langchain/README.md +++ b/packages/langchain/README.md @@ -9,7 +9,7 @@ This package provides LangChain clients built on top of the foundation model and - [Installation](#installation) - [Prerequisites](#prerequisites) - [Usage](#usage) - - [SAP Orchestration Client](#sap-orchestration-client) + - [Orchestration Client](#orchestration-client) - [Azure OpenAI Client](#azure-openai-client) - [Local Testing](#local-testing) - [Support, Feedback, Contribution](#support-feedback-contribution) @@ -28,8 +28,8 @@ $ npm install @sap-ai-sdk/langchain - Configure the project with **Node.js v20 or higher** and **native ESM** support. - Ensure that a relevant deployment is available in the SAP Generative AI Hub: - Use the [`DeploymentApi`](https://github.com/SAP/ai-sdk-js/blob/main/packages/ai-api/README.md#create-a-deployment) from `@sap-ai-sdk/ai-api` or the [SAP AI Launchpad](https://help.sap.com/docs/sap-ai-core/generative-ai-hub/activate-generative-ai-hub-for-sap-ai-launchpad?locale=en-US&q=launchpad) to create a deployment. - - For **OpenAI models**, follow [this guide](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). - - For **orchestration services**, follow [this guide](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-orchestration). + - For **OpenAI model**, follow [this guide](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-generative-ai-model-in-sap-ai-core). + - For **orchestration service**, follow [this guide](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/create-deployment-for-orchestration). - Once deployed, access the service via the `deploymentUrl`. > **Accessing the AI Core Service via the SDK** diff --git a/packages/langchain/src/orchestration/README.md b/packages/langchain/src/orchestration/README.md index e39bb4f64..a23d11edf 100644 --- a/packages/langchain/src/orchestration/README.md +++ b/packages/langchain/src/orchestration/README.md @@ -41,6 +41,7 @@ Additionally, it is possible to set [default LangChain options](https://v03.api. A minimal example for instantiating the orchestration client uses a template and model name: ```ts +import { OrchestrationClient } from '@sap-ai-sdk/langchain'; const config: OrchestrationModuleConfig = { llm: { model_name: 'gpt-35-turbo' @@ -52,7 +53,7 @@ const config: OrchestrationModuleConfig = { } }; -const client = new OrchestratioClient(config); +const client = new OrchestrationClient(config); ``` #### Custom Destination