feat: Add streaming API to orchestration (#352)

Author: tomfrenken

.changeset/hip-melons-destroy.md

@@ -0,0 +1,5 @@
+---
+'@sap-ai-sdk/orchestration': minor
+---
+
+[New Functionality] Add support for streaming in the orchestration client.

packages/core/src/index.ts

@@ -9,3 +9,4 @@ export {
   AwsBedrockChatModel,
   AiCoreOpenSourceChatModel
 } from './model-types.js';
+export { SseStream, LineDecoder, SSEDecoder } from './stream/index.js';

packages/core/src/stream/index.ts

@@ -0,0 +1,3 @@
+export * from './sse-stream.js';
+export * from './sse-decoder.js';
+export * from './line-decoder.js';

…/src/azure-openai/stream/line-decoder.ts packages/core/src/stream/line-decoder.tspackages/foundation-models/src/azure-openai/stream/line-decoder.ts renamed to packages/core/src/stream/line-decoder.ts packages/foundation-models/src/azure-openai/stream/line-decoder.ts renamed to packages/core/src/stream/line-decoder.ts

@@ -5,7 +5,6 @@ type Bytes = string | ArrayBuffer | Uint8Array | Buffer | null | undefined;
  * reading lines from text.
  *
  * Https://github.com/encode/httpx/blob/920333ea98118e9cf617f246905d7b202510941c/httpx/_decoders.py#L258.
- * @internal
  */
 export class LineDecoder {
   // prettier-ignore

…s/src/azure-openai/stream/sse-decoder.ts packages/core/src/stream/sse-decoder.tspackages/foundation-models/src/azure-openai/stream/sse-decoder.ts renamed to packages/core/src/stream/sse-decoder.ts packages/foundation-models/src/azure-openai/stream/sse-decoder.ts renamed to packages/core/src/stream/sse-decoder.ts

@@ -19,7 +19,6 @@ export interface ServerSentEvent {
 
 /**
  * Server-Sent Event decoder.
- * @internal
  */
 export class SSEDecoder {
   private data: string[];

…c/azure-openai/stream/sse-stream.test.ts …kages/core/src/stream/sse-stream.test.tspackages/foundation-models/src/azure-openai/stream/sse-stream.test.ts renamed to packages/core/src/stream/sse-stream.test.ts packages/foundation-models/src/azure-openai/stream/sse-stream.test.ts renamed to packages/core/src/stream/sse-stream.test.ts

@@ -13,7 +13,6 @@ type Bytes = string | ArrayBuffer | Uint8Array | Buffer | null | undefined;
 
 /**
  * Stream implemented as an async iterable.
- * @internal
  */
 export class SseStream<Item> implements AsyncIterable<Item> {
   protected static transformToSseStream<Item>(

…ls/src/azure-openai/stream/sse-stream.ts packages/core/src/stream/sse-stream.tspackages/foundation-models/src/azure-openai/stream/sse-stream.ts renamed to packages/core/src/stream/sse-stream.ts packages/foundation-models/src/azure-openai/stream/sse-stream.ts renamed to packages/core/src/stream/sse-stream.ts

@@ -152,7 +152,7 @@ Refer to `AzureOpenAiChatCompletionParameters` interface for other parameters th
 The `AzureOpenAiChatClient` supports streaming response for chat completion requests based on the [Server-sent events](https://html.spec.whatwg.org/multipage/server-sent-events.html#server-sent-events) standard.
 
 Use the `stream()` method to receive a stream of chunk responses from the model.
-After consuming the stream, call the helper methods to get the finish reason and token usage information respectively.
+After consuming the stream, call the helper methods to get the finish reason and token usage information.
 
 ```ts
 const chatClient = new AzureOpenAiChatClient('gpt-4o');
@@ -178,7 +178,7 @@ console.log(`Token usage: ${JSON.stringify(tokenUsage)}\n`);
 
 ##### Streaming the Delta Content
 
-The client provides a helper method to extract delta content and stream string directly.
+The client provides a helper method to extract the text chunks as strings:
 
 ```ts
 for await (const chunk of response.stream.toContentStream()) {
@@ -198,7 +198,7 @@ Additionally, it can be aborted manually by calling the `stream()` method with a
 ```ts
 const chatClient = new AzureOpenAiChatClient('gpt-4o');
 const controller = new AbortController();
-const response = await new AzureOpenAiChatClient('gpt-35-turbo').stream(
+const response = await chatClient.stream(
   {
     messages: [
       {

packages/foundation-models/README.md

@@ -198,7 +198,7 @@ describe('Azure OpenAI chat client', () => {
 
     const response = await client.stream(prompt);
     for await (const chunk of response.stream) {
-      expect(JSON.stringify(chunk.data)).toEqual(initialResponse);
+      expect(chunk.data).toEqual(JSON.parse(initialResponse));
       break;
     }
   });

packages/foundation-models/src/azure-openai/azure-openai-chat-client.test.ts

@@ -1,9 +1,8 @@
 import { createLogger } from '@sap-cloud-sdk/util';
 import { jest } from '@jest/globals';
+import { LineDecoder, SSEDecoder } from '@sap-ai-sdk/core';
 import { parseFileToString } from '../../../../test-util/mock-http.js';
 import { AzureOpenAiChatCompletionStream } from './azure-openai-chat-completion-stream.js';
-import { LineDecoder } from './stream/line-decoder.js';
-import { SSEDecoder } from './stream/sse-decoder.js';
 
 describe('OpenAI chat completion stream', () => {
   let sseChunks: string[];
@@ -39,12 +38,12 @@ describe('OpenAI chat completion stream', () => {
 
   it('should wrap the raw chunk', async () => {
     let output = '';
-    const asnycGenerator = AzureOpenAiChatCompletionStream._processChunk(
+    const asyncGenerator = AzureOpenAiChatCompletionStream._processChunk(
       originalChatCompletionStream
     );
-    for await (const chunk of asnycGenerator) {
+    for await (const chunk of asyncGenerator) {
       expect(chunk).toBeDefined();
-      chunk.getDeltaContent() ? (output += chunk.getDeltaContent()) : null;
+      output += chunk.getDeltaContent() ?? '';
     }
     expect(output).toEqual('The capital of France is Paris.');
   });