From 1d58a06e1135d9f7017eeb7edbe378965c58fc2f Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Tue, 29 Apr 2025 13:32:36 -0400 Subject: [PATCH 1/8] feat(vertexai): Gemini multimodal output --- common/api-review/vertexai.api.md | 12 ++++ ...ertexai.enhancedgeneratecontentresponse.md | 11 ++++ docs-devsite/vertexai.generationconfig.md | 20 ++++++ docs-devsite/vertexai.md | 31 +++++++++ .../src/requests/response-helpers.test.ts | 64 +++++++++++++++++++ .../vertexai/src/requests/response-helpers.ts | 58 +++++++++++++++++ packages/vertexai/src/types/enums.ts | 26 ++++++++ packages/vertexai/src/types/requests.ts | 13 +++- packages/vertexai/src/types/responses.ts | 11 +++- 9 files changed, 244 insertions(+), 2 deletions(-) diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index e7f00c2f4e..2c06a5b67b 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -124,6 +124,7 @@ export { Date_2 as Date } export interface EnhancedGenerateContentResponse extends GenerateContentResponse { // (undocumented) functionCalls: () => FunctionCall[] | undefined; + inlineData: () => GenerativeContentBlob[] | undefined; text: () => string; } @@ -304,6 +305,8 @@ export interface GenerationConfig { // (undocumented) presencePenalty?: number; responseMimeType?: string; + // @beta + responseModalities?: ResponseModality[]; responseSchema?: TypedSchema | SchemaRequest; // (undocumented) stopSequences?: string[]; @@ -596,6 +599,15 @@ export interface RequestOptions { timeout?: number; } +// @beta +export const ResponseModality: { + readonly TEXT: "TEXT"; + readonly IMAGE: "IMAGE"; +}; + +// @beta +export type ResponseModality = (typeof ResponseModality)[keyof typeof ResponseModality]; + // @public (undocumented) export interface RetrievedContextAttribution { // (undocumented) diff --git a/docs-devsite/vertexai.enhancedgeneratecontentresponse.md b/docs-devsite/vertexai.enhancedgeneratecontentresponse.md index 535fb9def8..338b6b17f5 100644 --- a/docs-devsite/vertexai.enhancedgeneratecontentresponse.md +++ b/docs-devsite/vertexai.enhancedgeneratecontentresponse.md @@ -24,6 +24,7 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse | Property | Type | Description | | --- | --- | --- | | [functionCalls](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsefunctioncalls) | () => [FunctionCall](./vertexai.functioncall.md#functioncall_interface)\[\] \| undefined | | +| [inlineData](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponseinlinedata) | () => [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface)\[\] \| undefined | Aggregates and returns all [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) from the [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface)'s first candidate. | | [text](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsetext) | () => string | Returns the text string from the response, if available. Throws if the prompt or candidate was blocked. | ## EnhancedGenerateContentResponse.functionCalls @@ -34,6 +35,16 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse functionCalls: () => FunctionCall[] | undefined; ``` +## EnhancedGenerateContentResponse.inlineData + +Aggregates and returns all [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) from the [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface)'s first candidate. + +Signature: + +```typescript +inlineData: () => GenerativeContentBlob[] | undefined; +``` + ## EnhancedGenerateContentResponse.text Returns the text string from the response, if available. Throws if the prompt or candidate was blocked. diff --git a/docs-devsite/vertexai.generationconfig.md b/docs-devsite/vertexai.generationconfig.md index d3e9879f93..f1c54c1100 100644 --- a/docs-devsite/vertexai.generationconfig.md +++ b/docs-devsite/vertexai.generationconfig.md @@ -27,7 +27,12 @@ export interface GenerationConfig | [maxOutputTokens](./vertexai.generationconfig.md#generationconfigmaxoutputtokens) | number | | | [presencePenalty](./vertexai.generationconfig.md#generationconfigpresencepenalty) | number | | | [responseMimeType](./vertexai.generationconfig.md#generationconfigresponsemimetype) | string | Output response MIME type of the generated candidate text. Supported MIME types are text/plain (default, text output), application/json (JSON response in the candidates), and text/x.enum. | +<<<<<<< HEAD | [responseSchema](./vertexai.generationconfig.md#generationconfigresponseschema) | [TypedSchema](./vertexai.md#typedschema) \| [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) | Output response schema of the generated candidate text. This value can be a class generated with a [Schema](./vertexai.schema.md#schema_class) static method like Schema.string() or Schema.object() or it can be a plain JS object matching the [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) interface.
Note: This only applies when the specified responseMIMEType supports a schema; currently this is limited to application/json and text/x.enum. | +======= +| [responseModalities](./vertexai.generationconfig.md#generationconfigresponsemodalities) | [ResponseModality](./vertexai.md#responsemodality)\[\] | (Public Preview) Generation modalities to be returned in generation responses. | +| [responseSchema](./vertexai.generationconfig.md#generationconfigresponseschema) | [TypedSchema](./vertexai.md#typedschema) \| [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) | Output response schema of the generated candidate text. This value can be a class generated with a [Schema](./vertexai.schema.md#schema_class) static method like Schema.string() or Schema.object() or it can be a plain JS object matching the [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) interface.
Note: This only applies when the specified responseMIMEType supports a schema; currently this is limited to application/json and text/x.enum. | +>>>>>>> 4f7f1ecb1 (feat(vertexai): Gemini multimodal output) | [stopSequences](./vertexai.generationconfig.md#generationconfigstopsequences) | string\[\] | | | [temperature](./vertexai.generationconfig.md#generationconfigtemperature) | number | | | [topK](./vertexai.generationconfig.md#generationconfigtopk) | number | | @@ -75,6 +80,21 @@ Output response MIME type of the generated candidate text. Supported MIME types responseMimeType?: string; ``` +## GenerationConfig.responseModalities + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Generation modalities to be returned in generation responses. + +- Multimodal response generation is only supported in `gemini-2.0-flash-exp`, not `gemini-2.0-flash`. - Only image generation (`ResponseModality.IMAGE`) is supported. + +Signature: + +```typescript +responseModalities?: ResponseModality[]; +``` + ## GenerationConfig.responseSchema Output response schema of the generated candidate text. This value can be a class generated with a [Schema](./vertexai.schema.md#schema_class) static method like `Schema.string()` or `Schema.object()` or it can be a plain JS object matching the [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) interface.
Note: This only applies when the specified `responseMIMEType` supports a schema; currently this is limited to `application/json` and `text/x.enum`. diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index f67254eef2..47d45a492e 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -125,12 +125,14 @@ The Vertex AI in Firebase Web SDK. | Variable | Description | | --- | --- | | [POSSIBLE\_ROLES](./vertexai.md#possible_roles) | Possible roles. | +| [ResponseModality](./vertexai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | ## Type Aliases | Type Alias | Description | | --- | --- | | [Part](./vertexai.md#part) | Content part - includes text, image/video, or function call/response part types. | +| [ResponseModality](./vertexai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | | [Role](./vertexai.md#role) | Role is the producer of the content. | | [Tool](./vertexai.md#tool) | Defines a tool that model can call to access external knowledge. | | [TypedSchema](./vertexai.md#typedschema) | A type that includes all specific Schema types. | @@ -223,6 +225,22 @@ Possible roles. POSSIBLE_ROLES: readonly ["user", "model", "function", "system"] ``` +## ResponseModality + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Generation modalities to be returned in generation responses. + +Signature: + +```typescript +ResponseModality: { + readonly TEXT: "TEXT"; + readonly IMAGE: "IMAGE"; +} +``` + ## Part Content part - includes text, image/video, or function call/response part types. @@ -233,6 +251,19 @@ Content part - includes text, image/video, or function call/response part types. export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; ``` +## ResponseModality + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Generation modalities to be returned in generation responses. + +Signature: + +```typescript +export type ResponseModality = (typeof ResponseModality)[keyof typeof ResponseModality]; +``` + ## Role Role is the producer of the content. diff --git a/packages/vertexai/src/requests/response-helpers.test.ts b/packages/vertexai/src/requests/response-helpers.test.ts index 5371d04025..97dd2f9fe3 100644 --- a/packages/vertexai/src/requests/response-helpers.test.ts +++ b/packages/vertexai/src/requests/response-helpers.test.ts @@ -29,6 +29,7 @@ import { FinishReason, GenerateContentResponse, ImagenGCSImage, + InlineDataPart, ImagenInlineImage } from '../types'; import { getMockResponse } from '../../test-utils/mock-response'; @@ -132,6 +133,44 @@ const fakeResponseMixed3: GenerateContentResponse = { ] }; +const inlineDataPart1: InlineDataPart = { + inlineData: { + mimeType: 'image/png', + data: 'base64encoded...' + } +}; + +const inlineDataPart2: InlineDataPart = { + inlineData: { + mimeType: 'image/jpeg', + data: 'anotherbase64...' + } +}; + +const fakeResponseInlineData: GenerateContentResponse = { + candidates: [ + { + index: 0, + content: { + role: 'model', + parts: [inlineDataPart1, inlineDataPart2] + } + } + ] +}; + +const fakeResponseTextAndInlineData: GenerateContentResponse = { + candidates: [ + { + index: 0, + content: { + role: 'model', + parts: [{ text: 'Describe this:' }, inlineDataPart1] + } + } + ] +}; + const badFakeResponse: GenerateContentResponse = { promptFeedback: { blockReason: BlockReason.SAFETY, @@ -148,6 +187,7 @@ describe('response-helpers methods', () => { const enhancedResponse = addHelpers(fakeResponseText); expect(enhancedResponse.text()).to.equal('Some text and some more text'); expect(enhancedResponse.functionCalls()).to.be.undefined; + expect(enhancedResponse.inlineDataParts()).to.be.undefined; }); it('good response functionCall', async () => { const enhancedResponse = addHelpers(fakeResponseFunctionCall); @@ -155,6 +195,7 @@ describe('response-helpers methods', () => { expect(enhancedResponse.functionCalls()).to.deep.equal([ functionCallPart1.functionCall ]); + expect(enhancedResponse.inlineDataParts()).to.be.undefined; }); it('good response functionCalls', async () => { const enhancedResponse = addHelpers(fakeResponseFunctionCalls); @@ -163,6 +204,7 @@ describe('response-helpers methods', () => { functionCallPart1.functionCall, functionCallPart2.functionCall ]); + expect(enhancedResponse.inlineDataParts()).to.be.undefined; }); it('good response text/functionCall', async () => { const enhancedResponse = addHelpers(fakeResponseMixed1); @@ -170,6 +212,7 @@ describe('response-helpers methods', () => { functionCallPart2.functionCall ]); expect(enhancedResponse.text()).to.equal('some text'); + expect(enhancedResponse.inlineDataParts()).to.be.undefined; }); it('good response functionCall/text', async () => { const enhancedResponse = addHelpers(fakeResponseMixed2); @@ -177,6 +220,7 @@ describe('response-helpers methods', () => { functionCallPart1.functionCall ]); expect(enhancedResponse.text()).to.equal('some text'); + expect(enhancedResponse.inlineDataParts()).to.be.undefined; }); it('good response text/functionCall/text', async () => { const enhancedResponse = addHelpers(fakeResponseMixed3); @@ -184,10 +228,30 @@ describe('response-helpers methods', () => { functionCallPart1.functionCall ]); expect(enhancedResponse.text()).to.equal('some text and more text'); + expect(enhancedResponse.inlineDataParts()).to.be.undefined; }); it('bad response safety', async () => { const enhancedResponse = addHelpers(badFakeResponse); expect(enhancedResponse.text).to.throw('SAFETY'); + expect(enhancedResponse.functionCalls).to.throw('SAFETY'); + expect(enhancedResponse.inlineDataParts).to.throw('SAFETY'); + }); + it('good response inlineData', async () => { + const enhancedResponse = addHelpers(fakeResponseInlineData); + expect(enhancedResponse.text()).to.equal(''); + expect(enhancedResponse.functionCalls()).to.be.undefined; + expect(enhancedResponse.inlineDataParts()).to.deep.equal([ + inlineDataPart1, + inlineDataPart2 + ]); + }); + it('good response text/inlineData', async () => { + const enhancedResponse = addHelpers(fakeResponseTextAndInlineData); + expect(enhancedResponse.text()).to.equal('Describe this:'); + expect(enhancedResponse.functionCalls()).to.be.undefined; + expect(enhancedResponse.inlineDataParts()).to.deep.equal([ + inlineDataPart1 + ]); }); }); describe('getBlockString', () => { diff --git a/packages/vertexai/src/requests/response-helpers.ts b/packages/vertexai/src/requests/response-helpers.ts index 6d0e3bf2a0..909d7f97b1 100644 --- a/packages/vertexai/src/requests/response-helpers.ts +++ b/packages/vertexai/src/requests/response-helpers.ts @@ -23,6 +23,7 @@ import { GenerateContentResponse, ImagenGCSImage, ImagenInlineImage, + InlineDataPart, VertexAIErrorCode } from '../types'; import { VertexAIError } from '../errors'; @@ -89,6 +90,38 @@ export function addHelpers( } return ''; }; + (response as EnhancedGenerateContentResponse).inlineDataParts = (): InlineDataPart[] | undefined => { + if (response.candidates && response.candidates.length > 0) { + if (response.candidates.length > 1) { + logger.warn( + `This response had ${response.candidates.length} ` + + `candidates. Returning data from the first candidate only. ` + + `Access response.candidates directly to use the other candidates.` + ); + } + if (hadBadFinishReason(response.candidates[0])) { + throw new VertexAIError( + VertexAIErrorCode.RESPONSE_ERROR, + `Response error: ${formatBlockErrorMessage( + response + )}. Response body stored in error.response`, + { + response + } + ); + } + return getInlineDataParts(response); + } else if (response.promptFeedback) { + throw new VertexAIError( + VertexAIErrorCode.RESPONSE_ERROR, + `Data not available. ${formatBlockErrorMessage(response)}`, + { + response + } + ); + } + return undefined; + }; (response as EnhancedGenerateContentResponse).functionCalls = () => { if (response.candidates && response.candidates.length > 0) { if (response.candidates.length > 1) { @@ -164,6 +197,31 @@ export function getFunctionCalls( } } +/** + * Returns {@link InlineDataPart}s in the first candidate if present. + * + * @internal + */ +export function getInlineDataParts( + response: GenerateContentResponse +): InlineDataPart[] | undefined { + const data: InlineDataPart[] = []; + + if (response.candidates?.[0].content?.parts) { + for (const part of response.candidates?.[0].content?.parts) { + if (part.inlineData) { + data.push(part); + } + } + } + + if (data.length > 0) { + return data; + } else { + return undefined; + } +} + const badFinishReasons = [FinishReason.RECITATION, FinishReason.SAFETY]; function hadBadFinishReason(candidate: GenerateContentCandidate): boolean { diff --git a/packages/vertexai/src/types/enums.ts b/packages/vertexai/src/types/enums.ts index a9481d40f5..d6702a0f1a 100644 --- a/packages/vertexai/src/types/enums.ts +++ b/packages/vertexai/src/types/enums.ts @@ -240,3 +240,29 @@ export enum Modality { */ DOCUMENT = 'DOCUMENT' } + +/** + * Generation modalities to be returned in generation responses. + * + * @beta + */ +export const ResponseModality = { + /** + * Text. + * @beta + */ + TEXT: 'TEXT', + /** + * Image. + * @beta + */ + IMAGE: 'IMAGE' +} as const; + +/** + * Generation modalities to be returned in generation responses. + * + * @beta + */ +export type ResponseModality = + (typeof ResponseModality)[keyof typeof ResponseModality]; diff --git a/packages/vertexai/src/types/requests.ts b/packages/vertexai/src/types/requests.ts index c15258b06d..1ba5da8d6e 100644 --- a/packages/vertexai/src/types/requests.ts +++ b/packages/vertexai/src/types/requests.ts @@ -21,7 +21,8 @@ import { FunctionCallingMode, HarmBlockMethod, HarmBlockThreshold, - HarmCategory + HarmCategory, + ResponseModality } from './enums'; import { ObjectSchemaInterface, SchemaRequest } from './schema'; @@ -95,6 +96,16 @@ export interface GenerationConfig { * this is limited to `application/json` and `text/x.enum`. */ responseSchema?: TypedSchema | SchemaRequest; + /** + * Generation modalities to be returned in generation responses. + * + * @remarks + * - Multimodal response generation is only supported in `gemini-2.0-flash-exp`, not `gemini-2.0-flash`. + * - Only image generation (`ResponseModality.IMAGE`) is supported. + * + * @beta + */ + responseModalities?: ResponseModality[]; } /** diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts index 7f68df1e67..f49d25cecf 100644 --- a/packages/vertexai/src/types/responses.ts +++ b/packages/vertexai/src/types/responses.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { Content, FunctionCall } from './content'; +import { Content, FunctionCall, InlineDataPart } from './content'; import { BlockReason, FinishReason, @@ -59,6 +59,15 @@ export interface EnhancedGenerateContentResponse * Throws if the prompt or candidate was blocked. */ text: () => string; + /** + * Aggregates and returns all {@link InlineDataPart} from the {@link GenerateContentResponse}'s + * first candidate. + * + * @returns An array of {@link InlineDataPart} containing data from the response, if available. + * + * @throws If the prompt or candidate was blocked. + */ + inlineDataParts: () => InlineDataPart[] | undefined; functionCalls: () => FunctionCall[] | undefined; } From 172bc4bacf6f82a3478d4452f20f781b942811de Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 30 Apr 2025 10:57:19 -0400 Subject: [PATCH 2/8] Add changeset --- .changeset/perfect-camels-try.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/perfect-camels-try.md diff --git a/.changeset/perfect-camels-try.md b/.changeset/perfect-camels-try.md new file mode 100644 index 0000000000..9121e082f7 --- /dev/null +++ b/.changeset/perfect-camels-try.md @@ -0,0 +1,5 @@ +--- +'@firebase/vertexai': minor +--- + +Add support for Gemini multimodal output From 755d28be0497347ba3778b5e6a93b653953d2a0e Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 30 Apr 2025 10:57:42 -0400 Subject: [PATCH 3/8] fix formatting --- packages/vertexai/src/requests/response-helpers.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/vertexai/src/requests/response-helpers.ts b/packages/vertexai/src/requests/response-helpers.ts index 909d7f97b1..d820f100a5 100644 --- a/packages/vertexai/src/requests/response-helpers.ts +++ b/packages/vertexai/src/requests/response-helpers.ts @@ -90,7 +90,9 @@ export function addHelpers( } return ''; }; - (response as EnhancedGenerateContentResponse).inlineDataParts = (): InlineDataPart[] | undefined => { + (response as EnhancedGenerateContentResponse).inlineDataParts = (): + | InlineDataPart[] + | undefined => { if (response.candidates && response.candidates.length > 0) { if (response.candidates.length > 1) { logger.warn( From f0fbe8be794fb61990e81239cbab619045bdc20d Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 30 Apr 2025 11:23:56 -0400 Subject: [PATCH 4/8] add minor firebase bump --- .changeset/perfect-camels-try.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changeset/perfect-camels-try.md b/.changeset/perfect-camels-try.md index 9121e082f7..409a598531 100644 --- a/.changeset/perfect-camels-try.md +++ b/.changeset/perfect-camels-try.md @@ -1,4 +1,5 @@ --- +'firebase': minor '@firebase/vertexai': minor --- From 2d498c2f6f387d01ef04d17cc1c1e8f81ed846e0 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 30 Apr 2025 13:11:20 -0400 Subject: [PATCH 5/8] update docs --- common/api-review/vertexai.api.md | 2 +- docs-devsite/vertexai.enhancedgeneratecontentresponse.md | 6 +++--- docs-devsite/vertexai.generationconfig.md | 6 +----- packages/vertexai/src/types/responses.ts | 2 +- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 2c06a5b67b..f9cf3dac5b 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -124,7 +124,7 @@ export { Date_2 as Date } export interface EnhancedGenerateContentResponse extends GenerateContentResponse { // (undocumented) functionCalls: () => FunctionCall[] | undefined; - inlineData: () => GenerativeContentBlob[] | undefined; + inlineDataParts: () => InlineDataPart[] | undefined; text: () => string; } diff --git a/docs-devsite/vertexai.enhancedgeneratecontentresponse.md b/docs-devsite/vertexai.enhancedgeneratecontentresponse.md index 338b6b17f5..e46e5a4f81 100644 --- a/docs-devsite/vertexai.enhancedgeneratecontentresponse.md +++ b/docs-devsite/vertexai.enhancedgeneratecontentresponse.md @@ -24,7 +24,7 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse | Property | Type | Description | | --- | --- | --- | | [functionCalls](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsefunctioncalls) | () => [FunctionCall](./vertexai.functioncall.md#functioncall_interface)\[\] \| undefined | | -| [inlineData](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponseinlinedata) | () => [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface)\[\] \| undefined | Aggregates and returns all [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) from the [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface)'s first candidate. | +| [inlineDataParts](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponseinlinedataparts) | () => [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface)\[\] \| undefined | Aggregates and returns all [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) from the [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface)'s first candidate. | | [text](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsetext) | () => string | Returns the text string from the response, if available. Throws if the prompt or candidate was blocked. | ## EnhancedGenerateContentResponse.functionCalls @@ -35,14 +35,14 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse functionCalls: () => FunctionCall[] | undefined; ``` -## EnhancedGenerateContentResponse.inlineData +## EnhancedGenerateContentResponse.inlineDataParts Aggregates and returns all [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) from the [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface)'s first candidate. Signature: ```typescript -inlineData: () => GenerativeContentBlob[] | undefined; +inlineDataParts: () => InlineDataPart[] | undefined; ``` ## EnhancedGenerateContentResponse.text diff --git a/docs-devsite/vertexai.generationconfig.md b/docs-devsite/vertexai.generationconfig.md index f1c54c1100..51ca84a627 100644 --- a/docs-devsite/vertexai.generationconfig.md +++ b/docs-devsite/vertexai.generationconfig.md @@ -27,12 +27,8 @@ export interface GenerationConfig | [maxOutputTokens](./vertexai.generationconfig.md#generationconfigmaxoutputtokens) | number | | | [presencePenalty](./vertexai.generationconfig.md#generationconfigpresencepenalty) | number | | | [responseMimeType](./vertexai.generationconfig.md#generationconfigresponsemimetype) | string | Output response MIME type of the generated candidate text. Supported MIME types are text/plain (default, text output), application/json (JSON response in the candidates), and text/x.enum. | -<<<<<<< HEAD -| [responseSchema](./vertexai.generationconfig.md#generationconfigresponseschema) | [TypedSchema](./vertexai.md#typedschema) \| [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) | Output response schema of the generated candidate text. This value can be a class generated with a [Schema](./vertexai.schema.md#schema_class) static method like Schema.string() or Schema.object() or it can be a plain JS object matching the [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) interface.
Note: This only applies when the specified responseMIMEType supports a schema; currently this is limited to application/json and text/x.enum. | -======= | [responseModalities](./vertexai.generationconfig.md#generationconfigresponsemodalities) | [ResponseModality](./vertexai.md#responsemodality)\[\] | (Public Preview) Generation modalities to be returned in generation responses. | -| [responseSchema](./vertexai.generationconfig.md#generationconfigresponseschema) | [TypedSchema](./vertexai.md#typedschema) \| [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) | Output response schema of the generated candidate text. This value can be a class generated with a [Schema](./vertexai.schema.md#schema_class) static method like Schema.string() or Schema.object() or it can be a plain JS object matching the [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) interface.
Note: This only applies when the specified responseMIMEType supports a schema; currently this is limited to application/json and text/x.enum. | ->>>>>>> 4f7f1ecb1 (feat(vertexai): Gemini multimodal output) +| [responseSchema](./vertexai.generationconfig.md#generationconfigresponseschema) | [TypedSchema](./vertexai.md#typedschema) \| [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) | Output response schema of the generated candidate text. This value can be a class generated with a [Schema](./vertexai.schema.md#schema_class) static method like Schema.string() or Schema.object() or it can be a plain JS object matching the [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) interface.
Note: This only applies when the specified responseMIMEType supports a schema; currently this is limited to application/json and text/x.enum. | | [stopSequences](./vertexai.generationconfig.md#generationconfigstopsequences) | string\[\] | | | [temperature](./vertexai.generationconfig.md#generationconfigtemperature) | number | | | [topK](./vertexai.generationconfig.md#generationconfigtopk) | number | | diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts index f49d25cecf..739a0d4409 100644 --- a/packages/vertexai/src/types/responses.ts +++ b/packages/vertexai/src/types/responses.ts @@ -63,7 +63,7 @@ export interface EnhancedGenerateContentResponse * Aggregates and returns all {@link InlineDataPart} from the {@link GenerateContentResponse}'s * first candidate. * - * @returns An array of {@link InlineDataPart} containing data from the response, if available. + * @returns An array of {@link InlineDataPart}s containing data from the response, if available. * * @throws If the prompt or candidate was blocked. */ From fea4dda75b309d510ea26419d88bb6f378b6f053 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 2 May 2025 16:45:49 -0400 Subject: [PATCH 6/8] update inlineDataParts docs to add plural --- docs-devsite/vertexai.enhancedgeneratecontentresponse.md | 4 ++-- packages/vertexai/src/types/responses.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs-devsite/vertexai.enhancedgeneratecontentresponse.md b/docs-devsite/vertexai.enhancedgeneratecontentresponse.md index e46e5a4f81..b557219bf8 100644 --- a/docs-devsite/vertexai.enhancedgeneratecontentresponse.md +++ b/docs-devsite/vertexai.enhancedgeneratecontentresponse.md @@ -24,7 +24,7 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse | Property | Type | Description | | --- | --- | --- | | [functionCalls](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsefunctioncalls) | () => [FunctionCall](./vertexai.functioncall.md#functioncall_interface)\[\] \| undefined | | -| [inlineDataParts](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponseinlinedataparts) | () => [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface)\[\] \| undefined | Aggregates and returns all [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) from the [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface)'s first candidate. | +| [inlineDataParts](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponseinlinedataparts) | () => [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface)\[\] \| undefined | Aggregates and returns all [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface)s from the [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface)'s first candidate. | | [text](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsetext) | () => string | Returns the text string from the response, if available. Throws if the prompt or candidate was blocked. | ## EnhancedGenerateContentResponse.functionCalls @@ -37,7 +37,7 @@ functionCalls: () => FunctionCall[] | undefined; ## EnhancedGenerateContentResponse.inlineDataParts -Aggregates and returns all [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) from the [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface)'s first candidate. +Aggregates and returns all [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface)s from the [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface)'s first candidate. Signature: diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts index 739a0d4409..e4a247bec4 100644 --- a/packages/vertexai/src/types/responses.ts +++ b/packages/vertexai/src/types/responses.ts @@ -60,7 +60,7 @@ export interface EnhancedGenerateContentResponse */ text: () => string; /** - * Aggregates and returns all {@link InlineDataPart} from the {@link GenerateContentResponse}'s + * Aggregates and returns all {@link InlineDataPart}s from the {@link GenerateContentResponse}'s * first candidate. * * @returns An array of {@link InlineDataPart}s containing data from the response, if available. From b898f357242bab8fc616b460e01f6d169228e209 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Mon, 5 May 2025 10:25:44 -0400 Subject: [PATCH 7/8] Refer to docs for gemini model support --- docs-devsite/vertexai.generationconfig.md | 2 +- packages/vertexai/src/types/requests.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-devsite/vertexai.generationconfig.md b/docs-devsite/vertexai.generationconfig.md index 51ca84a627..afb50a98e9 100644 --- a/docs-devsite/vertexai.generationconfig.md +++ b/docs-devsite/vertexai.generationconfig.md @@ -83,7 +83,7 @@ responseMimeType?: string; Generation modalities to be returned in generation responses. -- Multimodal response generation is only supported in `gemini-2.0-flash-exp`, not `gemini-2.0-flash`. - Only image generation (`ResponseModality.IMAGE`) is supported. +- Multimodal response generation is only supported in some Gemini models and versions; see [model versions](https://firebase.google.com/docs/vertex-ai/models). - Only image generation (`ResponseModality.IMAGE`) is supported. Signature: diff --git a/packages/vertexai/src/types/requests.ts b/packages/vertexai/src/types/requests.ts index 1ba5da8d6e..716ca78b2d 100644 --- a/packages/vertexai/src/types/requests.ts +++ b/packages/vertexai/src/types/requests.ts @@ -100,7 +100,7 @@ export interface GenerationConfig { * Generation modalities to be returned in generation responses. * * @remarks - * - Multimodal response generation is only supported in `gemini-2.0-flash-exp`, not `gemini-2.0-flash`. + * - Multimodal response generation is only supported in some Gemini models and versions; see {@link https://firebase.google.com/docs/vertex-ai/models | model versions}. * - Only image generation (`ResponseModality.IMAGE`) is supported. * * @beta From de07f12f12d82e518c9cbc2fbba456e405c80008 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Thu, 8 May 2025 13:31:57 -0400 Subject: [PATCH 8/8] Fix docs --- docs-devsite/vertexai.generationconfig.md | 2 +- packages/vertexai/src/types/requests.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-devsite/vertexai.generationconfig.md b/docs-devsite/vertexai.generationconfig.md index afb50a98e9..360ef70941 100644 --- a/docs-devsite/vertexai.generationconfig.md +++ b/docs-devsite/vertexai.generationconfig.md @@ -83,7 +83,7 @@ responseMimeType?: string; Generation modalities to be returned in generation responses. -- Multimodal response generation is only supported in some Gemini models and versions; see [model versions](https://firebase.google.com/docs/vertex-ai/models). - Only image generation (`ResponseModality.IMAGE`) is supported. +- Multimodal response generation is only supported by some Gemini models and versions; see [model versions](https://firebase.google.com/docs/vertex-ai/models). - Only image generation (`ResponseModality.IMAGE`) is supported. Signature: diff --git a/packages/vertexai/src/types/requests.ts b/packages/vertexai/src/types/requests.ts index 716ca78b2d..ee45b63667 100644 --- a/packages/vertexai/src/types/requests.ts +++ b/packages/vertexai/src/types/requests.ts @@ -100,7 +100,7 @@ export interface GenerationConfig { * Generation modalities to be returned in generation responses. * * @remarks - * - Multimodal response generation is only supported in some Gemini models and versions; see {@link https://firebase.google.com/docs/vertex-ai/models | model versions}. + * - Multimodal response generation is only supported by some Gemini models and versions; see {@link https://firebase.google.com/docs/vertex-ai/models | model versions}. * - Only image generation (`ResponseModality.IMAGE`) is supported. * * @beta