diff --git a/core/llm/autodetect.ts b/core/llm/autodetect.ts index a2e13a75d82..cf0630c5932 100644 --- a/core/llm/autodetect.ts +++ b/core/llm/autodetect.ts @@ -201,6 +201,9 @@ function modelSupportsReasoning( ) { return true; } + if (model.model.includes("command-a-reasoning")) { + return true; + } if (model.model.includes("deepseek-r")) { return true; } diff --git a/core/llm/llms/Cohere.ts b/core/llm/llms/Cohere.ts index ca12161ad47..b0a27b6cf8c 100644 --- a/core/llm/llms/Cohere.ts +++ b/core/llm/llms/Cohere.ts @@ -4,10 +4,10 @@ import { Chunk, CompletionOptions, LLMOptions, - MessageContent, } from "../../index.js"; import { renderChatMessage, stripImages } from "../../util/messageContent.js"; import { BaseLLM } from "../index.js"; +import { DEFAULT_REASONING_TOKENS } from "../constants.js"; class Cohere extends BaseLLM { static providerName = "cohere"; @@ -19,7 +19,6 @@ class Cohere extends BaseLLM { private _convertMessages(msgs: ChatMessage[]): any[] { const messages = []; - let lastToolPlan: MessageContent | undefined; for (const m of msgs) { if (!m.content) { continue; @@ -48,36 +47,48 @@ class Cohere extends BaseLLM { }); break; case "thinking": - lastToolPlan = m.content; + messages.push({ + role: "assistant", + content: [ + { + type: "thinking", + thinking: m.content, + }, + ], + }); break; case "assistant": + let msg: any; + if (messages.at(-1)?.content[0]?.thinking) { + msg = messages.pop(); + } else { + msg = { + role: m.role, + content: [], + }; + } + if (m.toolCalls) { - if (!lastToolPlan) { - throw new Error("No tool plan found"); + msg.tool_calls = m.toolCalls.map((toolCall) => ({ + id: toolCall.id, + type: "function", + function: { + name: toolCall.function?.name, + arguments: toolCall.function?.arguments, + }, + })); + } else { + if (typeof m.content === "string") { + msg.content.push({ + type: "text", + text: m.content, + }); + } else { + msg.content.push(...m.content); } - messages.push({ - role: m.role, - tool_calls: m.toolCalls.map((toolCall) => ({ - id: toolCall.id, - type: "function", - function: { - name: toolCall.function?.name, - arguments: toolCall.function?.arguments, - }, - })), - // Ideally the tool plan would be in this message, but it is - // split in another, usually the previous, this one's content is - // a space. - // tool_plan: m.content, - tool_plan: lastToolPlan, - }); - lastToolPlan = undefined; - break; } - messages.push({ - role: m.role, - content: m.content, - }); + + messages.push(msg); break; case "system": messages.push({ @@ -110,6 +121,15 @@ class Cohere extends BaseLLM { stop_sequences: options.stop?.slice(0, Cohere.maxStopSequences), frequency_penalty: options.frequencyPenalty, presence_penalty: options.presencePenalty, + thinking: options.reasoning + ? { + type: "enabled" as const, + token_budget: + options.reasoningBudgetTokens ?? DEFAULT_REASONING_TOKENS, + } + : // Reasoning is enabled by default for models that support it. + // https://docs.cohere.com/reference/chat-stream#request.body.thinking + { type: "disabled" as const }, tools: options.tools?.map((tool) => ({ type: "function", function: { @@ -159,14 +179,17 @@ class Cohere extends BaseLLM { if (options.stream === false) { const data = await resp.json(); + for (const content of data.message.content) { + if (content.thinking) { + yield { role: "thinking", content: content.thinking }; + continue; + } + yield { role: "assistant", content: content.text }; + } if (data.message.tool_calls) { - yield { - // Use the "thinking" role for `tool_plan`, since there is no such - // role in the Cohere API at the moment and it is a "a - // chain-of-thought style reflection". - role: "thinking", - content: data.message.tool_plan, - }; + if (data.message.tool_plan) { + yield { role: "thinking", content: data.message.tool_plan }; + } yield { role: "assistant", content: "", @@ -181,7 +204,6 @@ class Cohere extends BaseLLM { }; return; } - yield { role: "assistant", content: data.message.content[0].text }; return; } @@ -192,6 +214,13 @@ class Cohere extends BaseLLM { switch (value.type) { // https://docs.cohere.com/v2/docs/streaming#content-delta case "content-delta": + if (value.delta.message.content.thinking) { + yield { + role: "thinking", + content: value.delta.message.content.thinking, + }; + break; + } yield { role: "assistant", content: value.delta.message.content.text, @@ -199,9 +228,6 @@ class Cohere extends BaseLLM { break; // https://docs.cohere.com/reference/chat-stream#request.body.messages.assistant.tool_plan case "tool-plan-delta": - // Use the "thinking" role for `tool_plan`, since there is no such - // role in the Cohere API at the moment and it is a "a - // chain-of-thought style reflection". yield { role: "thinking", content: value.delta.message.tool_plan, diff --git a/core/llm/llms/Cohere.vitest.ts b/core/llm/llms/Cohere.vitest.ts new file mode 100644 index 00000000000..074fd5357cf --- /dev/null +++ b/core/llm/llms/Cohere.vitest.ts @@ -0,0 +1,845 @@ +import { afterEach, describe, expect, test, vi } from "vitest"; +import { ILLM } from "../../index.js"; +import Cohere from "./Cohere.js"; + +interface LlmTestCase { + llm: ILLM; + methodToTest: keyof ILLM; + params: any[]; + expectedRequest: { + url: string; + method: string; + headers?: Record; + body?: Record; + }; + mockResponse?: any; + mockStream?: any[]; +} + +function createMockStream(mockStream: any[]) { + const encoder = new TextEncoder(); + return new ReadableStream({ + start(controller) { + for (const chunk of mockStream) { + controller.enqueue( + encoder.encode( + `data: ${ + typeof chunk === "string" ? chunk : JSON.stringify(chunk) + }\n\n`, + ), + ); + } + controller.close(); + }, + }); +} + +function setupMockFetch(mockResponse?: any, mockStream?: any[]) { + const mockFetch = vi.fn(); + + if (mockStream) { + const stream = createMockStream(mockStream); + mockFetch.mockResolvedValue( + new Response(stream, { + headers: { + "Content-Type": "text/event-stream", + }, + }), + ); + } else { + mockFetch.mockResolvedValue( + new Response(JSON.stringify(mockResponse), { + headers: { "Content-Type": "application/json" }, + }), + ); + } + + return mockFetch; +} + +function setupReadableStreamPolyfill() { + // This can be removed if https://github.com/nodejs/undici/issues/2888 is resolved + // @ts-ignore + const originalFrom = ReadableStream.from; + // @ts-ignore + ReadableStream.from = (body) => { + if (body?.source) { + return body; + } + return originalFrom(body); + }; +} + +async function executeLlmMethod( + llm: ILLM, + methodToTest: keyof ILLM, + params: any[], +) { + if (typeof (llm as any)[methodToTest] !== "function") { + throw new Error( + `Method ${String(methodToTest)} does not exist on the LLM instance.`, + ); + } + + const result = await (llm as any)[methodToTest](...params); + if (result?.next) { + for await (const _ of result) { + } + } +} + +function assertFetchCall(mockFetch: any, expectedRequest: any) { + expect(mockFetch).toHaveBeenCalledTimes(1); + const [url, options] = mockFetch.mock.calls[0]; + + expect(url.toString()).toBe(expectedRequest.url); + expect(options.method).toBe(expectedRequest.method); + + if (expectedRequest.headers) { + expect(options.headers).toEqual( + expect.objectContaining(expectedRequest.headers), + ); + } + + if (expectedRequest.body) { + const actualBody = JSON.parse(options.body as string); + expect(actualBody).toEqual(expectedRequest.body); + } +} + +async function runLlmTest(testCase: LlmTestCase) { + const { + llm, + methodToTest, + params, + expectedRequest, + mockResponse, + mockStream, + } = testCase; + + const mockFetch = setupMockFetch(mockResponse, mockStream); + setupReadableStreamPolyfill(); + + (llm as any).fetch = mockFetch; + + await executeLlmMethod(llm, methodToTest, params); + assertFetchCall(mockFetch, expectedRequest); +} + +describe("Cohere", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + test("streamChat should send a valid request", async () => { + const cohere = new Cohere({ + apiKey: "test-api-key", + model: "command-a-03-2025", + apiBase: "https://api.cohere.com/v2/", + }); + + await runLlmTest({ + llm: cohere, + methodToTest: "streamChat", + params: [ + [{ role: "user", content: "hello" }], + new AbortController().signal, + ], + expectedRequest: { + url: "https://api.cohere.com/v2/chat", + method: "POST", + headers: { + Authorization: "Bearer test-api-key", + "Content-Type": "application/json", + }, + body: { + model: "command-a-03-2025", + max_tokens: 8192, + messages: [{ role: "user", content: "hello" }], + stream: true, + thinking: { type: "disabled" }, + }, + }, + mockStream: [ + '{"id":"94e3907b-d214-475e-a53f-ae81c76b6e43","type":"message-start","delta":{"message":{"role":"assistant","content":[],"tool_plan":"","tool_calls":[],"citations":[]}}}', + '{"type":"content-start","index":0,"delta":{"message":{"content":{"type":"text","text":""}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":"Hello!"}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":" How can I help you today?"}}}}', + '{"type":"content-end","index":0}', + '{"type":"message-end","delta":{"finish_reason":"COMPLETE","usage":{"billed_units":{"input_tokens":1,"output_tokens":9},"tokens":{"input_tokens":496,"output_tokens":11},"cached_tokens":448}}}', + ], + }); + }); + + test("streamChat should send a valid request with tool calls messages", async () => { + const cohere = new Cohere({ + apiKey: "test-api-key", + model: "command-a-03-2025", + apiBase: "https://api.cohere.com/v2/", + }); + + await runLlmTest({ + llm: cohere, + methodToTest: "streamChat", + params: [ + [ + { role: "user", content: "What's the weather in New York?" }, + { + role: "thinking", + content: "I will look up the weather in New York", + }, + { + role: "assistant", + content: " ", + toolCalls: [ + { + id: "get_weather_qm3vz6v54dmw", + function: { + name: "get_weather", + arguments: '{"location": "New York"}', + }, + }, + ], + }, + { + role: "tool", + content: "Sunny", + toolCallId: "get_weather_qm3vz6v54dmw", + }, + ], + new AbortController().signal, + ], + expectedRequest: { + url: "https://api.cohere.com/v2/chat", + method: "POST", + headers: { + Authorization: "Bearer test-api-key", + "Content-Type": "application/json", + }, + body: { + model: "command-a-03-2025", + max_tokens: 8192, + messages: [ + { role: "user", content: "What's the weather in New York?" }, + { + role: "assistant", + content: [ + { + type: "thinking", + thinking: "I will look up the weather in New York", + }, + ], + tool_calls: [ + { + id: "get_weather_qm3vz6v54dmw", + type: "function", + function: { + name: "get_weather", + arguments: '{"location": "New York"}', + }, + }, + ], + }, + { + role: "tool", + content: "Sunny", + tool_call_id: "get_weather_qm3vz6v54dmw", + }, + ], + stream: true, + thinking: { type: "disabled" }, + }, + }, + mockStream: [ + '{"id":"94e3907b-d214-475e-a53f-ae81c76b6e43","type":"message-start","delta":{"message":{"role":"assistant","content":[],"tool_plan":"","tool_calls":[],"citations":[]}}}', + '{"type":"content-start","index":0,"delta":{"message":{"content":{"type":"text","text":""}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":"It\'s sunny"}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":" in New York."}}}}', + '{"type":"content-end","index":1}', + '{"type":"message-end","delta":{"finish_reason":"COMPLETE","usage":{"billed_units":{"input_tokens":11,"output_tokens":7},"tokens":{"input_tokens":600,"output_tokens":9},"cached_tokens":448}}}', + ], + }); + }); + + test("streamChat should send a valid request with images", async () => { + const cohere = new Cohere({ + apiKey: "test-api-key", + model: "command-a-vision-07-2025", + apiBase: "https://api.cohere.com/v2/", + }); + + await runLlmTest({ + llm: cohere, + methodToTest: "streamChat", + params: [ + [ + { + role: "user", + content: [ + { + type: "text", + text: "hello", + }, + { + type: "imageUrl", + imageUrl: { + url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mMsKL5ZDwAFVAI9A6eNqwAAAABJRU5ErkJggg==", + }, + }, + ], + }, + ], + new AbortController().signal, + ], + expectedRequest: { + url: "https://api.cohere.com/v2/chat", + method: "POST", + headers: { + Authorization: "Bearer test-api-key", + "Content-Type": "application/json", + }, + body: { + model: "command-a-vision-07-2025", + max_tokens: 8192, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "hello", + }, + { + type: "image_url", + image_url: { + url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mMsKL5ZDwAFVAI9A6eNqwAAAABJRU5ErkJggg==", + }, + }, + ], + }, + ], + stream: true, + thinking: { type: "disabled" }, + }, + }, + mockStream: [ + '{"id":"94e3907b-d214-475e-a53f-ae81c76b6e43","type":"message-start","delta":{"message":{"role":"assistant","content":[],"tool_plan":"","tool_calls":[],"citations":[]}}}', + '{"type":"content-start","index":0,"delta":{"message":{"content":{"type":"text","text":""}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":"Hello!"}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":" How can I assist you today?"}}}}', + '{"type":"content-end","index":0}', + '{"type":"message-end","delta":{"finish_reason":"COMPLETE","usage":{"billed_units":{"input_tokens":260,"output_tokens":10},"tokens":{"input_tokens":497,"output_tokens":10,"image_tokens":259},"cached_tokens":480}}}', + ], + }); + }); + + test("streamChat should send a valid request with thinking messages", async () => { + const cohere = new Cohere({ + apiKey: "test-api-key", + model: "command-a-reasoning-08-2025", + apiBase: "https://api.cohere.com/v2/", + completionOptions: { + model: "command-a-reasoning-08-2025", + reasoning: true, + }, + }); + + await runLlmTest({ + llm: cohere, + methodToTest: "streamChat", + params: [ + [ + { role: "user", content: "hello" }, + { + role: "thinking", + content: + 'Okay, the user just said "hello". Let me figure out how to respond. Since they\'re greeting me, I should greet them back.', + }, + { role: "assistant", content: "Hello! How can I assist you today?" }, + { role: "user", content: "hello again" }, + ], + new AbortController().signal, + ], + expectedRequest: { + url: "https://api.cohere.com/v2/chat", + method: "POST", + headers: { + Authorization: "Bearer test-api-key", + "Content-Type": "application/json", + }, + body: { + model: "command-a-reasoning-08-2025", + max_tokens: 32768, + messages: [ + { role: "user", content: "hello" }, + { + role: "assistant", + content: [ + { + type: "thinking", + thinking: + 'Okay, the user just said "hello". Let me figure out how to respond. Since they\'re greeting me, I should greet them back.', + }, + { type: "text", text: "Hello! How can I assist you today?" }, + ], + }, + { role: "user", content: "hello again" }, + ], + stream: true, + thinking: { type: "enabled", token_budget: 2048 }, + }, + }, + mockStream: [ + '{"id":"94e3907b-d214-475e-a53f-ae81c76b6e43","type":"message-start","delta":{"message":{"role":"assistant","content":[],"tool_plan":"","tool_calls":[],"citations":[]}}}', + '{"type":"content-start","index":0,"delta":{"message":{"content":{"type":"thinking","thinking":""}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"thinking":"Alright, the user said \"hello again\". Let me check the chat history."}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"thinking":" The user first said \"hello\", and I\'ll keep it friendly and open-ended."}}}}', + '{"type":"content-end","index":0}', + '{"type":"content-start","index":1,"delta":{"message":{"content":{"type":"text","text":""}}}}', + '{"type":"content-delta","index":1,"delta":{"message":{"content":{"text":"Hello again!"}}}}', + '{"type":"content-delta","index":1,"delta":{"message":{"content":{"text":" It\'s nice to see you back. Is there something specific you need help with, or would you just like to chat?"}}}}', + '{"type":"content-end","index":1}', + '{"type":"message-end","delta":{"finish_reason":"COMPLETE","usage":{"billed_units":{"input_tokens":12,"output_tokens":138},"tokens":{"input_tokens":1437,"output_tokens":142},"cached_tokens":1344}}}', + ], + }); + }); + + test("chat should send a valid request", async () => { + const cohere = new Cohere({ + apiKey: "test-api-key", + model: "command-a-03-2025", + apiBase: "https://api.cohere.com/v2/", + }); + + await runLlmTest({ + llm: cohere, + methodToTest: "chat", + params: [ + [{ role: "user", content: "hello" }], + new AbortController().signal, + ], + expectedRequest: { + url: "https://api.cohere.com/v2/chat", + method: "POST", + headers: { + Authorization: "Bearer test-api-key", + "Content-Type": "application/json", + }, + body: { + model: "command-a-03-2025", + max_tokens: 8192, + messages: [{ role: "user", content: "hello" }], + stream: true, + thinking: { type: "disabled" }, + }, + }, + mockStream: [ + '{"id":"94e3907b-d214-475e-a53f-ae81c76b6e43","type":"message-start","delta":{"message":{"role":"assistant","content":[],"tool_plan":"","tool_calls":[],"citations":[]}}}', + '{"type":"content-start","index":0,"delta":{"message":{"content":{"type":"text","text":""}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":"Hello!"}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":" How can I help you today?"}}}}', + '{"type":"content-end","index":0}', + '{"type":"message-end","delta":{"finish_reason":"COMPLETE","usage":{"billed_units":{"input_tokens":1,"output_tokens":9},"tokens":{"input_tokens":496,"output_tokens":11},"cached_tokens":448}}}', + ], + }); + }); + + test("streamComplete should send a valid request", async () => { + const cohere = new Cohere({ + apiKey: "test-api-key", + model: "command-a-03-2025", + apiBase: "https://api.cohere.com/v2/", + }); + + await runLlmTest({ + llm: cohere, + methodToTest: "streamComplete", + params: ["Complete this: Hello", new AbortController().signal], + expectedRequest: { + url: "https://api.cohere.com/v2/chat", + method: "POST", + headers: { + Authorization: "Bearer test-api-key", + "Content-Type": "application/json", + }, + body: { + model: "command-a-03-2025", + max_tokens: 8192, + messages: [{ role: "user", content: "Complete this: Hello" }], + stream: true, + thinking: { type: "disabled" }, + }, + }, + mockStream: [ + '{"id":"94e3907b-d214-475e-a53f-ae81c76b6e43","type":"message-start","delta":{"message":{"role":"assistant","content":[],"tool_plan":"","tool_calls":[],"citations":[]}}}', + '{"type":"content-start","index":0,"delta":{"message":{"content":{"type":"text","text":""}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":" world!"}}}}', + '{"type":"content-end","index":0}', + '{"type":"message-end","delta":{"finish_reason":"COMPLETE","usage":{"billed_units":{"input_tokens":4,"output_tokens":3},"tokens":{"input_tokens":499,"output_tokens":5},"cached_tokens":448}}}', + ], + }); + }); + + test("complete should send a valid request", async () => { + const cohere = new Cohere({ + apiKey: "test-api-key", + model: "command-a-03-2025", + apiBase: "https://api.cohere.com/v2/", + }); + + await runLlmTest({ + llm: cohere, + methodToTest: "complete", + params: ["Complete this: Hello", new AbortController().signal], + expectedRequest: { + url: "https://api.cohere.com/v2/chat", + method: "POST", + headers: { + Authorization: "Bearer test-api-key", + "Content-Type": "application/json", + }, + body: { + model: "command-a-03-2025", + max_tokens: 8192, + messages: [{ role: "user", content: "Complete this: Hello" }], + stream: true, + thinking: { type: "disabled" }, + }, + }, + mockStream: [ + '{"id":"94e3907b-d214-475e-a53f-ae81c76b6e43","type":"message-start","delta":{"message":{"role":"assistant","content":[],"tool_plan":"","tool_calls":[],"citations":[]}}}', + '{"type":"content-start","index":0,"delta":{"message":{"content":{"type":"text","text":""}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":" world!"}}}}', + '{"type":"content-end","index":0}', + '{"type":"message-end","delta":{"finish_reason":"COMPLETE","usage":{"billed_units":{"input_tokens":4,"output_tokens":3},"tokens":{"input_tokens":499,"output_tokens":5},"cached_tokens":448}}}', + ], + }); + }); + + describe("Different configurations", () => { + test("should handle system message", async () => { + const cohere = new Cohere({ + apiKey: "test-api-key", + model: "command-a-03-2025", + apiBase: "https://api.cohere.com/v2/", + }); + + await runLlmTest({ + llm: cohere, + methodToTest: "streamChat", + params: [ + [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello!" }, + ], + new AbortController().signal, + ], + expectedRequest: { + url: "https://api.cohere.com/v2/chat", + method: "POST", + headers: { + Authorization: "Bearer test-api-key", + "Content-Type": "application/json", + }, + body: { + model: "command-a-03-2025", + max_tokens: 8192, + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello!" }, + ], + stream: true, + thinking: { type: "disabled" }, + }, + }, + mockStream: [ + '{"id":"94e3907b-d214-475e-a53f-ae81c76b6e43","type":"message-start","delta":{"message":{"role":"assistant","content":[],"tool_plan":"","tool_calls":[],"citations":[]}}}', + '{"type":"content-start","index":0,"delta":{"message":{"content":{"type":"text","text":""}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":"Hello!"}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":" How can I assist you today?"}}}}', + '{"type":"content-end","index":0}', + '{"type":"message-end","delta":{"finish_reason":"COMPLETE","usage":{"billed_units":{"input_tokens":10,"output_tokens":9},"tokens":{"input_tokens":539,"output_tokens":11},"cached_tokens":496}}}', + ], + }); + }); + + test("should handle tool calls with tool plan (legacy)", async () => { + const cohere = new Cohere({ + apiKey: "test-api-key", + model: "command-a-03-2025", + apiBase: "https://api.cohere.com/v2/", + }); + + const tools = [ + { + type: "function" as const, + function: { + name: "get_weather", + description: "Get the current weather", + parameters: { + type: "object", + properties: { + location: { type: "string" }, + }, + required: ["location"], + }, + }, + }, + ]; + + await runLlmTest({ + llm: cohere, + methodToTest: "streamChat", + params: [ + [{ role: "user", content: "What's the weather in New York?" }], + new AbortController().signal, + { tools }, + ], + expectedRequest: { + url: "https://api.cohere.com/v2/chat", + method: "POST", + headers: { + Authorization: "Bearer test-api-key", + "Content-Type": "application/json", + }, + body: { + model: "command-a-03-2025", + max_tokens: 8192, + messages: [ + { role: "user", content: "What's the weather in New York?" }, + ], + stream: true, + thinking: { type: "disabled" }, + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get the current weather", + parameters: { + type: "object", + properties: { + location: { type: "string" }, + }, + required: ["location"], + }, + }, + }, + ], + }, + }, + mockStream: [ + '{"id":"94e3907b-d214-475e-a53f-ae81c76b6e43","type":"message-start","delta":{"message":{"role":"assistant","content":[],"tool_plan":"","tool_calls":[],"citations":[]}}}', + '{"type":"tool-plan-delta","delta":{"message":{"tool_plan":"I will look up the weather"}}}', + '{"type":"tool-plan-delta","delta":{"message":{"tool_plan":" in New York"}}}', + '{"type":"tool-call-start","index":0,"delta":{"message":{"tool_calls":{"id":"get_weather_qm3vz6v54dmw","type":"function","function":{"name":"get_weather","arguments":""}}}}}', + '{"type":"tool-call-delta","index":0,"delta":{"message":{"tool_calls":{"function":{"arguments":"{\\"location\\": \\""}}}}', + '{"type":"tool-call-delta","index":0,"delta":{"message":{"tool_calls":{"function":{"arguments":"\\"New York\\"}"}}}}', + '{"type":"tool-call-end","index":0}', + '{"type":"message-end","delta":{"finish_reason":"TOOL_CALL","usage":{"billed_units":{"input_tokens":17,"output_tokens":19},"tokens":{"input_tokens":1434,"output_tokens":48},"cached_tokens":992}}}', + ], + }); + }); + + test("should handle tool calls with thinking disabled", async () => { + const cohere = new Cohere({ + apiKey: "test-api-key", + model: "command-a-reasoning-08-2025", + apiBase: "https://api.cohere.com/v2/", + }); + + const tools = [ + { + type: "function" as const, + function: { + name: "get_weather", + description: "Get the current weather", + parameters: { + type: "object", + properties: { + location: { type: "string" }, + }, + required: ["location"], + }, + }, + }, + ]; + + await runLlmTest({ + llm: cohere, + methodToTest: "streamChat", + params: [ + [{ role: "user", content: "What's the weather in New York?" }], + new AbortController().signal, + { tools }, + ], + expectedRequest: { + url: "https://api.cohere.com/v2/chat", + method: "POST", + headers: { + Authorization: "Bearer test-api-key", + "Content-Type": "application/json", + }, + body: { + model: "command-a-reasoning-08-2025", + max_tokens: 32768, + messages: [ + { role: "user", content: "What's the weather in New York?" }, + ], + stream: true, + thinking: { type: "disabled" }, + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get the current weather", + parameters: { + type: "object", + properties: { + location: { type: "string" }, + }, + required: ["location"], + }, + }, + }, + ], + }, + }, + mockStream: [ + '{"id":"94e3907b-d214-475e-a53f-ae81c76b6e43","type":"message-start","delta":{"message":{"role":"assistant","content":[],"tool_plan":"","tool_calls":[],"citations":[]}}}', + '{"type":"tool-call-start","index":0,"delta":{"message":{"tool_calls":{"id":"get_weather_qm3vz6v54dmw","type":"function","function":{"name":"get_weather","arguments":""}}}}}', + '{"type":"tool-call-delta","index":0,"delta":{"message":{"tool_calls":{"function":{"arguments":"{\\"location\\": \\""}}}}', + '{"type":"tool-call-delta","index":0,"delta":{"message":{"tool_calls":{"function":{"arguments":"\\"New York\\"}"}}}}', + '{"type":"tool-call-end","index":0}', + '{"type":"message-end","delta":{"finish_reason":"TOOL_CALL","usage":{"billed_units":{"input_tokens":17,"output_tokens":9},"tokens":{"input_tokens":1174,"output_tokens":36},"cached_tokens":0}}}', + ], + }); + }); + + test("should handle tool calls with thinking enabled", async () => { + const cohere = new Cohere({ + apiKey: "test-api-key", + model: "command-a-reasoning-08-2025", + apiBase: "https://api.cohere.com/v2/", + completionOptions: { + model: "command-a-reasoning-08-2025", + reasoning: true, + }, + }); + + const tools = [ + { + type: "function" as const, + function: { + name: "get_weather", + description: "Get the current weather", + parameters: { + type: "object", + properties: { + location: { type: "string" }, + }, + required: ["location"], + }, + }, + }, + ]; + + await runLlmTest({ + llm: cohere, + methodToTest: "streamChat", + params: [ + [{ role: "user", content: "What's the weather in New York?" }], + new AbortController().signal, + { tools }, + ], + expectedRequest: { + url: "https://api.cohere.com/v2/chat", + method: "POST", + headers: { + Authorization: "Bearer test-api-key", + "Content-Type": "application/json", + }, + body: { + model: "command-a-reasoning-08-2025", + max_tokens: 32768, + messages: [ + { role: "user", content: "What's the weather in New York?" }, + ], + stream: true, + thinking: { type: "enabled", token_budget: 2048 }, + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get the current weather", + parameters: { + type: "object", + properties: { + location: { type: "string" }, + }, + required: ["location"], + }, + }, + }, + ], + }, + }, + mockStream: [ + '{"id":"94e3907b-d214-475e-a53f-ae81c76b6e43","type":"message-start","delta":{"message":{"role":"assistant","content":[],"tool_plan":"","tool_calls":[],"citations":[]}}}', + '{"type":"content-start","index":0,"delta":{"message":{"content":{"type":"thinking","thinking":""}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"thinking":"Okay, the user is asking for the weather in New York. Let me check the available tools. There\'s a tool called get_weather that requires a location parameter. Since the user mentioned New York, I can use that as the location."}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"thinking":" I need to call the get_weather tool with location set to \"New York\" to retrieve the current weather data. No other tools are available, so this should be the only step needed. Once I get the weather data, I can present it to the user."}}}}', + '{"type":"content-end","index":0}', + '{"type":"tool-call-start","index":0,"delta":{"message":{"tool_calls":{"id":"get_weather_qm3vz6v54dmw","type":"function","function":{"name":"get_weather","arguments":""}}}}}', + '{"type":"tool-call-delta","index":0,"delta":{"message":{"tool_calls":{"function":{"arguments":"{\\"location\\": \\""}}}}', + '{"type":"tool-call-delta","index":0,"delta":{"message":{"tool_calls":{"function":{"arguments":"\\"New York\\"}"}}}}', + '{"type":"tool-call-end","index":0}', + '{"type":"message-end","delta":{"finish_reason":"TOOL_CALL","usage":{"billed_units":{"input_tokens":17,"output_tokens":9},"tokens":{"input_tokens":1174,"output_tokens":36},"cached_tokens":0}}}', + ], + }); + }); + + test("should handle custom max tokens", async () => { + const cohere = new Cohere({ + apiKey: "test-api-key", + model: "command-a-03-2025", + apiBase: "https://api.cohere.com/v2/", + }); + + await runLlmTest({ + llm: cohere, + methodToTest: "streamChat", + params: [ + [{ role: "user", content: "hello" }], + new AbortController().signal, + { maxTokens: 1000 }, + ], + expectedRequest: { + url: "https://api.cohere.com/v2/chat", + method: "POST", + headers: { + Authorization: "Bearer test-api-key", + "Content-Type": "application/json", + }, + body: { + model: "command-a-03-2025", + max_tokens: 1000, + messages: [{ role: "user", content: "hello" }], + stream: true, + thinking: { type: "disabled" }, + }, + }, + mockStream: [ + '{"id":"94e3907b-d214-475e-a53f-ae81c76b6e43","type":"message-start","delta":{"message":{"role":"assistant","content":[],"tool_plan":"","tool_calls":[],"citations":[]}}}', + '{"type":"content-start","index":0,"delta":{"message":{"content":{"type":"text","text":""}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":"Hello! "}}}}', + '{"type":"content-delta","index":0,"delta":{"message":{"content":{"text":"How can I help you today?"}}}}', + '{"type":"content-end","index":0}', + '{"type":"message-end","delta":{"finish_reason":"COMPLETE","usage":{"billed_units":{"input_tokens":1,"output_tokens":9},"tokens":{"input_tokens":496,"output_tokens":11},"cached_tokens":448}}}', + ], + }); + }); + }); +}); diff --git a/core/llm/toolSupport.test.ts b/core/llm/toolSupport.test.ts index ad883e4cbf1..30056f85a05 100644 --- a/core/llm/toolSupport.test.ts +++ b/core/llm/toolSupport.test.ts @@ -124,6 +124,10 @@ describe("PROVIDER_TOOL_SUPPORT", () => { describe("cohere", () => { const supportsFn = PROVIDER_TOOL_SUPPORT["cohere"]; + it("should return false for Command A Vision models", () => { + expect(supportsFn("command-a-vision")).toBe(false); + }); + it("should return true for Command models", () => { expect(supportsFn("command-r")).toBe(true); expect(supportsFn("command-a")).toBe(true); diff --git a/core/llm/toolSupport.ts b/core/llm/toolSupport.ts index 5e4f26241a0..a101779b979 100644 --- a/core/llm/toolSupport.ts +++ b/core/llm/toolSupport.ts @@ -100,7 +100,11 @@ export const PROVIDER_TOOL_SUPPORT: Record boolean> = return false; }, cohere: (model) => { - return model.toLowerCase().startsWith("command"); + const lower = model.toLowerCase(); + if (lower.startsWith("command-a-vision")) { + return false; + } + return lower.startsWith("command"); }, gemini: (model) => { // All gemini models support function calling diff --git a/docs/customize/deep-dives/model-capabilities.mdx b/docs/customize/deep-dives/model-capabilities.mdx index eca29cf04ae..d7b26dc48b2 100644 --- a/docs/customize/deep-dives/model-capabilities.mdx +++ b/docs/customize/deep-dives/model-capabilities.mdx @@ -186,6 +186,15 @@ This matrix shows which models support tool use and image input capabilities. Co | Claude 3.5 Sonnet | Yes | Yes | 200k | | Claude 3.5 Haiku | Yes | Yes | 200k | +### Cohere + +| Model | Tool Use | Image Input | Context Window | +| :------------------ | -------- | ----------- | -------------- | +| Command A | Yes | No | 256k | +| Command A Reasoning | Yes | No | 256k | +| Command A Translate | Yes | No | 8k | +| Command A Vision | No | Yes | 128k | + ### Google | Model | Tool Use | Image Input | Context Window | diff --git a/extensions/vscode/config_schema.json b/extensions/vscode/config_schema.json index 71f0f5e3fe2..111db4046da 100644 --- a/extensions/vscode/config_schema.json +++ b/extensions/vscode/config_schema.json @@ -893,7 +893,10 @@ "command-r-plus-08-2024", "command-r7b-12-2024", "command-r7b-arabic-02-2025", - "command-a-03-2025" + "command-a-03-2025", + "command-a-vision-07-2025", + "command-a-reasoning-08-2025", + "command-a-translate-08-2025" ] } } diff --git a/gui/src/pages/AddNewModel/configs/models.ts b/gui/src/pages/AddNewModel/configs/models.ts index 941b0be46e6..49a805fa4fa 100644 --- a/gui/src/pages/AddNewModel/configs/models.ts +++ b/gui/src/pages/AddNewModel/configs/models.ts @@ -1067,6 +1067,48 @@ export const models: { [key: string]: ModelPackage } = { icon: "cohere.png", isOpenSource: false, }, + commandAVision072025: { + title: "Command A Vision 07-2025", + description: + "Command A Vision is Cohere's first model capable of processing images, excelling in enterprise use cases such as analyzing charts, graphs, and diagrams, table understanding, OCR, document Q&A, and object detection.", + params: { + model: "command-a-vision-07-2025", + contextLength: 128_000, + title: "Command A Vision 07-2025", + apiKey: "", + }, + providerOptions: ["cohere"], + icon: "cohere.png", + isOpenSource: false, + }, + commandAReasoning082025: { + title: "Command A Reasoning 08-2025", + description: + "Command A Reasoning is Cohere’s first reasoning model, able to ‘think’ before generating an output in a way that allows it to perform well in certain kinds of nuanced problem-solving and agent-based tasks in 23 languages.", + params: { + model: "command-a-reasoning-08-2025", + contextLength: 256_000, + title: "Command A Reasoning 08-2025", + apiKey: "", + }, + providerOptions: ["cohere"], + icon: "cohere.png", + isOpenSource: false, + }, + commandATranslate082025: { + title: "Command A Translate 08-2025", + description: + "Command A Translate is Cohere’s state of the art machine translation model, excelling at a variety of translation tasks on 23 languages", + params: { + model: "command-a-translate-08-2025", + contextLength: 8_000, + title: "Command A Translate 08-2025", + apiKey: "", + }, + providerOptions: ["cohere"], + icon: "cohere.png", + isOpenSource: false, + }, gpt5: { title: "GPT-5", description: "OpenAI's next-generation, high-intelligence flagship model", diff --git a/packages/llm-info/src/providers/cohere.ts b/packages/llm-info/src/providers/cohere.ts index 91f8d0154b9..e7e1ce2570c 100644 --- a/packages/llm-info/src/providers/cohere.ts +++ b/packages/llm-info/src/providers/cohere.ts @@ -2,6 +2,33 @@ import { ModelProvider } from "../types.js"; export const Cohere: ModelProvider = { models: [ + { + model: "command-a-translate-08-2025", + displayName: "Command A Translate 08-2025", + contextLength: 8000, + maxCompletionTokens: 8192, + description: + "Command A Translate is Cohere’s state of the art machine translation model, excelling at a variety of translation tasks on 23 languages", + recommendedFor: ["chat"], + }, + { + model: "command-a-reasoning-08-2025", + displayName: "Command A Reasoning 08-2025", + contextLength: 256000, + maxCompletionTokens: 32768, + description: + "Command A Reasoning is Cohere’s first reasoning model, able to ‘think’ before generating an output in a way that allows it to perform well in certain kinds of nuanced problem-solving and agent-based tasks in 23 languages.", + recommendedFor: ["chat"], + }, + { + model: "command-a-vision-07-2025", + displayName: "Command A Vision 07-2025", + contextLength: 128000, + maxCompletionTokens: 8192, + description: + "Command A Vision is Cohere's first model capable of processing images, excelling in enterprise use cases such as analyzing charts, graphs, and diagrams, table understanding, OCR, document Q&A, and object detection.", + recommendedFor: ["chat"], + }, { model: "command-a-03-2025", displayName: "Command A 03-2025",