feat(gen-ai): add eval tests COMPASS-10084

packages/compass-generative-ai/package.json

@@ -49,7 +49,8 @@
     "test-watch": "npm run test -- --watch",
     "test-ci": "npm run test-cov",
     "test-ci-electron": "npm run test-electron",
-    "reformat": "npm run eslint . -- --fix && npm run prettier -- --write ."
+    "reformat": "npm run eslint . -- --fix && npm run prettier -- --write .",
+    "eval": "braintrust eval tests/evals"
   },
   "dependencies": {
     "@mongodb-js/atlas-service": "^0.73.0",
@@ -84,10 +85,13 @@
     "@types/mocha": "^9.0.0",
     "@types/react": "^17.0.5",
     "@types/sinon-chai": "^3.2.5",
+    "autoevals": "^0.0.130",
+    "braintrust": "^0.2.4",
     "chai": "^4.3.6",
     "depcheck": "^1.4.1",
     "electron-mocha": "^12.2.0",
     "mocha": "^10.2.0",
+    "mongodb-ns": "^3.1.0",
     "nyc": "^15.1.0",
     "p-queue": "^7.4.1",
     "sinon": "^9.2.3",

packages/compass-generative-ai/tests/evals/chatbot-api.ts

@@ -0,0 +1,42 @@
+import { streamText } from 'ai';
+import { createOpenAI } from '@ai-sdk/openai';
+import type {
+  ConversationEvalCaseInput,
+  ConversationTaskOutput,
+} from './types';
+
+export async function makeChatbotCall(
+  input: ConversationEvalCaseInput
+): Promise<ConversationTaskOutput> {
+  const openai = createOpenAI({
+    baseURL:
+      process.env.COMPASS_ASSISTANT_BASE_URL_OVERRIDE ??
+      'https://eval.knowledge-dev.mongodb.com/api/v1',
+    apiKey: '',
+    headers: {
+      'X-Request-Origin': 'compass-gen-ai-braintrust',
+      'User-Agent': 'mongodb-compass/x.x.x',
+    },
+  });
+  const result = streamText({
+    model: openai.responses('mongodb-slim-latest'),
+    temperature: undefined,
+    prompt: input.messages,
+    providerOptions: {
+      openai: {
+        instructions: input.instructions.content,
+        store: false,
+      },
+    },
+  });
+
+  const chunks: string[] = [];
+  for await (const chunk of result.toUIMessageStream()) {
+    if (chunk.type === 'text-delta') {
+      chunks.push(chunk.delta);
+    }
+  }
+  return {
+    messages: [{ content: chunks.join('') }],
+  };
+}