[Security Assistant] Fix Inference endpoint setup (elastic#219024)

## Summary

Fixes an issue introduced in version 8.18.0, where the system switched
to using the default inference endpoint and, as part of this change,
cleaned up the dedicated inference endpoint. However, `Index entries` in
the Knowledge Base might still reference the dedicated endpoint. This PR
addresses the problem by deleting the dedicated endpoint only when it is
no longer in use, and adds logic to recreate the endpoint if any `Index
entries` still point to it.
Additionally, a new `responseTimeout` parameter has been added to the
plugin configuration. This allows controlling the response timeout in
the Security Assistant, which can be useful when running local models.


Testing guideline:
1. Set up version `8.17.5` using
[QAF](https://docs.elastic.dev/appex-qa/qaf/getting-started)
```qaf deployments create --stack-version 8.17.5 --no-es-ssl --name upgrade-test-elser --bootstrap-password changeme```

2. Set up Knowledge base http://localhost:5601/app/management/kibana/securityAiAssistantManagement?tab=knowledge_base

3. Add custom index with `semantic_text` using `elastic-security-ai-assistant-elser2`

4. Add KB `Index entry` using this index

5. Upgrade to `8.18.0`
```qaf deployments upgrade upgrade-test-elser --stack-version 8.18.0
--full-cluster-restart```

6. Check if `elastic-security-ai-assistant-elser2` was deleted in
http://localhost:5601/app/ml/overview

7. Stop kibana 
````qaf deployments stop-kibana upgrade-test-elser```

8. Run kibana on this branch https://github.com/patrykkopycinski/kibana/tree/fix/security-inference-endpoint-8-18

9. Check if `elastic-security-ai-assistant-elser2` was recreated http://localhost:5601/app/ml/overview

Author: patrykkopycinski

x-pack/platform/packages/shared/kbn-elastic-assistant-common/constants.ts

@@ -125,16 +125,3 @@ export const ATTACK_DISCOVERY_ALERTS_COMMON_INDEX_PREFIX =
  */
 export const ATTACK_DISCOVERY_ALERTS_AD_HOC_INDEX_RESOURCE_PREFIX =
   `${ATTACK_DISCOVERY_ALERTS_COMMON_INDEX_PREFIX}-ad-hoc` as const;
-
-/**
- * The server timeout is set to 4 minutes to allow for long-running requests.
- * The allows slower LLMs (like Llama 3.1 70B) and complex tasks such as ESQL generation to complete
- * without being interrupted.
- */
-export const INVOKE_LLM_SERVER_TIMEOUT = 4 * 60 * 1000; // 4 minutes
-/**
- * The client timeout is set to 3 seconds less than the server timeout to prevent
- * the `core-http-browser` from retrying the request.
- *
- */
-export const INVOKE_LLM_CLIENT_TIMEOUT = INVOKE_LLM_SERVER_TIMEOUT - 3000; // 4 minutes - 3 second

x-pack/platform/packages/shared/kbn-elastic-assistant/impl/assistant/api/chat_complete/use_chat_complete.ts

@@ -6,11 +6,10 @@
  */
 
 import { useCallback, useMemo, useRef, useState } from 'react';
-import { INVOKE_LLM_CLIENT_TIMEOUT, PromptIds, Replacements } from '@kbn/elastic-assistant-common';
+import { PromptIds, Replacements } from '@kbn/elastic-assistant-common';
 import { HttpFetchQuery } from '@kbn/core-http-browser';
 import { ChatCompleteResponse, postChatComplete } from './post_chat_complete';
 import { useAssistantContext, useLoadConnectors } from '../../../..';
-import { FETCH_MESSAGE_TIMEOUT_ERROR } from '../../use_send_message/translations';
 
 interface SendMessageProps {
   message: string;
@@ -39,11 +38,6 @@ export const useChatComplete = ({ connectorId }: { connectorId: string }): UseCh
     async ({ message, promptIds, replacements, query }: SendMessageProps) => {
       setIsLoading(true);
 
-      const timeoutId = setTimeout(() => {
-        abortController.current.abort(FETCH_MESSAGE_TIMEOUT_ERROR);
-        abortController.current = new AbortController();
-      }, INVOKE_LLM_CLIENT_TIMEOUT);
-
       try {
         return await postChatComplete({
           actionTypeId,
@@ -58,7 +52,6 @@ export const useChatComplete = ({ connectorId }: { connectorId: string }): UseCh
           traceOptions,
         });
       } finally {
-        clearTimeout(timeoutId);
         setIsLoading(false);
       }
     },

x-pack/platform/packages/shared/kbn-elastic-assistant/impl/assistant/use_send_message/index.tsx

@@ -7,11 +7,10 @@
 
 import { HttpSetup } from '@kbn/core-http-browser';
 import { useCallback, useRef, useState } from 'react';
-import { ApiConfig, INVOKE_LLM_CLIENT_TIMEOUT, Replacements } from '@kbn/elastic-assistant-common';
+import { ApiConfig, Replacements } from '@kbn/elastic-assistant-common';
 import moment from 'moment';
 import { useAssistantContext } from '../../assistant_context';
 import { fetchConnectorExecuteAction, FetchConnectorExecuteResponse } from '../api';
-import * as i18n from './translations';
 
 interface SendMessageProps {
   apiConfig: ApiConfig;
@@ -40,11 +39,6 @@ export const useSendMessage = (): UseSendMessage => {
     async ({ apiConfig, http, message, conversationId, replacements }: SendMessageProps) => {
       setIsLoading(true);
 
-      const timeoutId = setTimeout(() => {
-        abortController.current.abort(i18n.FETCH_MESSAGE_TIMEOUT_ERROR);
-        abortController.current = new AbortController();
-      }, INVOKE_LLM_CLIENT_TIMEOUT);
-
       try {
         return await fetchConnectorExecuteAction({
           conversationId,
@@ -62,7 +56,6 @@ export const useSendMessage = (): UseSendMessage => {
           },
         });
       } finally {
-        clearTimeout(timeoutId);
         setIsLoading(false);
       }
     },

x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.ts

@@ -125,6 +125,7 @@ describe('PackageInstaller', () => {
 
       expect(populateIndexMock).toHaveBeenCalledTimes(1);
       expect(populateIndexMock).toHaveBeenCalledWith({
+        elserInferenceId: defaultInferenceEndpoints.ELSER,
         indexName,
         archive: zipArchive,
         manifestVersion: TEST_FORMAT_VERSION,

x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts

@@ -191,6 +191,7 @@ export class PackageInstaller {
         archive: zipArchive,
         esClient: this.esClient,
         log: this.log,
+        elserInferenceId: this.elserInferenceId,
       });
       await this.productDocClient.setInstallationSuccessful(productName, indexName);
 

x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts

@@ -19,12 +19,14 @@ export const populateIndex = async ({
   manifestVersion,
   archive,
   log,
+  elserInferenceId,
 }: {
   esClient: ElasticsearchClient;
   indexName: string;
   manifestVersion: string;
   archive: ZipArchive;
   log: Logger;
+  elserInferenceId?: string;
 }) => {
   log.debug(`Starting populating index ${indexName}`);
 
@@ -36,7 +38,13 @@ export const populateIndex = async ({
     const entryPath = contentEntries[i];
     log.debug(`Indexing content for entry ${entryPath}`);
     const contentBuffer = await archive.getEntryContent(entryPath);
-    await indexContentFile({ indexName, esClient, contentBuffer, legacySemanticText });
+    await indexContentFile({
+      indexName,
+      esClient,
+      contentBuffer,
+      legacySemanticText,
+      elserInferenceId,
+    });
   }
 
   log.debug(`Done populating index ${indexName}`);

x-pack/platform/plugins/shared/inference/server/tasks/nl_to_esql/task.ts

@@ -68,8 +68,8 @@ export function naturalLanguageToEsql<TToolOptions extends ToolOptions>({
         switchMap((documentationEvent) => {
           return askLlmToRespond({
             documentationRequest: {
-              commands: documentationEvent.output.commands,
-              functions: documentationEvent.output.functions,
+              commands: documentationEvent.output?.commands,
+              functions: documentationEvent.output?.functions,
             },
           });
         })

x-pack/solutions/security/plugins/elastic_assistant/server/__mocks__/data_clients.mock.ts

@@ -98,7 +98,6 @@ const createKnowledgeBaseDataClientMock = () => {
     getRequiredKnowledgeBaseDocumentEntries: jest.fn(),
     getWriter: jest.fn().mockResolvedValue({ bulk: jest.fn() }),
     isInferenceEndpointExists: jest.fn(),
-    getInferenceEndpointId: jest.fn(),
     isModelInstalled: jest.fn(),
     isSecurityLabsDocsLoaded: jest.fn(),
     isSetupAvailable: jest.fn(),

x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.test.ts

@@ -162,6 +162,20 @@ describe('AIAssistantKnowledgeBaseDataClient', () => {
 
   describe('isInferenceEndpointExists', () => {
     it('returns true when the model is fully allocated and started in ESS', async () => {
+      esClientMock.inference.get.mockResolvedValue({
+        endpoints: [
+          {
+            service_settings: {
+              model_id: 'elser-id',
+              deployment_id: ASSISTANT_ELSER_INFERENCE_ID,
+              allocation_status: { state: 'fully_allocated' },
+            },
+            inference_id: ASSISTANT_ELSER_INFERENCE_ID,
+            task_type: 'sparse_embedding',
+            service: 'elasticsearch',
+          },
+        ],
+      });
       const client = new AIAssistantKnowledgeBaseDataClient(mockOptions);
       trainedModelsProviderMock.getTrainedModelsStats.mockResolvedValueOnce({
         trained_model_stats: [