[Security Solution] [Elastic AI Assistant] LangChain integration (experimental) (elastic#164908)

## [Security Solution] [Elastic AI Assistant] LangChain integration (experimental)

This PR integrates [LangChain](https://www.langchain.com/) with the [Elastic AI Assistant](https://www.elastic.co/blog/introducing-elastic-ai-assistant) as an experimental, alternative execution path.

### How it works

- There are virtually no client side changes to the assistant, apart from a new branch in `x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx` that chooses a path based on the value of the `assistantLangChain` flag:

```typescript
    const path = assistantLangChain
      ? `/internal/elastic_assistant/actions/connector/${apiConfig?.connectorId}/_execute`
      : `/api/actions/connector/${apiConfig?.connectorId}/_execute`;
```

Execution of the LangChain chain happens server-side. The new route still executes the request via the `connectorId` in the route, but the connector won't execute the request exactly as it was sent by the client. Instead, the connector will execute one (or more) prompts that are generated by LangChain.

Requests routed to `/internal/elastic_assistant/actions/connector/${apiConfig?.connectorId}/_execute` will be processed by a new Kibana plugin located in:

```
x-pack/plugins/elastic_assistant
```

- Requests are processed in the `postActionsConnectorExecuteRoute` handler in `x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts`.

The `postActionsConnectorExecuteRoute` route handler:

1. Extracts the chat messages sent by the assistant
2. Converts the extracted messages to the format expected by LangChain
3. Passes the converted messages to `executeCustomLlmChain`

- The `executeCustomLlmChain` function in `x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts`:

1. Splits the messages into `pastMessages` and `latestMessage`, where the latter contains only the last message sent by the user
2. Wraps the conversation history in the `BufferMemory` LangChain abstraction
3. Executes the chain, kicking it off with `latestMessage`

```typescript
  const llm = new ActionsClientLlm({ actions, connectorId, request });

  const pastMessages = langchainMessages.slice(0, -1); // all but the last message
  const latestMessage = langchainMessages.slice(-1); // the last message

  const memory = new BufferMemory({
    chatHistory: new ChatMessageHistory(pastMessages),
  });

  const chain = new ConversationChain({ llm, memory });

  await chain.call({ input: latestMessage[0].content }); // kick off the chain with the last message
};
```

- When LangChain executes the chain, it will invoke `ActionsClientLlm`'s `_call` function in `x-pack/plugins/elastic_assistant/server/lib/langchain/llm/actions_client_llm.ts` one or more times.

The `_call` function's signature is defined by LangChain:

```
async _call(prompt: string): Promise<string>
```

- The contents of `prompt` are completely determined by LangChain.
- The string returned by the promise is the "answer" from the LLM

The `ActionsClientLlm` extends LangChain's LLM interface:

```typescript
export class ActionsClientLlm extends LLM
```

This let's us do additional "work" in the `_call` function:

1. Create a new assistant message using the contents of the `prompt` (`string`) argument to `_call`
2. Create a request body in the format expected by the connector
3. Create an actions client from the authenticated request context
4. Execute the actions client with the request body
5. Save the raw response from the connector, because that's what the assistant expects
6. Return the result as a plain string, as per the contact of `_call`

## Desk testing

This experimental LangChain integration may NOT be enabled via a feature flag (yet).

Set

```typescript
assistantLangChain={true}
```

in `x-pack/plugins/security_solution/public/app/app.tsx` to enable this experimental feature in development environments.

Author: andrew-goldstein

.eslintrc.js

@@ -984,6 +984,7 @@ module.exports = {
       // front end and common typescript and javascript files only
       files: [
         'x-pack/plugins/ecs_data_quality_dashboard/common/**/*.{js,mjs,ts,tsx}',
+        'x-pack/plugins/elastic_assistant/common/**/*.{js,mjs,ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant/**/*.{js,mjs,ts,tsx}',
         'x-pack/packages/security-solution/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/security_solution/public/**/*.{js,mjs,ts,tsx}',
@@ -1016,6 +1017,7 @@ module.exports = {
       // This should be a very small set as most linter rules are useful for tests as well.
       files: [
         'x-pack/plugins/ecs_data_quality_dashboard/**/*.{ts,tsx}',
+        'x-pack/plugins/elastic_assistant/**/*.{ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant/**/*.{ts,tsx}',
         'x-pack/packages/security-solution/**/*.{ts,tsx}',
         'x-pack/plugins/security_solution/**/*.{ts,tsx}',
@@ -1026,6 +1028,7 @@ module.exports = {
       ],
       excludedFiles: [
         'x-pack/plugins/ecs_data_quality_dashboard/**/*.{test,mock,test_helper}.{ts,tsx}',
+        'x-pack/plugins/elastic_assistant/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/packages/security-solution/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/plugins/security_solution/**/*.{test,mock,test_helper}.{ts,tsx}',
@@ -1042,6 +1045,7 @@ module.exports = {
       // typescript only for front and back end
       files: [
         'x-pack/plugins/ecs_data_quality_dashboard/**/*.{ts,tsx}',
+        'x-pack/plugins/elastic_assistant/**/*.{ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant/**/*.{ts,tsx}',
         'x-pack/packages/security-solution/**/*.{ts,tsx}',
         'x-pack/plugins/security_solution/**/*.{ts,tsx}',
@@ -1077,6 +1081,7 @@ module.exports = {
       // typescript and javascript for front and back end
       files: [
         'x-pack/plugins/ecs_data_quality_dashboard/**/*.{js,mjs,ts,tsx}',
+        'x-pack/plugins/elastic_assistant/**/*.{js,mjs,ts,tsx}',
         'x-pack/packages/kbn-elastic-assistant/**/*.{js,mjs,ts,tsx}',
         'x-pack/packages/security-solution/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/security_solution/**/*.{js,mjs,ts,tsx}',

.github/CODEOWNERS

@@ -340,6 +340,7 @@ packages/kbn-ecs @elastic/kibana-core @elastic/security-threat-hunting-investiga
 x-pack/packages/security-solution/ecs_data_quality_dashboard @elastic/security-threat-hunting-investigations
 x-pack/plugins/ecs_data_quality_dashboard @elastic/security-threat-hunting-investigations
 x-pack/packages/kbn-elastic-assistant @elastic/security-solution
+x-pack/plugins/elastic_assistant @elastic/security-solution
 test/plugin_functional/plugins/elasticsearch_client_plugin @elastic/kibana-core
 x-pack/test/plugin_api_integration/plugins/elasticsearch_client @elastic/kibana-core
 x-pack/plugins/embeddable_enhanced @elastic/kibana-presentation

docs/developer/plugin-list.asciidoc

@@ -528,6 +528,10 @@ Plugin server-side only. Plugin has three main functions:
 |This plugin implements (server) APIs used to render the content of the Data Quality dashboard.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/elastic_assistant/README.md[elasticAssistant]
+|This plugin implements (only) server APIs for the Elastic AI Assistant.
+
+
 |<<enhanced-embeddables-plugin>>
 |Enhances Embeddables by registering a custom factory provider. The enhanced factory provider
 adds dynamic actions to every embeddables state, in order to support drilldowns.

package.json

@@ -383,6 +383,7 @@
     "@kbn/ecs-data-quality-dashboard": "link:x-pack/packages/security-solution/ecs_data_quality_dashboard",
     "@kbn/ecs-data-quality-dashboard-plugin": "link:x-pack/plugins/ecs_data_quality_dashboard",
     "@kbn/elastic-assistant": "link:x-pack/packages/kbn-elastic-assistant",
+    "@kbn/elastic-assistant-plugin": "link:x-pack/plugins/elastic_assistant",
     "@kbn/elasticsearch-client-plugin": "link:test/plugin_functional/plugins/elasticsearch_client_plugin",
     "@kbn/elasticsearch-client-xpack-plugin": "link:x-pack/test/plugin_api_integration/plugins/elasticsearch_client",
     "@kbn/embeddable-enhanced-plugin": "link:x-pack/plugins/embeddable_enhanced",
@@ -897,6 +898,7 @@
     "jsonwebtoken": "^9.0.0",
     "jsts": "^1.6.2",
     "kea": "^2.4.2",
+    "langchain": "^0.0.132",
     "launchdarkly-js-client-sdk": "^2.22.1",
     "launchdarkly-node-server-sdk": "^6.4.2",
     "load-json-file": "^6.2.0",
@@ -1559,6 +1561,7 @@
     "val-loader": "^1.1.1",
     "vinyl-fs": "^4.0.0",
     "watchpack": "^1.6.0",
+    "web-streams-polyfill": "^3.2.1",
     "webpack": "^4.41.5",
     "webpack-bundle-analyzer": "^4.5.0",
     "webpack-cli": "^4.10.0",

packages/kbn-test/jest-preset.js

@@ -105,8 +105,10 @@ module.exports = {
   transformIgnorePatterns: [
     // ignore all node_modules except monaco-editor and react-monaco-editor which requires babel transforms to handle dynamic import()
     // since ESM modules are not natively supported in Jest yet (https://github.com/facebook/jest/issues/4842)
-    '[/\\\\]node_modules(?![\\/\\\\](byte-size|monaco-editor|monaco-yaml|vscode-languageserver-types|react-monaco-editor|d3-interpolate|d3-color))[/\\\\].+\\.js$',
+    '[/\\\\]node_modules(?![\\/\\\\](byte-size|monaco-editor|monaco-yaml|vscode-languageserver-types|react-monaco-editor|d3-interpolate|d3-color|langchain|langsmith))[/\\\\].+\\.js$',
     'packages/kbn-pm/dist/index.js',
+    '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))/dist/[/\\\\].+\\.js$',
+    '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))/dist/util/[/\\\\].+\\.js$',
   ],
 
   // An array of regexp pattern strings that are matched against all source file paths, matched files to include/exclude for code coverage

packages/kbn-test/jest_integration_node/jest-preset.js

@@ -19,6 +19,13 @@ module.exports = {
   testPathIgnorePatterns: preset.testPathIgnorePatterns.filter(
     (pattern) => !pattern.includes('integration_tests')
   ),
+  // An array of regexp pattern strings that are matched against, matched files will skip transformation:
+  transformIgnorePatterns: [
+    // since ESM modules are not natively supported in Jest yet (https://github.com/facebook/jest/issues/4842)
+    '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))[/\\\\].+\\.js$',
+    '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))/dist/[/\\\\].+\\.js$',
+    '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))/dist/util/[/\\\\].+\\.js$',
+  ],
   setupFilesAfterEnv: [
     '<rootDir>/packages/kbn-test/src/jest/setup/after_env.integration.js',
     '<rootDir>/packages/kbn-test/src/jest/setup/mocks.moment_timezone.js',

packages/kbn-test/src/jest/setup/setup_test.js

@@ -13,6 +13,7 @@
 
 import 'jest-styled-components';
 import '@testing-library/jest-dom';
+import 'web-streams-polyfill/es6'; // ReadableStream polyfill
 
 /**
  * Removed in Jest 27/jsdom, used in some transitive dependencies

tsconfig.base.json

@@ -674,6 +674,8 @@
       "@kbn/ecs-data-quality-dashboard-plugin/*": ["x-pack/plugins/ecs_data_quality_dashboard/*"],
       "@kbn/elastic-assistant": ["x-pack/packages/kbn-elastic-assistant"],
       "@kbn/elastic-assistant/*": ["x-pack/packages/kbn-elastic-assistant/*"],
+      "@kbn/elastic-assistant-plugin": ["x-pack/plugins/elastic_assistant"],
+      "@kbn/elastic-assistant-plugin/*": ["x-pack/plugins/elastic_assistant/*"],
       "@kbn/elasticsearch-client-plugin": ["test/plugin_functional/plugins/elasticsearch_client_plugin"],
       "@kbn/elasticsearch-client-plugin/*": ["test/plugin_functional/plugins/elasticsearch_client_plugin/*"],
       "@kbn/elasticsearch-client-xpack-plugin": ["x-pack/test/plugin_api_integration/plugins/elasticsearch_client"],
@@ -1645,4 +1647,5 @@
       "@kbn/ambient-storybook-types"
     ]
   }
-}
+}
+

x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx

@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { HttpSetup } from '@kbn/core-http-browser';
+import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/public/common';
+
+import { fetchConnectorExecuteAction, FetchConnectorExecuteAction } from './api';
+import type { Conversation, Message } from '../assistant_context/types';
+import { API_ERROR } from './translations';
+
+jest.mock('@kbn/core-http-browser');
+
+const mockHttp = {
+  fetch: jest.fn(),
+} as unknown as HttpSetup;
+
+const apiConfig: Conversation['apiConfig'] = {
+  connectorId: 'foo',
+  model: 'gpt-4',
+  provider: OpenAiProviderType.OpenAi,
+};
+
+const messages: Message[] = [
+  { content: 'This is a test', role: 'user', timestamp: new Date().toLocaleString() },
+];
+
+describe('fetchConnectorExecuteAction', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('calls the internal assistant API when assistantLangChain is true', async () => {
+    const testProps: FetchConnectorExecuteAction = {
+      assistantLangChain: true,
+      http: mockHttp,
+      messages,
+      apiConfig,
+    };
+
+    await fetchConnectorExecuteAction(testProps);
+
+    expect(mockHttp.fetch).toHaveBeenCalledWith(
+      '/internal/elastic_assistant/actions/connector/foo/_execute',
+      {
+        body: '{"params":{"subActionParams":{"body":"{\\"model\\":\\"gpt-4\\",\\"messages\\":[{\\"role\\":\\"user\\",\\"content\\":\\"This is a test\\"}],\\"n\\":1,\\"stop\\":null,\\"temperature\\":0.2}"},"subAction":"test"}}',
+        headers: { 'Content-Type': 'application/json' },
+        method: 'POST',
+        signal: undefined,
+      }
+    );
+  });
+
+  it('calls the actions connector api when assistantLangChain is false', async () => {
+    const testProps: FetchConnectorExecuteAction = {
+      assistantLangChain: false,
+      http: mockHttp,
+      messages,
+      apiConfig,
+    };
+
+    await fetchConnectorExecuteAction(testProps);
+
+    expect(mockHttp.fetch).toHaveBeenCalledWith('/api/actions/connector/foo/_execute', {
+      body: '{"params":{"subActionParams":{"body":"{\\"model\\":\\"gpt-4\\",\\"messages\\":[{\\"role\\":\\"user\\",\\"content\\":\\"This is a test\\"}],\\"n\\":1,\\"stop\\":null,\\"temperature\\":0.2}"},"subAction":"test"}}',
+      headers: { 'Content-Type': 'application/json' },
+      method: 'POST',
+      signal: undefined,
+    });
+  });
+
+  it('returns API_ERROR when the response status is not ok', async () => {
+    (mockHttp.fetch as jest.Mock).mockResolvedValue({ status: 'error' });
+
+    const testProps: FetchConnectorExecuteAction = {
+      assistantLangChain: false,
+      http: mockHttp,
+      messages,
+      apiConfig,
+    };
+
+    const result = await fetchConnectorExecuteAction(testProps);
+
+    expect(result).toBe(API_ERROR);
+  });
+
+  it('returns API_ERROR when there are no choices', async () => {
+    (mockHttp.fetch as jest.Mock).mockResolvedValue({ status: 'ok', data: {} });
+    const testProps: FetchConnectorExecuteAction = {
+      assistantLangChain: false,
+      http: mockHttp,
+      messages,
+      apiConfig,
+    };
+
+    const result = await fetchConnectorExecuteAction(testProps);
+
+    expect(result).toBe(API_ERROR);
+  });
+
+  it('return the trimmed first `choices` `message` `content` when the API call is successful', async () => {
+    (mockHttp.fetch as jest.Mock).mockResolvedValue({
+      status: 'ok',
+      data: {
+        choices: [
+          {
+            message: {
+              content: '   Test response    ', // leading and trailing whitespace
+            },
+          },
+        ],
+      },
+    });
+
+    const testProps: FetchConnectorExecuteAction = {
+      assistantLangChain: false,
+      http: mockHttp,
+      messages,
+      apiConfig,
+    };
+
+    const result = await fetchConnectorExecuteAction(testProps);
+
+    expect(result).toBe('Test response');
+  });
+});

x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx

@@ -14,13 +14,15 @@ import { API_ERROR } from './translations';
 import { MODEL_GPT_3_5_TURBO } from '../connectorland/models/model_selector/model_selector';
 
 export interface FetchConnectorExecuteAction {
+  assistantLangChain: boolean;
   apiConfig: Conversation['apiConfig'];
   http: HttpSetup;
   messages: Message[];
   signal?: AbortSignal | undefined;
 }
 
 export const fetchConnectorExecuteAction = async ({
+  assistantLangChain,
   http,
   messages,
   apiConfig,
@@ -54,19 +56,20 @@ export const fetchConnectorExecuteAction = async ({
   };
 
   try {
+    const path = assistantLangChain
+      ? `/internal/elastic_assistant/actions/connector/${apiConfig?.connectorId}/_execute`
+      : `/api/actions/connector/${apiConfig?.connectorId}/_execute`;
+
     // TODO: Find return type for this API
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const response = await http.fetch<any>(
-      `/api/actions/connector/${apiConfig?.connectorId}/_execute`,
-      {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify(requestBody),
-        signal,
-      }
-    );
+    const response = await http.fetch<any>(path, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(requestBody),
+      signal,
+    });
 
     const data = response.data;
     if (response.status !== 'ok') {