[Metrics][Discover] Fix fields dropdown by casting dimension fields to string (#242432)

Closes #239952

### Summary
This PR resolves the data inconsistency between metric charts and the
"Filter by value" dropdown, where values visible in charts were missing
from the filter list.

#### Problem

The root cause was that the same dimension field (e.g.,
`attributes.cpu`) could have different data types across different
indices:
- `long` in one index
- `keyword` in another index

Elasticsearch aggregations and ES|QL queries fail when trying to process
fields with inconsistent types, resulting in incomplete dimension values
in the dropdown.

#### Solution

**1. Updated `get_dimentions.ts`:**
- Migrated from Elasticsearch `_search` API with `terms` aggregations to
ES|QL
- Added explicit type casting to string using `::STRING` syntax
- Increased dimension value limit from 20 to 1000
- Simplified response mapping logic

**2. Updated `create_esql_query.ts`:**
- Added type casting logic for dimensions in `WHERE` clauses
- Applied the same `needsStringCasting()` check used in `CONCAT`
operations
- Ensures consistent string type handling for null checks

BEFORE
<img width="2304" height="560" alt="image"
src="https://github.com/user-attachments/assets/e74c73af-838e-4282-b6f7-9778f03e562d"
/>

AFTER
<img width="1043" height="292" alt="Screenshot 2025-11-10 at 14 37 35"
src="https://github.com/user-attachments/assets/ade48ce5-1290-4fc7-afb4-57adcf3ebaf3"
/>

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Carlos Crespo <carloshenrique.leonelcrespo@elastic.co>

Author: 3 people

src/platform/packages/shared/kbn-traced-es-client/src/create_traced_es_client.ts

@@ -176,30 +176,25 @@ export function createTracedEsClient({
           .query(
             { ...parameters },
             {
-              querystring: {
-                drop_null_columns: true,
-              },
+              querystring: { drop_null_columns: true },
               ...requestOpts,
             }
           )
           .then((response) => {
-            const esqlResponse = response as unknown as UnparsedEsqlResponseOf<EsqlOutput>;
-
             const transform = options?.transform ?? 'none';
 
             if (transform === 'none') {
-              return esqlResponse;
+              return response;
             }
 
-            const parsedResponse = { hits: esqlResultToPlainObjects(esqlResponse) };
+            const rawEsqlResponse = response.body as unknown as UnparsedEsqlResponseOf<EsqlOutput>;
+            const hits = esqlResultToPlainObjects(rawEsqlResponse);
 
             if (transform === 'plain') {
-              return parsedResponse;
+              return { ...response, body: { hits } };
             }
 
-            return {
-              hits: parsedResponse.hits.map((hit) => unflattenObject(hit)),
-            };
+            return { ...response, body: { hits: hits.map((hit) => unflattenObject(hit)) } };
           }) as Promise<{ body: InferEsqlResponseOf<EsqlOutput, EsqlOptions> }>;
       });
     },

src/platform/packages/shared/kbn-unified-metrics-grid/src/common/utils/esql/create_esql_query.test.ts

@@ -91,7 +91,7 @@ TS metrics-*
     expect(query).toBe(
       `
 TS metrics-*
-  | WHERE \`host.ip\` IS NOT NULL AND \`host.name\` IS NOT NULL
+  | WHERE \`host.ip\`::STRING IS NOT NULL AND \`host.name\` IS NOT NULL
   | STATS AVG(cpu.usage) BY BUCKET(@timestamp, 100, ?_tstart, ?_tend), \`host.ip\`, \`host.name\`
   | EVAL ${DIMENSIONS_COLUMN} = CONCAT(\`host.ip\`::STRING, " › ", \`host.name\`)
   | DROP \`host.ip\`, \`host.name\`
@@ -107,7 +107,7 @@ TS metrics-*
     expect(query).toBe(
       `
 TS metrics-*
-  | WHERE \`cpu.cores\` IS NOT NULL AND \`host.name\` IS NOT NULL
+  | WHERE \`cpu.cores\`::STRING IS NOT NULL AND \`host.name\` IS NOT NULL
   | STATS AVG(cpu.usage) BY BUCKET(@timestamp, 100, ?_tstart, ?_tend), \`cpu.cores\`, \`host.name\`
   | EVAL ${DIMENSIONS_COLUMN} = CONCAT(\`cpu.cores\`::STRING, " › ", \`host.name\`)
   | DROP \`cpu.cores\`, \`host.name\`
@@ -123,7 +123,7 @@ TS metrics-*
     expect(query).toBe(
       `
 TS metrics-*
-  | WHERE \`host.ip\` IS NOT NULL AND \`host.name\` IS NOT NULL AND \`cpu.cores\` IS NOT NULL
+  | WHERE \`host.ip\`::STRING IS NOT NULL AND \`host.name\` IS NOT NULL AND \`cpu.cores\`::STRING IS NOT NULL
   | STATS AVG(cpu.usage) BY BUCKET(@timestamp, 100, ?_tstart, ?_tend), \`host.ip\`, \`host.name\`, \`cpu.cores\`
   | EVAL ${DIMENSIONS_COLUMN} = CONCAT(\`host.ip\`::STRING, " › ", \`host.name\`, " › ", \`cpu.cores\`::STRING)
   | DROP \`host.ip\`, \`host.name\`, \`cpu.cores\`
@@ -271,7 +271,7 @@ TS metrics-*
       expect(query).toBe(
         `
 TS metrics-*
-  | WHERE \`host-ip\` IS NOT NULL AND \`service-name\` IS NOT NULL
+  | WHERE \`host-ip\`::STRING IS NOT NULL AND \`service-name\` IS NOT NULL
   | STATS AVG(cpu.usage) BY BUCKET(@timestamp, 100, ?_tstart, ?_tend), \`host-ip\`, \`service-name\`
   | EVAL ${DIMENSIONS_COLUMN} = CONCAT(\`host-ip\`::STRING, " › ", \`service-name\`)
   | DROP \`host-ip\`, \`service-name\`

src/platform/packages/shared/kbn-unified-metrics-grid/src/common/utils/esql/create_esql_query.ts

@@ -53,6 +53,8 @@ export function createESQLQuery({ metric, dimensions = [], filters }: CreateESQL
 
   const whereConditions: QueryOperator[] = [];
   const valuesByField = new Map<string, Set<string>>();
+  const dimensionTypeMap = new Map(metricDimensions?.map((dim) => [dim.name, dim.type]));
+
   if (filters && filters.length) {
     for (const filter of filters) {
       const currentValues = valuesByField.get(filter.field);
@@ -64,23 +66,31 @@ export function createESQLQuery({ metric, dimensions = [], filters }: CreateESQL
     }
 
     valuesByField.forEach((value, key) => {
+      const dimType = dimensionTypeMap.get(key);
+      const escapedKey = sanitazeESQLInput(key);
+      const castedKey =
+        dimType && needsStringCasting(dimType) ? `${escapedKey}::STRING` : escapedKey;
+
       whereConditions.push(
-        where(
-          `${sanitazeESQLInput(key)} IN (${new Array(value.size).fill('?').join(', ')})`,
-          Array.from(value)
-        )
+        where(`${castedKey} IN (${new Array(value.size).fill('?').join(', ')})`, Array.from(value))
       );
     });
   }
 
-  const dimensionTypeMap = new Map(metricDimensions?.map((dim) => [dim.name, dim.type]));
-
   const unfilteredDimensions = (dimensions ?? []).filter((dim) => !valuesByField.has(dim));
   const queryPipeline = source.pipe(
     ...whereConditions,
     unfilteredDimensions.length > 0
       ? where(
-          unfilteredDimensions.map((dim) => `${sanitazeESQLInput(dim)} IS NOT NULL`).join(' AND ')
+          unfilteredDimensions
+            .map((dim) => {
+              const dimType = dimensionTypeMap.get(dim);
+              const escapedDim = sanitazeESQLInput(dim);
+              const castedDim =
+                dimType && needsStringCasting(dimType) ? `${escapedDim}::STRING` : escapedDim;
+              return `${castedDim} IS NOT NULL`;
+            })
+            .join(' AND ')
         )
       : (query) => query,
     stats(

…rver/routes/dimensions/get_dimentions.ts …rver/routes/dimensions/get_dimensions.tssrc/platform/plugins/shared/metrics_experience/server/routes/dimensions/get_dimentions.ts renamed to src/platform/plugins/shared/metrics_experience/server/routes/dimensions/get_dimensions.ts src/platform/plugins/shared/metrics_experience/server/routes/dimensions/get_dimentions.ts renamed to src/platform/plugins/shared/metrics_experience/server/routes/dimensions/get_dimensions.ts

@@ -10,7 +10,7 @@
 import type { Logger } from '@kbn/core/server';
 import { dateRangeQuery } from '@kbn/es-query';
 import type { TracedElasticsearchClient } from '@kbn/traced-es-client';
-import type { estypes } from '@elastic/elasticsearch';
+import { from as fromCommand, evaluate, where, stats, sort, limit } from '@kbn/esql-composer';
 
 interface CreateDimensionsParams {
   esClient: TracedElasticsearchClient;
@@ -33,43 +33,38 @@ export const getDimensions = async ({
     return [];
   }
 
+  const source = fromCommand(indices);
+  const query = source
+    .pipe(
+      evaluate('??dim = ??dim::string', { dim: dimensions[0] }),
+      where('??dim IS NOT NULL', { dim: dimensions[0] }),
+      stats('BY ??dim', {
+        dim: dimensions[0],
+      }),
+      sort('??dim', { dim: dimensions[0] }),
+      limit(20)
+    )
+    .toString();
+
   try {
-    const response = await esClient.search('get_dimensions', {
-      index: indices.join(','),
-      track_total_hits: false,
-      size: 0,
-      query: {
-        bool: {
-          filter: [...dateRangeQuery(from, to)],
+    const response = await esClient.esql(
+      'get_dimensions',
+      {
+        query,
+        filter: {
+          bool: {
+            filter: [...dateRangeQuery(from, to)],
+          },
         },
       },
-      // Create aggregations for each dimension
-      aggs: dimensions.reduce((acc, currDimension) => {
-        acc[currDimension] = {
-          terms: {
-            field: currDimension,
-            size: 20,
-            order: { _key: 'asc' },
-          },
-        };
-
-        return acc;
-      }, {} as Record<string, Pick<estypes.AggregationsAggregationContainer, 'terms'>>),
-    });
-
-    const aggregations = response.aggregations;
-
-    const values = dimensions.flatMap((dimension) => {
-      const agg = aggregations?.[dimension];
-      return (
-        agg?.buckets?.map((bucket) => ({
-          value: String(bucket.key ?? ''),
-          field: dimension,
-        })) ?? []
-      );
-    });
-
-    return values;
+      {
+        transform: 'plain',
+      }
+    );
+    return response.hits.map((hit) => ({
+      value: String(hit[dimensions[0]]),
+      field: dimensions[0],
+    }));
   } catch (error) {
     logger.error('Error fetching dimension values:', error);
     return [];

src/platform/plugins/shared/metrics_experience/server/routes/dimensions/route.ts

@@ -13,7 +13,7 @@ import { isoToEpoch } from '@kbn/zod-helpers';
 import { parse as dateMathParse } from '@kbn/datemath';
 import { getRequestAbortedSignal } from '@kbn/data-plugin/server';
 import { createRoute } from '../create_route';
-import { getDimensions } from './get_dimentions';
+import { getDimensions } from './get_dimensions';
 import { throwNotFoundIfMetricsExperienceDisabled } from '../../lib/utils';
 
 export const getDimensionsRoute = createRoute({

src/platform/plugins/shared/metrics_experience/tsconfig.json

@@ -20,6 +20,7 @@
     "@kbn/otel-semantic-conventions",
     "@kbn/i18n",
     "@kbn/core-chrome-browser",
-    "@kbn/data-plugin"
+    "@kbn/data-plugin",
+    "@kbn/esql-composer"
   ]
 }