Elasticsearch: Guess field type from first non-empty value (#32290)

public/app/plugins/datasource/elasticsearch/elastic_response.ts

@@ -451,7 +451,7 @@ export class ElasticResponse {
         const { propNames, docs } = flattenHits(response.hits.hits);
         if (docs.length > 0) {
           let series = createEmptyDataFrame(
-            propNames,
+            propNames.map(toNameTypePair(docs)),
             this.targets[0].timeField!,
             isLogsRequest,
             logMessageField,
@@ -635,7 +635,7 @@ const flattenHits = (hits: Doc[]): { docs: Array<Record<string, any>>; propNames
  * @param logLevelField
  */
 const createEmptyDataFrame = (
-  propNames: string[],
+  props: Array<[string, FieldType]>,
   timeField: string,
   isLogsRequest: boolean,
   logMessageField?: string,
@@ -671,13 +671,13 @@ const createEmptyDataFrame = (
 
   const fieldNames = series.fields.map((field) => field.name);
 
-  for (const propName of propNames) {
+  for (const [name, type] of props) {
     // Do not duplicate fields. This can mean that we will shadow some fields.
-    if (fieldNames.includes(propName)) {
+    if (fieldNames.includes(name)) {
       continue;
     }
     // Do not add _source field (besides logs) as we are showing each _source field in table instead.
-    if (!isLogsRequest && propName === '_source') {
+    if (!isLogsRequest && name === '_source') {
       continue;
     }
 
@@ -685,8 +685,8 @@ const createEmptyDataFrame = (
       config: {
         filterable: true,
       },
-      name: propName,
-      type: FieldType.string,
+      name,
+      type,
     }).parse = (v: any) => {
       return v || '';
     };
@@ -705,3 +705,23 @@ const addPreferredVisualisationType = (series: any, type: PreferredVisualisation
 
   return s;
 };
+
+const toNameTypePair = (docs: Array<Record<string, any>>) => (propName: string): [string, FieldType] => [
+  propName,
+  guessType(docs.find((doc) => doc[propName] !== undefined)?.[propName]),
+];
+
+/**
+ * Trying to guess data type from its value. This is far from perfect, as in order to have accurate guess
+ * we should have access to the elasticsearch mapping, but it covers the most common use cases for numbers, strings & arrays.
+ */
+const guessType = (value: unknown): FieldType => {
+  switch (typeof value) {
+    case 'number':
+      return FieldType.number;
+    case 'string':
+      return FieldType.string;
+    default:
+      return FieldType.other;
+  }
+};

public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts

@@ -1244,6 +1244,7 @@ describe('ElasticResponse', () => {
                 _source: {
                   '@timestamp': '2019-06-24T09:51:19.765Z',
                   host: 'djisaodjsoad',
+                  number: 1,
                   message: 'hello, i am a message',
                   level: 'debug',
                   fields: {
@@ -1263,6 +1264,7 @@ describe('ElasticResponse', () => {
                 _source: {
                   '@timestamp': '2019-06-24T09:52:19.765Z',
                   host: 'dsalkdakdop',
+                  number: 2,
                   message: 'hello, i am also message',
                   level: 'error',
                   fields: {
@@ -1344,5 +1346,21 @@ describe('ElasticResponse', () => {
       const field = fieldCache.getFieldByName('level');
       expect(field?.values.toArray()).toEqual(['debug', 'info']);
     });
+
+    it('should correctly guess field types', () => {
+      const result = new ElasticResponse(targets, response).getLogs();
+      const logResults = result.data[0] as MutableDataFrame;
+
+      const fields = logResults.fields.map((f) => {
+        return {
+          name: f.name,
+          type: f.type,
+        };
+      });
+
+      expect(fields).toContainEqual({ name: '@timestamp', type: 'time' });
+      expect(fields).toContainEqual({ name: 'number', type: 'number' });
+      expect(fields).toContainEqual({ name: 'message', type: 'string' });
+    });
   });
 });