Elastic: To get fields, start with today's index and go backwards (#22318)

* Elastic: To get fields, start with today's index and go backwards

* Elastic: distinguish non-existing indices from other issues; change index traversal from recursive to iterative; go through a max of 7 days

* Elastic: fix the comments

Co-authored-by: Andrej Ocenas <mr.ocenas@gmail.com>

public/app/plugins/datasource/elasticsearch/datasource.test.ts

@@ -37,26 +37,41 @@ describe('ElasticDatasource', function(this: any) {
     getAdhocFilters: jest.fn(() => []),
   };
 
-  const timeSrv: any = {
-    time: { from: 'now-1h', to: 'now' },
-    timeRange: jest.fn(() => {
-      return {
-        from: dateMath.parse(timeSrv.time.from, false),
-        to: dateMath.parse(timeSrv.time.to, true),
-      };
-    }),
-    setTime: jest.fn(time => {
-      this.time = time;
-    }),
-  };
+  const timeSrv: any = createTimeSrv('now-1h');
 
   const ctx = {
     $rootScope,
   } as any;
 
+  function createTimeSrv(from: string) {
+    const srv: any = {
+      time: { from: from, to: 'now' },
+    };
+
+    srv.timeRange = jest.fn(() => {
+      return {
+        from: dateMath.parse(srv.time.from, false),
+        to: dateMath.parse(srv.time.to, true),
+      };
+    });
+
+    srv.setTime = jest.fn(time => {
+      srv.time = time;
+    });
+
+    return srv;
+  }
+
   function createDatasource(instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>) {
+    createDatasourceWithTime(instanceSettings, timeSrv as TimeSrv);
+  }
+
+  function createDatasourceWithTime(
+    instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>,
+    timeSrv: TimeSrv
+  ) {
     instanceSettings.jsonData = instanceSettings.jsonData || ({} as ElasticsearchOptions);
-    ctx.ds = new ElasticDatasource(instanceSettings, templateSrv as TemplateSrv, timeSrv as TimeSrv);
+    ctx.ds = new ElasticDatasource(instanceSettings, templateSrv as TemplateSrv, timeSrv);
   }
 
   describe('When testing datasource with index pattern', () => {
@@ -355,6 +370,123 @@ describe('ElasticDatasource', function(this: any) {
     });
   });
 
+  describe('When getting field mappings on indices with gaps', () => {
+    const twoWeekTimeSrv: any = createTimeSrv('now-2w');
+
+    const basicResponse = {
+      data: {
+        metricbeat: {
+          mappings: {
+            metricsets: {
+              _all: {},
+              properties: {
+                '@timestamp': { type: 'date' },
+                beat: {
+                  properties: {
+                    hostname: { type: 'string' },
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+    };
+
+    const alternateResponse = {
+      data: {
+        metricbeat: {
+          mappings: {
+            metricsets: {
+              _all: {},
+              properties: {
+                '@timestamp': { type: 'date' },
+              },
+            },
+          },
+        },
+      },
+    };
+
+    beforeEach(() => {
+      createDatasourceWithTime(
+        {
+          url: 'http://es.com',
+          database: '[asd-]YYYY.MM.DD',
+          jsonData: { interval: 'Daily', esVersion: 50 } as ElasticsearchOptions,
+        } as DataSourceInstanceSettings<ElasticsearchOptions>,
+        twoWeekTimeSrv
+      );
+    });
+
+    it('should return fields of the newest available index', async () => {
+      const twoDaysBefore = toUtc()
+        .subtract(2, 'day')
+        .format('YYYY.MM.DD');
+
+      const threeDaysBefore = toUtc()
+        .subtract(3, 'day')
+        .format('YYYY.MM.DD');
+
+      datasourceRequestMock.mockImplementation(options => {
+        if (options.url === `http://es.com/asd-${twoDaysBefore}/_mapping`) {
+          return Promise.resolve(basicResponse);
+        } else if (options.url === `http://es.com/asd-${threeDaysBefore}/_mapping`) {
+          return Promise.resolve(alternateResponse);
+        }
+        return Promise.reject({ status: 404 });
+      });
+
+      const fieldObjects = await ctx.ds.getFields({
+        find: 'fields',
+        query: '*',
+      });
+      const fields = _.map(fieldObjects, 'text');
+      expect(fields).toEqual(['@timestamp', 'beat.hostname']);
+    });
+
+    it('should not retry when ES is down', async () => {
+      const twoDaysBefore = toUtc()
+        .subtract(2, 'day')
+        .format('YYYY.MM.DD');
+
+      datasourceRequestMock.mockImplementation(options => {
+        if (options.url === `http://es.com/asd-${twoDaysBefore}/_mapping`) {
+          return Promise.resolve(basicResponse);
+        }
+        return Promise.reject({ status: 500 });
+      });
+
+      expect.assertions(2);
+      try {
+        await ctx.ds.getFields({
+          find: 'fields',
+          query: '*',
+        });
+      } catch (e) {
+        expect(e).toStrictEqual({ status: 500 });
+        expect(datasourceRequestMock).toBeCalledTimes(1);
+      }
+    });
+
+    it('should not retry more than 7 indices', async () => {
+      datasourceRequestMock.mockImplementation(() => {
+        return Promise.reject({ status: 404 });
+      });
+
+      expect.assertions(2);
+      try {
+        await ctx.ds.getFields({
+          find: 'fields',
+          query: '*',
+        });
+      } catch (e) {
+        expect(e).toStrictEqual({ status: 404 });
+        expect(datasourceRequestMock).toBeCalledTimes(7);
+      }
+    });
+  });
+
   describe('When getting fields from ES 7.0', () => {
     beforeEach(() => {
       createDatasource({

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -89,11 +89,19 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
     return getBackendSrv().datasourceRequest(options);
   }
 
+  /**
+   * Sends a GET request to the specified url on the newest matching and available index.
+   *
+   * When multiple indices span the provided time range, the request is sent starting from the newest index,
+   * and then going backwards until an index is found.
+   *
+   * @param url the url to query the index on, for example `/_mapping`.
+   */
   private get(url: string) {
     const range = this.timeSrv.timeRange();
     const indexList = this.indexPattern.getIndexList(range.from.valueOf(), range.to.valueOf());
     if (_.isArray(indexList) && indexList.length) {
-      return this.request('GET', indexList[0] + url).then((results: any) => {
+      return this.requestAllIndices(indexList, url).then((results: any) => {
         results.data.$$config = results.config;
         return results.data;
       });
@@ -105,6 +113,20 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
     }
   }
 
+  private async requestAllIndices(indexList: string[], url: string): Promise<any> {
+    const maxTraversals = 7; // do not go beyond one week (for a daily pattern)
+    const listLen = indexList.length;
+    for (let i = 0; i < Math.min(listLen, maxTraversals); i++) {
+      try {
+        return await this.request('GET', indexList[listLen - i - 1] + url);
+      } catch (err) {
+        if (err.status !== 404 || i === maxTraversals - 1) {
+          throw err;
+        }
+      }
+    }
+  }
+
   private post(url: string, data: any) {
     return this.request('POST', url, data)
       .then((results: any) => {