Annotations: Change indices and rewrites annotation find query to improve database query performance (#21915)

Drop indices and create new ones and rewrites annotation find query 
to address performance issues when querying annotation table and 
there is a large amount of rows.

Fixes #21902

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Co-authored-by: Kyle Brandt <kyle@kbrandt.com>

devenv/docker/loadtest/annotations_by_tag_test.js

@@ -0,0 +1,71 @@
+import { sleep, check, group } from 'k6';
+import { createClient, createBasicAuthClient } from './modules/client.js';
+import { createTestOrgIfNotExists, createTestdataDatasourceIfNotExists } from './modules/util.js';
+
+export let options = {
+  noCookiesReset: true
+};
+
+let endpoint = __ENV.URL || 'http://localhost:3000';
+const client = createClient(endpoint);
+
+export const setup = () => {
+  const basicAuthClient = createBasicAuthClient(endpoint, 'admin', 'admin');
+  const orgId = createTestOrgIfNotExists(basicAuthClient);
+  const datasourceId = createTestdataDatasourceIfNotExists(basicAuthClient);
+  client.withOrgId(orgId);
+  return {
+    orgId: orgId,
+    datasourceId: datasourceId,
+  };
+}
+
+export default (data) => {
+  group("annotation by tag test", () => {
+    if (__ITER === 0) {
+      group("user authenticates thru ui with username and password", () => {
+        let res = client.ui.login('admin', 'admin');
+
+        check(res, {
+          'response status is 200': (r) => r.status === 200,
+          'response has cookie \'grafana_session\' with 32 characters': (r) => r.cookies.grafana_session[0].value.length === 32,
+        });
+      });
+    }
+
+    if (__ITER !== 0) {
+      group("batch tsdb requests with annotations by tag", () => {
+        const batchCount = 20;
+        const requests = [];
+        const payload = {
+          from: '1547765247624',
+          to: '1547768847624',
+          queries: [{
+            refId: 'A',
+            scenarioId: 'random_walk',
+            intervalMs: 10000,
+            maxDataPoints: 433,
+            datasourceId: data.datasourceId,
+          }]
+        };
+
+        requests.push({ method: 'GET', url: '/api/annotations?from=1580825186534&to=1580846786535' });
+
+        for (let n = 0; n < batchCount; n++) {
+          requests.push({ method: 'POST', url: '/api/tsdb/query', body: payload });
+        }
+
+        let responses = client.batch(requests);
+        for (let n = 0; n < batchCount; n++) {
+          check(responses[n], {
+            'response status is 200': (r) => r.status === 200,
+          });
+        }
+      });
+    }
+  });
+
+  sleep(5)
+}
+
+export const teardown = (data) => {}

pkg/services/sqlstore/annotation.go

@@ -144,46 +144,48 @@ func (r *SqlAnnotationRepo) Find(query *annotations.ItemQuery) ([]*annotations.I
 		FROM annotation
 		LEFT OUTER JOIN ` + dialect.Quote("user") + ` as usr on usr.id = annotation.user_id
 		LEFT OUTER JOIN alert on alert.id = annotation.alert_id
+		INNER JOIN (
+			SELECT a.id from annotation a
 		`)
 
-	sql.WriteString(`WHERE annotation.org_id = ?`)
+	sql.WriteString(`WHERE a.org_id = ?`)
 	params = append(params, query.OrgId)
 
 	if query.AnnotationId != 0 {
 		// fmt.Print("annotation query")
-		sql.WriteString(` AND annotation.id = ?`)
+		sql.WriteString(` AND a.id = ?`)
 		params = append(params, query.AnnotationId)
 	}
 
 	if query.AlertId != 0 {
-		sql.WriteString(` AND annotation.alert_id = ?`)
+		sql.WriteString(` AND a.alert_id = ?`)
 		params = append(params, query.AlertId)
 	}
 
 	if query.DashboardId != 0 {
-		sql.WriteString(` AND annotation.dashboard_id = ?`)
+		sql.WriteString(` AND a.dashboard_id = ?`)
 		params = append(params, query.DashboardId)
 	}
 
 	if query.PanelId != 0 {
-		sql.WriteString(` AND annotation.panel_id = ?`)
+		sql.WriteString(` AND a.panel_id = ?`)
 		params = append(params, query.PanelId)
 	}
 
 	if query.UserId != 0 {
-		sql.WriteString(` AND annotation.user_id = ?`)
+		sql.WriteString(` AND a.user_id = ?`)
 		params = append(params, query.UserId)
 	}
 
 	if query.From > 0 && query.To > 0 {
-		sql.WriteString(` AND annotation.epoch <= ? AND annotation.epoch_end >= ?`)
+		sql.WriteString(` AND a.epoch <= ? AND a.epoch_end >= ?`)
 		params = append(params, query.To, query.From)
 	}
 
 	if query.Type == "alert" {
-		sql.WriteString(` AND annotation.alert_id > 0`)
+		sql.WriteString(` AND a.alert_id > 0`)
 	} else if query.Type == "annotation" {
-		sql.WriteString(` AND annotation.alert_id = 0`)
+		sql.WriteString(` AND a.alert_id = 0`)
 	}
 
 	if len(query.Tags) > 0 {
@@ -204,7 +206,7 @@ func (r *SqlAnnotationRepo) Find(query *annotations.ItemQuery) ([]*annotations.I
 			tagsSubQuery := fmt.Sprintf(`
         SELECT SUM(1) FROM annotation_tag at
           INNER JOIN tag on tag.id = at.tag_id
-          WHERE at.annotation_id = annotation.id
+          WHERE at.annotation_id = a.id
             AND (
               %s
             )
@@ -223,7 +225,8 @@ func (r *SqlAnnotationRepo) Find(query *annotations.ItemQuery) ([]*annotations.I
 		query.Limit = 100
 	}
 
-	sql.WriteString(" ORDER BY epoch DESC" + dialect.Limit(query.Limit))
+	// order of ORDER BY arguments match the order of a sql index for performance
+	sql.WriteString(" ORDER BY a.org_id, a.epoch_end DESC, a.epoch DESC" + dialect.Limit(query.Limit) + " ) dt on dt.id = annotation.id")
 
 	items := make([]*annotations.ItemDTO, 0)
 

pkg/services/sqlstore/migrations/annotation_mig.go

@@ -123,7 +123,28 @@ func addAnnotationMig(mg *Migrator) {
 	mg.AddMigration("Make epoch_end the same as epoch", NewRawSqlMigration("UPDATE annotation SET epoch_end = epoch"))
 	mg.AddMigration("Move region to single row", &AddMakeRegionSingleRowMigration{})
 
-	// TODO! drop region_id column?
+	//
+	// 6.6.1: Optimize annotation queries
+	//
+	mg.AddMigration("Remove index org_id_epoch from annotation table", NewDropIndexMigration(table, &Index{
+		Cols: []string{"org_id", "epoch"}, Type: IndexType,
+	}))
+
+	mg.AddMigration("Remove index org_id_dashboard_id_panel_id_epoch from annotation table", NewDropIndexMigration(table, &Index{
+		Cols: []string{"org_id", "dashboard_id", "panel_id", "epoch"}, Type: IndexType,
+	}))
+
+	mg.AddMigration("Add index for org_id_dashboard_id_epoch_end_epoch on annotation table", NewAddIndexMigration(table, &Index{
+		Cols: []string{"org_id", "dashboard_id", "epoch_end", "epoch"}, Type: IndexType,
+	}))
+
+	mg.AddMigration("Add index for org_id_epoch_end_epoch on annotation table", NewAddIndexMigration(table, &Index{
+		Cols: []string{"org_id", "epoch_end", "epoch"}, Type: IndexType,
+	}))
+
+	mg.AddMigration("Remove index org_id_epoch_epoch_end from annotation table", NewDropIndexMigration(table, &Index{
+		Cols: []string{"org_id", "epoch", "epoch_end"}, Type: IndexType,
+	}))
 }
 
 type AddMakeRegionSingleRowMigration struct {