search: fix search to limit dashboards better

Change the query to first select the dashboards, apply the limit then
join with tags. Means the limit will apply to the number of dashboards
returned in the search. The disadvantage is that the query will return
more rows than the limit, no. of dashboards x no. of tags. So hard limit
set to 5000 for all rows.

pkg/services/sqlstore/dashboard.go

@@ -189,14 +189,13 @@ type DashboardSearchProjection struct {
 }
 
 func findDashboards(query *search.FindPersistedDashboardsQuery) ([]DashboardSearchProjection, error) {
+	var sql bytes.Buffer
+	params := make([]interface{}, 0)
 	limit := query.Limit
 	if limit == 0 {
 		limit = 1000
 	}
 
-	var sql bytes.Buffer
-	params := make([]interface{}, 0)
-
 	sql.WriteString(`
 	SELECT
 		dashboard.id,
@@ -207,36 +206,69 @@ func findDashboards(query *search.FindPersistedDashboardsQuery) ([]DashboardSear
 		dashboard.folder_id,
 		folder.slug as folder_slug,
 		folder.title as folder_title
-	FROM (
+	FROM `)
+
+	// add tags filter
+	if len(query.Tags) > 0 {
+		sql.WriteString(
+			`(
 		SELECT
 			dashboard.id FROM dashboard
 			LEFT OUTER JOIN dashboard_tag ON dashboard_tag.dashboard_id = dashboard.id
 		`)
-
-	// add tags filter
-	if len(query.Tags) > 0 {
-		sql.WriteString(` WHERE dashboard_tag.term IN (?` + strings.Repeat(",?", len(query.Tags)-1) + `)`)
-		for _, tag := range query.Tags {
-			params = append(params, tag)
-		}
-	}
-
-	// this ends the inner select (tag filtered part)
-	sql.WriteString(`
-		  GROUP BY dashboard.id HAVING COUNT(dashboard.id) >= ?
-		  ORDER BY dashboard.title ASC LIMIT ?) as ids`)
-	params = append(params, len(query.Tags))
-	params = append(params, limit)
-
-	sql.WriteString(`
-		INNER JOIN dashboard on ids.id = dashboard.id
-		LEFT OUTER JOIN dashboard folder on folder.id = dashboard.folder_id
-		LEFT OUTER JOIN dashboard_tag on dashboard.id = dashboard_tag.dashboard_id`)
 		if query.IsStarred {
 			sql.WriteString(" INNER JOIN star on star.dashboard_id = dashboard.id")
 		}
 
-	sql.WriteString(` WHERE dashboard.org_id=?`)
+		sql.WriteString(` WHERE dashboard_tag.term IN (?` + strings.Repeat(",?", len(query.Tags)-1) + `) AND `)
+		for _, tag := range query.Tags {
+			params = append(params, tag)
+		}
+		params = createSearchWhereClause(query, &sql, params)
+		fmt.Printf("params2 %v", params)
+
+		// this ends the inner select (tag filtered part)
+		sql.WriteString(`
+			GROUP BY dashboard.id HAVING COUNT(dashboard.id) >= ?
+			LIMIT ?) as ids
+			INNER JOIN dashboard on ids.id = dashboard.id
+		`)
+
+		params = append(params, len(query.Tags))
+		params = append(params, limit)
+	} else {
+		sql.WriteString(`( SELECT dashboard.id FROM dashboard `)
+		if query.IsStarred {
+			sql.WriteString(" INNER JOIN star on star.dashboard_id = dashboard.id")
+		}
+		sql.WriteString(` WHERE `)
+		params = createSearchWhereClause(query, &sql, params)
+
+		sql.WriteString(`
+			LIMIT ?) as ids
+		INNER JOIN dashboard on ids.id = dashboard.id
+		`)
+		params = append(params, limit)
+	}
+
+	sql.WriteString(`
+		LEFT OUTER JOIN dashboard folder on folder.id = dashboard.folder_id
+		LEFT OUTER JOIN dashboard_tag on dashboard.id = dashboard_tag.dashboard_id`)
+
+	sql.WriteString(fmt.Sprintf(" ORDER BY dashboard.title ASC LIMIT 5000"))
+
+	var res []DashboardSearchProjection
+
+	err := x.Sql(sql.String(), params...).Find(&res)
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+func createSearchWhereClause(query *search.FindPersistedDashboardsQuery, sql *bytes.Buffer, params []interface{}) []interface{} {
+	sql.WriteString(` dashboard.org_id=?`)
 	params = append(params, query.SignedInUser.OrgId)
 
 	if query.IsStarred {
@@ -262,7 +294,8 @@ func findDashboards(query *search.FindPersistedDashboardsQuery) ([]DashboardSear
 			  d.has_acl = 1 and
 				(da.user_id = ? or ugm.user_id = ? or ou.id is not null)
 			  and d.org_id = ?
-			  ))`
+			)
+		)`
 
 		sql.WriteString(allowedDashboardsSubQuery)
 		params = append(params, query.SignedInUser.UserId, query.SignedInUser.UserId, query.SignedInUser.OrgId)
@@ -286,16 +319,7 @@ func findDashboards(query *search.FindPersistedDashboardsQuery) ([]DashboardSear
 		params = append(params, query.FolderId)
 	}
 
-	sql.WriteString(fmt.Sprintf(" ORDER BY dashboard.title ASC LIMIT 1000"))
-
-	var res []DashboardSearchProjection
-
-	err := x.Sql(sql.String(), params...).Find(&res)
-	if err != nil {
-		return nil, err
-	}
-
-	return res, nil
+	return params
 }
 
 func SearchDashboards(query *search.FindPersistedDashboardsQuery) error {

pkg/services/sqlstore/dashboard_test.go

@@ -3,6 +3,7 @@ package sqlstore
 import (
 	"testing"
 
+	"github.com/go-xorm/xorm"
 	. "github.com/smartystreets/goconvey/convey"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
@@ -12,9 +13,10 @@ import (
 )
 
 func TestDashboardDataAccess(t *testing.T) {
+	var x *xorm.Engine
 
 	Convey("Testing DB", t, func() {
-		InitTestDB(t)
+		x = InitTestDB(t)
 
 		Convey("Given saved dashboard", func() {
 			savedFolder := insertTestDashboard("1 test dash folder", 1, 0, true, "prod", "webapp")

pkg/services/sqlstore/datasource_test.go

@@ -11,11 +11,10 @@ import (
 	"github.com/grafana/grafana/pkg/services/sqlstore/sqlutil"
 )
 
-func InitTestDB(t *testing.T) {
+func InitTestDB(t *testing.T) *xorm.Engine {
 	x, err := xorm.NewEngine(sqlutil.TestDB_Sqlite3.DriverName, sqlutil.TestDB_Sqlite3.ConnStr)
 	//x, err := xorm.NewEngine(sqlutil.TestDB_Mysql.DriverName, sqlutil.TestDB_Mysql.ConnStr)
 	//x, err := xorm.NewEngine(sqlutil.TestDB_Postgres.DriverName, sqlutil.TestDB_Postgres.ConnStr)
-	// x.ShowSQL()
 
 	// x.ShowSQL()
 
@@ -28,6 +27,8 @@ func InitTestDB(t *testing.T) {
 	if err := SetEngine(x); err != nil {
 		t.Fatal(err)
 	}
+
+	return x
 }
 
 type Test struct {