Search: fix missing dashboards due to empty uid, log errors (#48361)

pkg/services/searchV2/extract/dashboard.go

@@ -12,10 +12,11 @@ func logf(format string, a ...interface{}) {
 
 // nolint:gocyclo
 // ReadDashboard will take a byte stream and return dashboard info
-func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo {
+func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, error) {
-	iter := jsoniter.Parse(jsoniter.ConfigDefault, stream, 1024)
 	dash := &DashboardInfo{}
 
+	iter := jsoniter.Parse(jsoniter.ConfigDefault, stream, 1024)
+
 	for l1Field := iter.ReadObject(); l1Field != ""; l1Field = iter.ReadObject() {
 		// Skip null values so we don't need special int handling
 		if iter.WhatIsNext() == jsoniter.NilValue {
@@ -28,7 +29,7 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo {
 			dash.ID = iter.ReadInt64()
 
 		case "uid":
-			dash.UID = iter.ReadString()
+			iter.ReadString()
 
 		case "title":
 			dash.Title = iter.ReadString()
@@ -67,10 +68,13 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo {
 		case "time":
 			obj, ok := iter.Read().(map[string]interface{})
 			if ok {
-				dash.TimeFrom, _ = obj["from"].(string)
+				if timeFrom, ok := obj["from"].(string); ok {
-				dash.TimeTo, _ = obj["to"].(string)
+					dash.TimeFrom = timeFrom
+				}
+				if timeTo, ok := obj["to"].(string); ok {
+					dash.TimeTo = timeTo
+				}
 			}
-
 		case "panels":
 			for iter.ReadArray() {
 				dash.Panels = append(dash.Panels, readPanelInfo(iter, lookup))
@@ -79,7 +83,7 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo {
 		case "rows":
 			for iter.ReadArray() {
 				v := iter.Read()
-				logf("[DASHBOARD.ROW???] id=%s // %v\n", dash.UID, v)
+				logf("[DASHBOARD.ROW???] id=%s // %v\n", dash.ID, v)
 			}
 
 		case "annotations":
@@ -125,17 +129,13 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo {
 		}
 	}
 
-	if dash.UID == "" {
-		logf("All dashbaords should have a UID defined")
-	}
-
 	targets := newTargetInfo(lookup)
 	for _, panel := range dash.Panels {
 		targets.addPanel(panel)
 	}
 	dash.Datasource = targets.GetDatasourceInfo()
 
-	return dash
+	return dash, iter.Error
 }
 
 // will always return strings for now

pkg/services/searchV2/extract/dashboard_test.go

@@ -44,7 +44,8 @@ func TestReadDashboard(t *testing.T) {
 		}
 		require.NoError(t, err)
 
-		dash := ReadDashboard(f, ds)
+		dash, err := ReadDashboard(f, ds)
+		require.NoError(t, err)
 		out, err := json.MarshalIndent(dash, "", "  ")
 		require.NoError(t, err)
 

pkg/services/searchV2/extract/testdata/check-string-datasource-id-info.json

@@ -1,6 +1,5 @@
 {
   "id": 250,
-  "uid": "K2X7hzwGk",
   "title": "fast streaming",
   "tags": null,
   "datasource": [

pkg/services/searchV2/extract/testdata/devdash-all-panels-info.json

@@ -1,5 +1,4 @@
 {
-  "uid": "n1jR8vnnz",
   "title": "Panel tests - All panels",
   "tags": [
     "gdev",

pkg/services/searchV2/extract/testdata/devdash-graph-shared-tooltips-info.json

@@ -1,5 +1,4 @@
 {
-  "uid": "TX2VU59MZ",
   "title": "Panel Tests - shared tooltips",
   "tags": [
     "gdev",

pkg/services/searchV2/extract/types.go

@@ -23,7 +23,6 @@ type PanelInfo struct {
 
 type DashboardInfo struct {
 	ID            int64           `json:"id,omitempty"` // internal ID
-	UID           string          `json:"uid,omitempty"`
 	Title         string          `json:"title"`
 	Description   string          `json:"description,omitempty"`
 	Tags          []string        `json:"tags"`

pkg/services/searchV2/index.go

@@ -74,10 +74,12 @@ func (i *dashboardIndex) run(ctx context.Context) error {
 	}
 
 	// Build on start for orgID 1 but keep lazy for others.
-	_, err = i.getDashboards(ctx, 1)
+	started := time.Now()
+	dashboards, err := i.getDashboards(ctx, 1)
 	if err != nil {
 		return fmt.Errorf("can't build dashboard search index for org ID 1: %w", err)
 	}
+	i.logger.Info("Indexing for main org finished", "mainOrgIndexElapsed", time.Since(started), "numDashboards", len(dashboards))
 
 	for {
 		select {
@@ -245,7 +247,12 @@ func (i *dashboardIndex) getDashboards(ctx context.Context, orgId int64) ([]dash
 }
 
 type sqlDashboardLoader struct {
-	sql *sqlstore.SQLStore
+	sql    *sqlstore.SQLStore
+	logger log.Logger
+}
+
+func newSQLDashboardLoader(sql *sqlstore.SQLStore) *sqlDashboardLoader {
+	return &sqlDashboardLoader{sql: sql, logger: log.New("sqlDashboardLoader")}
 }
 
 func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, dashboardUID string) ([]dashboard, error) {
@@ -260,6 +267,7 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
 		// Add the root folder ID (does not exist in SQL).
 		dashboards = append(dashboards, dashboard{
 			id:       0,
+			uid:      "",
 			isFolder: true,
 			folderID: 0,
 			slug:     "",
@@ -267,7 +275,6 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
 			updated:  time.Now(),
 			info: &extract.DashboardInfo{
 				ID:    0,
-				UID:   "",
 				Title: "General",
 			},
 		})
@@ -308,6 +315,11 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
 		}
 
 		for _, row := range rows {
+			info, err := extract.ReadDashboard(bytes.NewReader(row.Data), lookup)
+			if err != nil {
+				l.logger.Warn("Error indexing dashboard data", "error", err, "dashboardId", row.Id, "dashboardSlug", row.Slug)
+				// But append info anyway for now, since we possibly extracted useful information.
+			}
 			dashboards = append(dashboards, dashboard{
 				id:       row.Id,
 				uid:      row.Uid,
@@ -316,7 +328,7 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
 				slug:     row.Slug,
 				created:  row.Created,
 				updated:  row.Updated,
-				info:     extract.ReadDashboard(bytes.NewReader(row.Data), lookup),
+				info:     info,
 			})
 			lastID = row.Id
 		}

pkg/services/searchV2/service.go

@@ -37,7 +37,7 @@ func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore s
 		auth: &simpleSQLAuthService{
 			sql: sql,
 		},
-		dashboardIndex: newDashboardIndex(&sqlDashboardLoader{sql: sql}, entityEventStore),
+		dashboardIndex: newDashboardIndex(newSQLDashboardLoader(sql), entityEventStore),
 		logger:         log.New("searchV2"),
 	}
 }
@@ -56,7 +56,7 @@ func (s *StandardSearchService) Run(ctx context.Context) error {
 func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, _ DashboardQuery) *backend.DataResponse {
 	rsp := &backend.DataResponse{}
 
-	dash, err := s.dashboardIndex.getDashboards(ctx, orgId)
+	dashboards, err := s.dashboardIndex.getDashboards(ctx, orgId)
 	if err != nil {
 		rsp.Error = err
 		return rsp
@@ -82,27 +82,27 @@ func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *back
 		return rsp
 	}
 
-	dash, err = s.applyAuthFilter(getSignedInUserQuery.Result, dash)
+	dashboards, err = s.applyAuthFilter(getSignedInUserQuery.Result, dashboards)
 	if err != nil {
 		rsp.Error = err
 		return rsp
 	}
 
-	rsp.Frames = metaToFrame(dash)
+	rsp.Frames = metaToFrame(dashboards)
 
 	return rsp
 }
 
-func (s *StandardSearchService) applyAuthFilter(user *models.SignedInUser, dash []dashboard) ([]dashboard, error) {
+func (s *StandardSearchService) applyAuthFilter(user *models.SignedInUser, dashboards []dashboard) ([]dashboard, error) {
 	filter, err := s.auth.GetDashboardReadFilter(user)
 	if err != nil {
 		return nil, err
 	}
 
-	// create a list of all viewable dashboards for this user
+	// create a list of all viewable dashboards for this user.
-	res := make([]dashboard, 0, len(dash))
+	res := make([]dashboard, 0, len(dashboards))
-	for _, dash := range dash {
+	for _, dash := range dashboards {
-		if filter(dash.info.UID) || (dash.isFolder && dash.info.UID == "") { // include the "General" folder
+		if filter(dash.uid) || (dash.isFolder && dash.uid == "") { // include the "General" folder
 			res = append(res, dash)
 		}
 	}
@@ -208,14 +208,14 @@ func metaToFrame(meta []dashboard) data.Frames {
 	for _, row := range meta {
 		if row.isFolder {
 			folderID.Append(row.id)
-			folderUID.Append(row.info.UID)
+			folderUID.Append(row.uid)
 			folderName.Append(row.info.Title)
 			folderDashCount.Append(int64(0)) // filled in later
 			continue
 		}
 
 		dashID.Append(row.id)
-		dashUID.Append(row.info.UID)
+		dashUID.Append(row.uid)
 		dashFolderID.Append(row.folderID)
 		dashName.Append(row.info.Title)
 		dashDescr.Append(row.info.Title)
@@ -230,7 +230,7 @@ func metaToFrame(meta []dashboard) data.Frames {
 		}
 		folderCounter[row.folderID] = fcount + 1
 
-		url := fmt.Sprintf("/d/%s/%s", row.info.UID, row.slug)
+		url := fmt.Sprintf("/d/%s/%s", row.uid, row.slug)
 		dashURL.Append(url)
 
 		// stats