From 4551f059949c5341b23d6b62cbdea93e9a960d85 Mon Sep 17 00:00:00 2001 From: Alexander Emelin Date: Thu, 28 Apr 2022 19:29:09 +0300 Subject: [PATCH] Search: fix missing dashboards due to empty uid, log errors (#48361) --- pkg/services/searchV2/extract/dashboard.go | 24 +++++++++---------- .../searchV2/extract/dashboard_test.go | 3 ++- .../check-string-datasource-id-info.json | 1 - .../testdata/devdash-all-panels-info.json | 1 - .../devdash-graph-shared-tooltips-info.json | 1 - pkg/services/searchV2/extract/types.go | 1 - pkg/services/searchV2/index.go | 20 ++++++++++++---- pkg/services/searchV2/service.go | 24 +++++++++---------- 8 files changed, 42 insertions(+), 33 deletions(-) diff --git a/pkg/services/searchV2/extract/dashboard.go b/pkg/services/searchV2/extract/dashboard.go index fdfa5da231a..5ce13f38c2c 100644 --- a/pkg/services/searchV2/extract/dashboard.go +++ b/pkg/services/searchV2/extract/dashboard.go @@ -12,10 +12,11 @@ func logf(format string, a ...interface{}) { // nolint:gocyclo // ReadDashboard will take a byte stream and return dashboard info -func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo { - iter := jsoniter.Parse(jsoniter.ConfigDefault, stream, 1024) +func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, error) { dash := &DashboardInfo{} + iter := jsoniter.Parse(jsoniter.ConfigDefault, stream, 1024) + for l1Field := iter.ReadObject(); l1Field != ""; l1Field = iter.ReadObject() { // Skip null values so we don't need special int handling if iter.WhatIsNext() == jsoniter.NilValue { @@ -28,7 +29,7 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo { dash.ID = iter.ReadInt64() case "uid": - dash.UID = iter.ReadString() + iter.ReadString() case "title": dash.Title = iter.ReadString() @@ -67,10 +68,13 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo { case "time": obj, ok := iter.Read().(map[string]interface{}) if ok { - dash.TimeFrom, _ = obj["from"].(string) - dash.TimeTo, _ = obj["to"].(string) + if timeFrom, ok := obj["from"].(string); ok { + dash.TimeFrom = timeFrom + } + if timeTo, ok := obj["to"].(string); ok { + dash.TimeTo = timeTo + } } - case "panels": for iter.ReadArray() { dash.Panels = append(dash.Panels, readPanelInfo(iter, lookup)) @@ -79,7 +83,7 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo { case "rows": for iter.ReadArray() { v := iter.Read() - logf("[DASHBOARD.ROW???] id=%s // %v\n", dash.UID, v) + logf("[DASHBOARD.ROW???] id=%s // %v\n", dash.ID, v) } case "annotations": @@ -125,17 +129,13 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo { } } - if dash.UID == "" { - logf("All dashbaords should have a UID defined") - } - targets := newTargetInfo(lookup) for _, panel := range dash.Panels { targets.addPanel(panel) } dash.Datasource = targets.GetDatasourceInfo() - return dash + return dash, iter.Error } // will always return strings for now diff --git a/pkg/services/searchV2/extract/dashboard_test.go b/pkg/services/searchV2/extract/dashboard_test.go index 0bc3a17090b..e017f9c672a 100644 --- a/pkg/services/searchV2/extract/dashboard_test.go +++ b/pkg/services/searchV2/extract/dashboard_test.go @@ -44,7 +44,8 @@ func TestReadDashboard(t *testing.T) { } require.NoError(t, err) - dash := ReadDashboard(f, ds) + dash, err := ReadDashboard(f, ds) + require.NoError(t, err) out, err := json.MarshalIndent(dash, "", " ") require.NoError(t, err) diff --git a/pkg/services/searchV2/extract/testdata/check-string-datasource-id-info.json b/pkg/services/searchV2/extract/testdata/check-string-datasource-id-info.json index ec9dd91d010..ba57bdcecda 100644 --- a/pkg/services/searchV2/extract/testdata/check-string-datasource-id-info.json +++ b/pkg/services/searchV2/extract/testdata/check-string-datasource-id-info.json @@ -1,6 +1,5 @@ { "id": 250, - "uid": "K2X7hzwGk", "title": "fast streaming", "tags": null, "datasource": [ diff --git a/pkg/services/searchV2/extract/testdata/devdash-all-panels-info.json b/pkg/services/searchV2/extract/testdata/devdash-all-panels-info.json index 26e3218ec73..7ed3b10892f 100644 --- a/pkg/services/searchV2/extract/testdata/devdash-all-panels-info.json +++ b/pkg/services/searchV2/extract/testdata/devdash-all-panels-info.json @@ -1,5 +1,4 @@ { - "uid": "n1jR8vnnz", "title": "Panel tests - All panels", "tags": [ "gdev", diff --git a/pkg/services/searchV2/extract/testdata/devdash-graph-shared-tooltips-info.json b/pkg/services/searchV2/extract/testdata/devdash-graph-shared-tooltips-info.json index a9d9a9eccd8..9580ae315c5 100644 --- a/pkg/services/searchV2/extract/testdata/devdash-graph-shared-tooltips-info.json +++ b/pkg/services/searchV2/extract/testdata/devdash-graph-shared-tooltips-info.json @@ -1,5 +1,4 @@ { - "uid": "TX2VU59MZ", "title": "Panel Tests - shared tooltips", "tags": [ "gdev", diff --git a/pkg/services/searchV2/extract/types.go b/pkg/services/searchV2/extract/types.go index 7cbd72adf16..02eddbdcbfb 100644 --- a/pkg/services/searchV2/extract/types.go +++ b/pkg/services/searchV2/extract/types.go @@ -23,7 +23,6 @@ type PanelInfo struct { type DashboardInfo struct { ID int64 `json:"id,omitempty"` // internal ID - UID string `json:"uid,omitempty"` Title string `json:"title"` Description string `json:"description,omitempty"` Tags []string `json:"tags"` diff --git a/pkg/services/searchV2/index.go b/pkg/services/searchV2/index.go index 43571ca0757..79dbed598ed 100644 --- a/pkg/services/searchV2/index.go +++ b/pkg/services/searchV2/index.go @@ -74,10 +74,12 @@ func (i *dashboardIndex) run(ctx context.Context) error { } // Build on start for orgID 1 but keep lazy for others. - _, err = i.getDashboards(ctx, 1) + started := time.Now() + dashboards, err := i.getDashboards(ctx, 1) if err != nil { return fmt.Errorf("can't build dashboard search index for org ID 1: %w", err) } + i.logger.Info("Indexing for main org finished", "mainOrgIndexElapsed", time.Since(started), "numDashboards", len(dashboards)) for { select { @@ -245,7 +247,12 @@ func (i *dashboardIndex) getDashboards(ctx context.Context, orgId int64) ([]dash } type sqlDashboardLoader struct { - sql *sqlstore.SQLStore + sql *sqlstore.SQLStore + logger log.Logger +} + +func newSQLDashboardLoader(sql *sqlstore.SQLStore) *sqlDashboardLoader { + return &sqlDashboardLoader{sql: sql, logger: log.New("sqlDashboardLoader")} } func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, dashboardUID string) ([]dashboard, error) { @@ -260,6 +267,7 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das // Add the root folder ID (does not exist in SQL). dashboards = append(dashboards, dashboard{ id: 0, + uid: "", isFolder: true, folderID: 0, slug: "", @@ -267,7 +275,6 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das updated: time.Now(), info: &extract.DashboardInfo{ ID: 0, - UID: "", Title: "General", }, }) @@ -308,6 +315,11 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das } for _, row := range rows { + info, err := extract.ReadDashboard(bytes.NewReader(row.Data), lookup) + if err != nil { + l.logger.Warn("Error indexing dashboard data", "error", err, "dashboardId", row.Id, "dashboardSlug", row.Slug) + // But append info anyway for now, since we possibly extracted useful information. + } dashboards = append(dashboards, dashboard{ id: row.Id, uid: row.Uid, @@ -316,7 +328,7 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das slug: row.Slug, created: row.Created, updated: row.Updated, - info: extract.ReadDashboard(bytes.NewReader(row.Data), lookup), + info: info, }) lastID = row.Id } diff --git a/pkg/services/searchV2/service.go b/pkg/services/searchV2/service.go index c08ff51d2a3..4ee5d82063e 100644 --- a/pkg/services/searchV2/service.go +++ b/pkg/services/searchV2/service.go @@ -37,7 +37,7 @@ func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore s auth: &simpleSQLAuthService{ sql: sql, }, - dashboardIndex: newDashboardIndex(&sqlDashboardLoader{sql: sql}, entityEventStore), + dashboardIndex: newDashboardIndex(newSQLDashboardLoader(sql), entityEventStore), logger: log.New("searchV2"), } } @@ -56,7 +56,7 @@ func (s *StandardSearchService) Run(ctx context.Context) error { func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, _ DashboardQuery) *backend.DataResponse { rsp := &backend.DataResponse{} - dash, err := s.dashboardIndex.getDashboards(ctx, orgId) + dashboards, err := s.dashboardIndex.getDashboards(ctx, orgId) if err != nil { rsp.Error = err return rsp @@ -82,27 +82,27 @@ func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *back return rsp } - dash, err = s.applyAuthFilter(getSignedInUserQuery.Result, dash) + dashboards, err = s.applyAuthFilter(getSignedInUserQuery.Result, dashboards) if err != nil { rsp.Error = err return rsp } - rsp.Frames = metaToFrame(dash) + rsp.Frames = metaToFrame(dashboards) return rsp } -func (s *StandardSearchService) applyAuthFilter(user *models.SignedInUser, dash []dashboard) ([]dashboard, error) { +func (s *StandardSearchService) applyAuthFilter(user *models.SignedInUser, dashboards []dashboard) ([]dashboard, error) { filter, err := s.auth.GetDashboardReadFilter(user) if err != nil { return nil, err } - // create a list of all viewable dashboards for this user - res := make([]dashboard, 0, len(dash)) - for _, dash := range dash { - if filter(dash.info.UID) || (dash.isFolder && dash.info.UID == "") { // include the "General" folder + // create a list of all viewable dashboards for this user. + res := make([]dashboard, 0, len(dashboards)) + for _, dash := range dashboards { + if filter(dash.uid) || (dash.isFolder && dash.uid == "") { // include the "General" folder res = append(res, dash) } } @@ -208,14 +208,14 @@ func metaToFrame(meta []dashboard) data.Frames { for _, row := range meta { if row.isFolder { folderID.Append(row.id) - folderUID.Append(row.info.UID) + folderUID.Append(row.uid) folderName.Append(row.info.Title) folderDashCount.Append(int64(0)) // filled in later continue } dashID.Append(row.id) - dashUID.Append(row.info.UID) + dashUID.Append(row.uid) dashFolderID.Append(row.folderID) dashName.Append(row.info.Title) dashDescr.Append(row.info.Title) @@ -230,7 +230,7 @@ func metaToFrame(meta []dashboard) data.Frames { } folderCounter[row.folderID] = fcount + 1 - url := fmt.Sprintf("/d/%s/%s", row.info.UID, row.slug) + url := fmt.Sprintf("/d/%s/%s", row.uid, row.slug) dashURL.Append(url) // stats