Search: fix missing dashboards due to empty uid, log errors (#48361)

This commit is contained in:
Alexander Emelin 2022-04-28 19:29:09 +03:00 committed by GitHub
parent eeaa160ae8
commit 4551f05994
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 42 additions and 33 deletions

View File

@ -12,10 +12,11 @@ func logf(format string, a ...interface{}) {
// nolint:gocyclo
// ReadDashboard will take a byte stream and return dashboard info
func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo {
iter := jsoniter.Parse(jsoniter.ConfigDefault, stream, 1024)
func ReadDashboard(stream io.Reader, lookup DatasourceLookup) (*DashboardInfo, error) {
dash := &DashboardInfo{}
iter := jsoniter.Parse(jsoniter.ConfigDefault, stream, 1024)
for l1Field := iter.ReadObject(); l1Field != ""; l1Field = iter.ReadObject() {
// Skip null values so we don't need special int handling
if iter.WhatIsNext() == jsoniter.NilValue {
@ -28,7 +29,7 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo {
dash.ID = iter.ReadInt64()
case "uid":
dash.UID = iter.ReadString()
iter.ReadString()
case "title":
dash.Title = iter.ReadString()
@ -67,10 +68,13 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo {
case "time":
obj, ok := iter.Read().(map[string]interface{})
if ok {
dash.TimeFrom, _ = obj["from"].(string)
dash.TimeTo, _ = obj["to"].(string)
if timeFrom, ok := obj["from"].(string); ok {
dash.TimeFrom = timeFrom
}
if timeTo, ok := obj["to"].(string); ok {
dash.TimeTo = timeTo
}
}
case "panels":
for iter.ReadArray() {
dash.Panels = append(dash.Panels, readPanelInfo(iter, lookup))
@ -79,7 +83,7 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo {
case "rows":
for iter.ReadArray() {
v := iter.Read()
logf("[DASHBOARD.ROW???] id=%s // %v\n", dash.UID, v)
logf("[DASHBOARD.ROW???] id=%s // %v\n", dash.ID, v)
}
case "annotations":
@ -125,17 +129,13 @@ func ReadDashboard(stream io.Reader, lookup DatasourceLookup) *DashboardInfo {
}
}
if dash.UID == "" {
logf("All dashbaords should have a UID defined")
}
targets := newTargetInfo(lookup)
for _, panel := range dash.Panels {
targets.addPanel(panel)
}
dash.Datasource = targets.GetDatasourceInfo()
return dash
return dash, iter.Error
}
// will always return strings for now

View File

@ -44,7 +44,8 @@ func TestReadDashboard(t *testing.T) {
}
require.NoError(t, err)
dash := ReadDashboard(f, ds)
dash, err := ReadDashboard(f, ds)
require.NoError(t, err)
out, err := json.MarshalIndent(dash, "", " ")
require.NoError(t, err)

View File

@ -1,6 +1,5 @@
{
"id": 250,
"uid": "K2X7hzwGk",
"title": "fast streaming",
"tags": null,
"datasource": [

View File

@ -1,5 +1,4 @@
{
"uid": "n1jR8vnnz",
"title": "Panel tests - All panels",
"tags": [
"gdev",

View File

@ -1,5 +1,4 @@
{
"uid": "TX2VU59MZ",
"title": "Panel Tests - shared tooltips",
"tags": [
"gdev",

View File

@ -23,7 +23,6 @@ type PanelInfo struct {
type DashboardInfo struct {
ID int64 `json:"id,omitempty"` // internal ID
UID string `json:"uid,omitempty"`
Title string `json:"title"`
Description string `json:"description,omitempty"`
Tags []string `json:"tags"`

View File

@ -74,10 +74,12 @@ func (i *dashboardIndex) run(ctx context.Context) error {
}
// Build on start for orgID 1 but keep lazy for others.
_, err = i.getDashboards(ctx, 1)
started := time.Now()
dashboards, err := i.getDashboards(ctx, 1)
if err != nil {
return fmt.Errorf("can't build dashboard search index for org ID 1: %w", err)
}
i.logger.Info("Indexing for main org finished", "mainOrgIndexElapsed", time.Since(started), "numDashboards", len(dashboards))
for {
select {
@ -245,7 +247,12 @@ func (i *dashboardIndex) getDashboards(ctx context.Context, orgId int64) ([]dash
}
type sqlDashboardLoader struct {
sql *sqlstore.SQLStore
sql *sqlstore.SQLStore
logger log.Logger
}
func newSQLDashboardLoader(sql *sqlstore.SQLStore) *sqlDashboardLoader {
return &sqlDashboardLoader{sql: sql, logger: log.New("sqlDashboardLoader")}
}
func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, dashboardUID string) ([]dashboard, error) {
@ -260,6 +267,7 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
// Add the root folder ID (does not exist in SQL).
dashboards = append(dashboards, dashboard{
id: 0,
uid: "",
isFolder: true,
folderID: 0,
slug: "",
@ -267,7 +275,6 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
updated: time.Now(),
info: &extract.DashboardInfo{
ID: 0,
UID: "",
Title: "General",
},
})
@ -308,6 +315,11 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
}
for _, row := range rows {
info, err := extract.ReadDashboard(bytes.NewReader(row.Data), lookup)
if err != nil {
l.logger.Warn("Error indexing dashboard data", "error", err, "dashboardId", row.Id, "dashboardSlug", row.Slug)
// But append info anyway for now, since we possibly extracted useful information.
}
dashboards = append(dashboards, dashboard{
id: row.Id,
uid: row.Uid,
@ -316,7 +328,7 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
slug: row.Slug,
created: row.Created,
updated: row.Updated,
info: extract.ReadDashboard(bytes.NewReader(row.Data), lookup),
info: info,
})
lastID = row.Id
}

View File

@ -37,7 +37,7 @@ func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore s
auth: &simpleSQLAuthService{
sql: sql,
},
dashboardIndex: newDashboardIndex(&sqlDashboardLoader{sql: sql}, entityEventStore),
dashboardIndex: newDashboardIndex(newSQLDashboardLoader(sql), entityEventStore),
logger: log.New("searchV2"),
}
}
@ -56,7 +56,7 @@ func (s *StandardSearchService) Run(ctx context.Context) error {
func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, _ DashboardQuery) *backend.DataResponse {
rsp := &backend.DataResponse{}
dash, err := s.dashboardIndex.getDashboards(ctx, orgId)
dashboards, err := s.dashboardIndex.getDashboards(ctx, orgId)
if err != nil {
rsp.Error = err
return rsp
@ -82,27 +82,27 @@ func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *back
return rsp
}
dash, err = s.applyAuthFilter(getSignedInUserQuery.Result, dash)
dashboards, err = s.applyAuthFilter(getSignedInUserQuery.Result, dashboards)
if err != nil {
rsp.Error = err
return rsp
}
rsp.Frames = metaToFrame(dash)
rsp.Frames = metaToFrame(dashboards)
return rsp
}
func (s *StandardSearchService) applyAuthFilter(user *models.SignedInUser, dash []dashboard) ([]dashboard, error) {
func (s *StandardSearchService) applyAuthFilter(user *models.SignedInUser, dashboards []dashboard) ([]dashboard, error) {
filter, err := s.auth.GetDashboardReadFilter(user)
if err != nil {
return nil, err
}
// create a list of all viewable dashboards for this user
res := make([]dashboard, 0, len(dash))
for _, dash := range dash {
if filter(dash.info.UID) || (dash.isFolder && dash.info.UID == "") { // include the "General" folder
// create a list of all viewable dashboards for this user.
res := make([]dashboard, 0, len(dashboards))
for _, dash := range dashboards {
if filter(dash.uid) || (dash.isFolder && dash.uid == "") { // include the "General" folder
res = append(res, dash)
}
}
@ -208,14 +208,14 @@ func metaToFrame(meta []dashboard) data.Frames {
for _, row := range meta {
if row.isFolder {
folderID.Append(row.id)
folderUID.Append(row.info.UID)
folderUID.Append(row.uid)
folderName.Append(row.info.Title)
folderDashCount.Append(int64(0)) // filled in later
continue
}
dashID.Append(row.id)
dashUID.Append(row.info.UID)
dashUID.Append(row.uid)
dashFolderID.Append(row.folderID)
dashName.Append(row.info.Title)
dashDescr.Append(row.info.Title)
@ -230,7 +230,7 @@ func metaToFrame(meta []dashboard) data.Frames {
}
folderCounter[row.folderID] = fcount + 1
url := fmt.Sprintf("/d/%s/%s", row.info.UID, row.slug)
url := fmt.Sprintf("/d/%s/%s", row.uid, row.slug)
dashURL.Append(url)
// stats