From d00592ffa041c253953c4b0f9b2c4434919ecef7 Mon Sep 17 00:00:00 2001 From: owensmallwood Date: Wed, 15 Jan 2025 10:23:05 -0600 Subject: [PATCH] Unified Storage: Make all dashboard fields searchable (#98899) * wip. adding sprinkles fields. * some refactoring. Works with sprinkles now. * exclude top level dashboard hit fields from hit "fields" * adds unit test for DecodeCell helper * test can search for specific dashboard fields on bleve index * adds search handler tests for the fields and tests for fields when transforming the search req to a bleve search req * fix panic when calling fields.Set() with int32 * adds regression test * remove unneeded method on test mock client * fix linter issues * updates dashboard test data for bleve tests * remove DASHBOARD_LEGACY_ID from bleve_tests * dont cast twice * updates test to sort by dashboard_views_last_1_days * declare excludedFields outside of function * fixes sorting by dashboard fields - prepends "fields." to any dashboard fields we try to sort by * uses map for excludedFields * expects fields to be array-style url param * change method name * fixes failing tests - needed to add column type to mocks --- pkg/registry/apis/dashboard/search.go | 20 ++- pkg/registry/apis/dashboard/search_test.go | 100 +++++++++++ .../dashboards/service/dashboard_service.go | 34 +++- .../service/dashboard_service_test.go | 70 ++++++++ pkg/storage/unified/resource/document.go | 2 + pkg/storage/unified/resource/table.go | 13 ++ pkg/storage/unified/resource/table_test.go | 14 ++ pkg/storage/unified/search/bleve.go | 22 ++- pkg/storage/unified/search/bleve_mappings.go | 3 + pkg/storage/unified/search/bleve_test.go | 63 +++++-- pkg/storage/unified/search/dashboard.go | 153 +++++++++++++++++ .../search/testdata/manual-dashboard.json | 155 ++++++++++++++++++ 12 files changed, 629 insertions(+), 20 deletions(-) create mode 100644 pkg/registry/apis/dashboard/search_test.go diff --git a/pkg/registry/apis/dashboard/search.go b/pkg/registry/apis/dashboard/search.go index 22a9c840f5f..f7d33bad1d7 100644 --- a/pkg/registry/apis/dashboard/search.go +++ b/pkg/registry/apis/dashboard/search.go @@ -4,9 +4,11 @@ import ( "encoding/json" "net/http" "net/url" + "slices" "strconv" "strings" + "github.com/grafana/grafana/pkg/storage/unified/search" "go.opentelemetry.io/otel/trace" apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -226,12 +228,17 @@ func (s *SearchHandler) DoSearch(w http.ResponseWriter, r *http.Request) { Limit: int64(limit), Offset: int64(offset), Explain: queryParams.Has("explain") && queryParams.Get("explain") != "false", - Fields: []string{ - "title", - "folder", - "tags", - }, } + fields := []string{"title", "folder", "tags"} + if queryParams.Has("field") { + // add fields to search and exclude duplicates + for _, f := range queryParams["field"] { + if f != "" && !slices.Contains(fields, f) { + fields = append(fields, f) + } + } + } + searchRequest.Fields = fields // Add the folder constraint. Note this does not do recursive search folder := queryParams.Get("folder") @@ -277,6 +284,9 @@ func (s *SearchHandler) DoSearch(w http.ResponseWriter, r *http.Request) { // Add sorting if queryParams.Has("sort") { for _, sort := range queryParams["sort"] { + if slices.Contains(search.DashboardFields(), sort) { + sort = "fields." + sort + } s := &resource.ResourceSearchRequest_Sort{Field: sort} if strings.HasPrefix(sort, "-") { s.Desc = true diff --git a/pkg/registry/apis/dashboard/search_test.go b/pkg/registry/apis/dashboard/search_test.go new file mode 100644 index 00000000000..1feafbf88d0 --- /dev/null +++ b/pkg/registry/apis/dashboard/search_test.go @@ -0,0 +1,100 @@ +package dashboard + +import ( + "context" + "fmt" + "net/http/httptest" + "testing" + + "github.com/grafana/grafana/pkg/apimachinery/identity" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/grafana/grafana/pkg/services/user" + "github.com/grafana/grafana/pkg/storage/unified/resource" + "google.golang.org/grpc" +) + +func TestSearchHandlerFields(t *testing.T) { + // Create a mock client + mockClient := &MockClient{} + + // Initialize the search handler with the mock client + searchHandler := SearchHandler{ + log: log.New("test", "test"), + client: mockClient, + tracer: tracing.NewNoopTracerService(), + } + + t.Run("Multiple comma separated fields will be appended to default dashboard search fields", func(t *testing.T) { + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/search?field=field1&field=field2&field=field3", nil) + req.Header.Add("content-type", "application/json") + req = req.WithContext(identity.WithRequester(req.Context(), &user.SignedInUser{Namespace: "test"})) + + searchHandler.DoSearch(rr, req) + + if mockClient.LastSearchRequest == nil { + t.Fatalf("expected Search to be called, but it was not") + } + expectedFields := []string{"title", "folder", "tags", "field1", "field2", "field3"} + if fmt.Sprintf("%v", mockClient.LastSearchRequest.Fields) != fmt.Sprintf("%v", expectedFields) { + t.Errorf("expected fields %v, got %v", expectedFields, mockClient.LastSearchRequest.Fields) + } + }) + + t.Run("Single field will be appended to default dashboard search fields", func(t *testing.T) { + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/search?field=field1", nil) + req.Header.Add("content-type", "application/json") + req = req.WithContext(identity.WithRequester(req.Context(), &user.SignedInUser{Namespace: "test"})) + + searchHandler.DoSearch(rr, req) + + if mockClient.LastSearchRequest == nil { + t.Fatalf("expected Search to be called, but it was not") + } + expectedFields := []string{"title", "folder", "tags", "field1"} + if fmt.Sprintf("%v", mockClient.LastSearchRequest.Fields) != fmt.Sprintf("%v", expectedFields) { + t.Errorf("expected fields %v, got %v", expectedFields, mockClient.LastSearchRequest.Fields) + } + }) + + t.Run("Passing no fields will search using default dashboard fields", func(t *testing.T) { + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/search", nil) + req.Header.Add("content-type", "application/json") + req = req.WithContext(identity.WithRequester(req.Context(), &user.SignedInUser{Namespace: "test"})) + + searchHandler.DoSearch(rr, req) + + if mockClient.LastSearchRequest == nil { + t.Fatalf("expected Search to be called, but it was not") + } + expectedFields := []string{"title", "folder", "tags"} + if fmt.Sprintf("%v", mockClient.LastSearchRequest.Fields) != fmt.Sprintf("%v", expectedFields) { + t.Errorf("expected fields %v, got %v", expectedFields, mockClient.LastSearchRequest.Fields) + } + }) +} + +// MockClient implements the ResourceIndexClient interface for testing +type MockClient struct { + resource.ResourceIndexClient + + // Capture the last SearchRequest for assertions + LastSearchRequest *resource.ResourceSearchRequest +} + +func (m *MockClient) Search(ctx context.Context, in *resource.ResourceSearchRequest, opts ...grpc.CallOption) (*resource.ResourceSearchResponse, error) { + m.LastSearchRequest = in + + return &resource.ResourceSearchResponse{}, nil +} + +func (m *MockClient) GetStats(ctx context.Context, in *resource.ResourceStatsRequest, opts ...grpc.CallOption) (*resource.ResourceStatsResponse, error) { + return nil, nil +} + +func (m *MockClient) History(ctx context.Context, in *resource.HistoryRequest, opts ...grpc.CallOption) (*resource.HistoryResponse, error) { + return nil, nil +} diff --git a/pkg/services/dashboards/service/dashboard_service.go b/pkg/services/dashboards/service/dashboard_service.go index 99d064982e8..e9fa15309de 100644 --- a/pkg/services/dashboards/service/dashboard_service.go +++ b/pkg/services/dashboards/service/dashboard_service.go @@ -12,6 +12,7 @@ import ( "time" "github.com/google/uuid" + common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel" "golang.org/x/exp/slices" @@ -71,6 +72,16 @@ var ( tracer = otel.Tracer("github.com/grafana/grafana/pkg/services/dashboards/service") ) +var ( + excludedFields = map[string]string{ + resource.SEARCH_FIELD_EXPLAIN: "", + resource.SEARCH_FIELD_SCORE: "", + resource.SEARCH_FIELD_TITLE: "", + resource.SEARCH_FIELD_FOLDER: "", + resource.SEARCH_FIELD_TAGS: "", + } +) + type DashboardServiceImpl struct { cfg *setting.Cfg log log.Logger @@ -1949,11 +1960,11 @@ func ParseResults(result *resource.ResourceSearchResponse, offset int64) (*v0alp explainIDX = i case resource.SEARCH_FIELD_SCORE: scoreIDX = i - case "title": + case resource.SEARCH_FIELD_TITLE: titleIDX = i - case "folder": + case resource.SEARCH_FIELD_FOLDER: folderIDX = i - case "tags": + case resource.SEARCH_FIELD_TAGS: tagsIDX = i } } @@ -1967,11 +1978,28 @@ func ParseResults(result *resource.ResourceSearchResponse, offset int64) (*v0alp } for i, row := range result.Results.Rows { + fields := &common.Unstructured{} + for colIndex, col := range result.Results.Columns { + if _, ok := excludedFields[col.Name]; ok { + val, err := resource.DecodeCell(col, colIndex, row.Cells[colIndex]) + if err != nil { + return nil, err + } + // Some of the dashboard fields come in as int32, but we need to convert them to int64 or else fields.Set() will panic + int32Val, ok := val.(int32) + if ok { + val = int64(int32Val) + } + fields.Set(col.Name, val) + } + } + hit := &v0alpha1.DashboardHit{ Resource: row.Key.Resource, // folders | dashboards Name: row.Key.Name, // The Grafana UID Title: string(row.Cells[titleIDX]), Folder: string(row.Cells[folderIDX]), + Field: fields, } if tagsIDX > 0 && row.Cells[tagsIDX] != nil { _ = json.Unmarshal(row.Cells[tagsIDX], &hit.Tags) diff --git a/pkg/services/dashboards/service/dashboard_service_test.go b/pkg/services/dashboards/service/dashboard_service_test.go index 62e70c00cbe..a8768008f2b 100644 --- a/pkg/services/dashboards/service/dashboard_service_test.go +++ b/pkg/services/dashboards/service/dashboard_service_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/grafana/grafana/pkg/storage/unified/search" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -417,9 +418,11 @@ func TestGetDashboard(t *testing.T) { Columns: []*resource.ResourceTableColumnDefinition{ { Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, }, }, Rows: []*resource.ResourceTableRow{ @@ -636,9 +639,11 @@ func TestGetProvisionedDashboardData(t *testing.T) { Columns: []*resource.ResourceTableColumnDefinition{ { Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, }, }, Rows: []*resource.ResourceTableRow{ @@ -729,9 +734,11 @@ func TestGetProvisionedDashboardDataByDashboardID(t *testing.T) { Columns: []*resource.ResourceTableColumnDefinition{ { Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, }, }, Rows: []*resource.ResourceTableRow{ @@ -810,9 +817,11 @@ func TestGetProvisionedDashboardDataByDashboardUID(t *testing.T) { Columns: []*resource.ResourceTableColumnDefinition{ { Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, }, }, Rows: []*resource.ResourceTableRow{ @@ -917,9 +926,11 @@ func TestDeleteOrphanedProvisionedDashboards(t *testing.T) { Columns: []*resource.ResourceTableColumnDefinition{ { Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, }, }, Rows: []*resource.ResourceTableRow{ @@ -946,9 +957,11 @@ func TestDeleteOrphanedProvisionedDashboards(t *testing.T) { Columns: []*resource.ResourceTableColumnDefinition{ { Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, }, }, Rows: []*resource.ResourceTableRow{ @@ -1039,9 +1052,11 @@ func TestUnprovisionDashboard(t *testing.T) { Columns: []*resource.ResourceTableColumnDefinition{ { Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, }, }, Rows: []*resource.ResourceTableRow{ @@ -1095,9 +1110,11 @@ func TestGetDashboardsByPluginID(t *testing.T) { Columns: []*resource.ResourceTableColumnDefinition{ { Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, }, }, Rows: []*resource.ResourceTableRow{ @@ -1303,9 +1320,11 @@ func TestDeleteDashboard(t *testing.T) { Columns: []*resource.ResourceTableColumnDefinition{ { Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, }, }, Rows: []*resource.ResourceTableRow{ @@ -1422,12 +1441,15 @@ func TestSearchDashboards(t *testing.T) { Columns: []*resource.ResourceTableColumnDefinition{ { Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "tags", + Type: resource.ResourceTableColumnDefinition_STRING, }, }, Rows: []*resource.ResourceTableRow{ @@ -1521,9 +1543,11 @@ func TestGetDashboards(t *testing.T) { Columns: []*resource.ResourceTableColumnDefinition{ { Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, }, }, Rows: []*resource.ResourceTableRow{ @@ -1599,9 +1623,11 @@ func TestGetDashboardUIDByID(t *testing.T) { Columns: []*resource.ResourceTableColumnDefinition{ { Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, }, { Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, }, }, Rows: []*resource.ResourceTableRow{ @@ -1877,3 +1903,47 @@ func TestToUID(t *testing.T) { assert.Equal(t, "", result) }) } + +// regression test - parsing int32 values from search results was causing a panic +func TestParseResults(t *testing.T) { + resSearchResp := &resource.ResourceSearchResponse{ + Results: &resource.ResourceTable{ + Columns: []*resource.ResourceTableColumnDefinition{ + { + Name: "title", + Type: resource.ResourceTableColumnDefinition_STRING, + }, + { + Name: "folder", + Type: resource.ResourceTableColumnDefinition_STRING, + }, + { + Name: search.DASHBOARD_ERRORS_LAST_1_DAYS, + Type: resource.ResourceTableColumnDefinition_INT64, + }, + { + Name: search.DASHBOARD_LINK_COUNT, + Type: resource.ResourceTableColumnDefinition_INT32, + }, + }, + Rows: []*resource.ResourceTableRow{ + { + Key: &resource.ResourceKey{ + Name: "uid", + Resource: "dashboard", + }, + Cells: [][]byte{ + []byte("Dashboard 1"), + []byte("folder1"), + []byte("100"), + []byte("25"), + }, + }, + }, + }, + TotalHits: 1, + } + + _, err := ParseResults(resSearchResp, 0) + require.NoError(t, err) +} diff --git a/pkg/storage/unified/resource/document.go b/pkg/storage/unified/resource/document.go index 32116bb207d..ce5ecdf0d16 100644 --- a/pkg/storage/unified/resource/document.go +++ b/pkg/storage/unified/resource/document.go @@ -237,6 +237,8 @@ func (x *searchableDocumentFields) Fields() []string { } func (x *searchableDocumentFields) Field(name string) *ResourceTableColumnDefinition { + name = strings.TrimPrefix(name, "fields.") + f, ok := x.fields[name] if ok { return f.def diff --git a/pkg/storage/unified/resource/table.go b/pkg/storage/unified/resource/table.go index a58cf2dd13a..99c737ce3f7 100644 --- a/pkg/storage/unified/resource/table.go +++ b/pkg/storage/unified/resource/table.go @@ -182,6 +182,19 @@ type resourceTableColumn struct { OpenAPIFormat string } +// helper to decode a cell value +func DecodeCell(columnDef *ResourceTableColumnDefinition, index int, cellVal []byte) (any, error) { + col, err := newResourceTableColumn(columnDef, index) + if err != nil { + return nil, err + } + res, err := col.Decode(cellVal) + if err != nil { + return nil, err + } + return res, nil +} + // nolint:gocyclo func newResourceTableColumn(def *ResourceTableColumnDefinition, index int) (*resourceTableColumn, error) { col := &resourceTableColumn{def: def, index: index} diff --git a/pkg/storage/unified/resource/table_test.go b/pkg/storage/unified/resource/table_test.go index fcefe136749..9e37a902d1c 100644 --- a/pkg/storage/unified/resource/table_test.go +++ b/pkg/storage/unified/resource/table_test.go @@ -1,6 +1,8 @@ package resource import ( + "bytes" + "encoding/binary" "fmt" "path/filepath" "strings" @@ -312,3 +314,15 @@ func TestColumnEncoding(t *testing.T) { require.Empty(t, missingArrays, "missing array tests for types") }) } + +func TestDecodeCell(t *testing.T) { + colDef := &ResourceTableColumnDefinition{Type: ResourceTableColumnDefinition_INT64} + var buf bytes.Buffer + err := binary.Write(&buf, binary.BigEndian, int64(123)) + require.NoError(t, err) + + res, err := DecodeCell(colDef, 0, buf.Bytes()) + + require.NoError(t, err) + require.Equal(t, int64(123), res) +} diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index 916593eeb01..d0e0c376872 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -7,6 +7,7 @@ import ( "log/slog" "os" "path/filepath" + "slices" "strings" "sync" "time" @@ -489,8 +490,18 @@ func toBleveSearchRequest(req *resource.ResourceSearchRequest, access authz.Acce for _, f := range req.Facet { facets[f.Field] = bleve.NewFacetRequest(f.Field, int(f.Limit)) } + + // Convert resource-specific fields to bleve fields (just considers dashboard fields for now) + fields := make([]string, 0, len(req.Fields)) + for _, f := range req.Fields { + if slices.Contains(DashboardFields(), f) { + f = "fields." + f + } + fields = append(fields, f) + } + searchrequest := &bleve.SearchRequest{ - Fields: req.Fields, + Fields: fields, Size: int(req.Limit), From: int(req.Offset), Explain: req.Explain, @@ -720,7 +731,14 @@ func (b *bleveIndex) hitsToTable(selectFields []string, hits search.DocumentMatc row.Cells[i], err = json.Marshal(match.Expl) } default: - v := match.Fields[f.Name] + fieldName := f.Name + // since the bleve index fields mix common and resource-specific fields, it is possible a conflict can happen + // if a specific field is named the same as a common field + v := match.Fields[fieldName] + // fields that are specific to the resource get stored as fields., so we need to check for that + if v == nil { + v = match.Fields["fields."+fieldName] + } if v != nil { // Encode the value to protobuf row.Cells[i], err = encoders[i](v) diff --git a/pkg/storage/unified/search/bleve_mappings.go b/pkg/storage/unified/search/bleve_mappings.go index f9cdafc41d4..a1ec7d9fd6b 100644 --- a/pkg/storage/unified/search/bleve_mappings.go +++ b/pkg/storage/unified/search/bleve_mappings.go @@ -103,5 +103,8 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM labelMapper := bleve.NewDocumentMapping() mapper.AddSubDocumentMapping(resource.SEARCH_FIELD_LABELS, labelMapper) + fieldMapper := bleve.NewDocumentMapping() + mapper.AddSubDocumentMapping("fields", fieldMapper) + return mapper } diff --git a/pkg/storage/unified/search/bleve_test.go b/pkg/storage/unified/search/bleve_test.go index c9b650e7668..8f8f3bb902d 100644 --- a/pkg/storage/unified/search/bleve_test.go +++ b/pkg/storage/unified/search/bleve_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "testing" "time" @@ -76,9 +77,9 @@ func TestBleveBackend(t *testing.T) { TitleSort: "aaa (dash)", Folder: "xxx", Fields: map[string]any{ - DASHBOARD_LEGACY_ID: 12, - DASHBOARD_PANEL_TYPES: []string{"timeseries", "table"}, - DASHBOARD_ERRORS_TODAY: 25, + DASHBOARD_PANEL_TYPES: []string{"timeseries", "table"}, + DASHBOARD_ERRORS_TODAY: 25, + DASHBOARD_VIEWS_LAST_1_DAYS: 50, }, Labels: map[string]string{ utils.LabelKeyDeprecatedInternalID: "10", // nolint:staticcheck @@ -104,9 +105,9 @@ func TestBleveBackend(t *testing.T) { TitleSort: "bbb (dash)", Folder: "xxx", Fields: map[string]any{ - DASHBOARD_LEGACY_ID: 12, - DASHBOARD_PANEL_TYPES: []string{"timeseries"}, - DASHBOARD_ERRORS_TODAY: 40, + DASHBOARD_PANEL_TYPES: []string{"timeseries"}, + DASHBOARD_ERRORS_TODAY: 40, + DASHBOARD_VIEWS_LAST_1_DAYS: 100, }, Tags: []string{"aa"}, Labels: map[string]string{ @@ -136,10 +137,8 @@ func TestBleveBackend(t *testing.T) { Name: "repo2", Path: "path/in/repo2.yaml", }, - Fields: map[string]any{ - DASHBOARD_LEGACY_ID: 12, - }, - Tags: []string{"aa"}, + Fields: map[string]any{}, + Tags: []string{"aa"}, Labels: map[string]string{ "region": "west", }, @@ -217,6 +216,27 @@ func TestBleveBackend(t *testing.T) { rsp.Results.Rows[1].Key.Name, }) + // can get sprinkles fields + rsp, err = index.Search(ctx, nil, &resource.ResourceSearchRequest{ + Options: &resource.ListOptions{ + Key: key, + }, + Limit: 100000, + Fields: []string{DASHBOARD_ERRORS_TODAY, DASHBOARD_VIEWS_LAST_1_DAYS, "fieldThatDoesntExist"}, + SortBy: []*resource.ResourceSearchRequest_Sort{ + {Field: "fields." + DASHBOARD_VIEWS_LAST_1_DAYS, Desc: true}, + }, + }, nil) + require.NoError(t, err) + require.Equal(t, 2, len(rsp.Results.Columns)) + require.Equal(t, DASHBOARD_ERRORS_TODAY, rsp.Results.Columns[0].Name) + require.Equal(t, DASHBOARD_VIEWS_LAST_1_DAYS, rsp.Results.Columns[1].Name) + + // sorted descending so should start with highest dashboard_views_last_1_days (100) + val, err := resource.DecodeCell(rsp.Results.Columns[1], 0, rsp.Results.Rows[0].Cells[1]) + require.NoError(t, err) + require.Equal(t, int64(100), val) + // Now look for repositories found, err := index.ListRepositoryObjects(ctx, &resource.ListRepositoryObjectsRequest{ Name: "repo-1", @@ -408,6 +428,29 @@ func TestBleveBackend(t *testing.T) { }) } +func TestToBleveSearchRequest(t *testing.T) { + t.Run("will prepend 'fields.' to all dashboard fields", func(t *testing.T) { + fields := []string{"title", "name", "folder"} + fields = append(fields, DashboardFields()...) + resReq := &resource.ResourceSearchRequest{ + Options: &resource.ListOptions{}, + Fields: fields, + } + bleveReq, err := toBleveSearchRequest(resReq, nil) + if err != nil { + t.Fatalf("error creating bleve search request: %v", err) + } + + require.Equal(t, len(fields), len(bleveReq.Fields)) + for _, field := range DashboardFields() { + require.True(t, slices.Contains(bleveReq.Fields, "fields."+field)) + } + require.Contains(t, bleveReq.Fields, "title") + require.Contains(t, bleveReq.Fields, "name") + require.Contains(t, bleveReq.Fields, "folder") + }) +} + func asTimePointer(milli int64) *time.Time { if milli > 0 { t := time.UnixMilli(milli) diff --git a/pkg/storage/unified/search/dashboard.go b/pkg/storage/unified/search/dashboard.go index d4cd334ac04..b96cbf4ae0e 100644 --- a/pkg/storage/unified/search/dashboard.go +++ b/pkg/storage/unified/search/dashboard.go @@ -69,6 +69,126 @@ func DashboardBuilder(namespaced resource.NamespacedDocumentSupplier) (resource. Filterable: true, }, }, + { + Name: DASHBOARD_ERRORS_TODAY, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Number of errors that occurred today", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_ERRORS_LAST_1_DAYS, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Number of errors that occurred in the last 1 days", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_ERRORS_LAST_7_DAYS, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Number of errors that occurred in the last 7 days", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_ERRORS_LAST_30_DAYS, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Number of errors that occurred in the last 30 days", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_ERRORS_TOTAL, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Total number of errors", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_QUERIES_TODAY, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Number of queries that occurred today", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_QUERIES_LAST_1_DAYS, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Number of queries that occurred in the last 1 days", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_QUERIES_LAST_7_DAYS, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Number of queries that occurred in the last 7 days", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_QUERIES_LAST_30_DAYS, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Number of queries that occurred in the last 30 days", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_QUERIES_TOTAL, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Total number of queries", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_VIEWS_TODAY, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Number of views that occurred today", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_VIEWS_LAST_1_DAYS, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Number of views that occurred in the last 1 days", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_VIEWS_LAST_7_DAYS, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Number of views that occurred in the last 7 days", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_VIEWS_LAST_30_DAYS, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Number of views that occurred in the last 30 days", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, + { + Name: DASHBOARD_VIEWS_TOTAL, + Type: resource.ResourceTableColumnDefinition_INT64, + Description: "Total number of views", + Properties: &resource.ResourceTableColumnDefinition_Properties{ + Filterable: true, + }, + }, }) if namespaced == nil { namespaced = func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) { @@ -215,3 +335,36 @@ func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resou return doc, nil } + +func DashboardFields() []string { + baseFields := []string{ + DASHBOARD_LEGACY_ID, + DASHBOARD_SCHEMA_VERSION, + DASHBOARD_LINK_COUNT, + DASHBOARD_PANEL_TYPES, + DASHBOARD_DS_TYPES, + DASHBOARD_TRANSFORMATIONS, + } + + return append(baseFields, UsageInsightsFields()...) +} + +func UsageInsightsFields() []string { + return []string{ + DASHBOARD_VIEWS_LAST_1_DAYS, + DASHBOARD_VIEWS_LAST_7_DAYS, + DASHBOARD_VIEWS_LAST_30_DAYS, + DASHBOARD_VIEWS_TODAY, + DASHBOARD_VIEWS_TOTAL, + DASHBOARD_QUERIES_LAST_1_DAYS, + DASHBOARD_QUERIES_LAST_7_DAYS, + DASHBOARD_QUERIES_LAST_30_DAYS, + DASHBOARD_QUERIES_TODAY, + DASHBOARD_QUERIES_TOTAL, + DASHBOARD_ERRORS_LAST_1_DAYS, + DASHBOARD_ERRORS_LAST_7_DAYS, + DASHBOARD_ERRORS_LAST_30_DAYS, + DASHBOARD_ERRORS_TODAY, + DASHBOARD_ERRORS_TOTAL, + } +} diff --git a/pkg/storage/unified/search/testdata/manual-dashboard.json b/pkg/storage/unified/search/testdata/manual-dashboard.json index 5df8ab5def2..c8375063dbc 100644 --- a/pkg/storage/unified/search/testdata/manual-dashboard.json +++ b/pkg/storage/unified/search/testdata/manual-dashboard.json @@ -63,6 +63,111 @@ "format": "", "description": "How many links appear on the page", "priority": 0 + }, + { + "name": "errors_today", + "type": "number", + "format": "int64", + "description": "Number of errors that occurred today", + "priority": 0 + }, + { + "name": "errors_last_1_days", + "type": "number", + "format": "int64", + "description": "Number of errors that occurred in the last 1 days", + "priority": 0 + }, + { + "name": "errors_last_7_days", + "type": "number", + "format": "int64", + "description": "Number of errors that occurred in the last 7 days", + "priority": 0 + }, + { + "name": "errors_last_30_days", + "type": "number", + "format": "int64", + "description": "Number of errors that occurred in the last 30 days", + "priority": 0 + }, + { + "name": "errors_total", + "type": "number", + "format": "int64", + "description": "Total number of errors", + "priority": 0 + }, + { + "name": "queries_today", + "type": "number", + "format": "int64", + "description": "Number of queries that occurred today", + "priority": 0 + }, + { + "name": "queries_last_1_days", + "type": "number", + "format": "int64", + "description": "Number of queries that occurred in the last 1 days", + "priority": 0 + }, + { + "name": "queries_last_7_days", + "type": "number", + "format": "int64", + "description": "Number of queries that occurred in the last 7 days", + "priority": 0 + }, + { + "name": "queries_last_30_days", + "type": "number", + "format": "int64", + "description": "Number of queries that occurred in the last 30 days", + "priority": 0 + }, + { + "name": "queries_total", + "type": "number", + "format": "int64", + "description": "Total number of queries", + "priority": 0 + }, + { + "name": "views_today", + "type": "number", + "format": "int64", + "description": "Number of views that occurred today", + "priority": 0 + }, + { + "name": "views_last_1_days", + "type": "number", + "format": "int64", + "description": "Number of views that occurred in the last 1 days", + "priority": 0 + }, + { + "name": "views_last_7_days", + "type": "number", + "format": "int64", + "description": "Number of views that occurred in the last 7 days", + "priority": 0 + }, + { + "name": "views_last_30_days", + "type": "number", + "format": "int64", + "description": "Number of views that occurred in the last 30 days", + "priority": 0 + }, + { + "name": "views_total", + "type": "number", + "format": "int64", + "description": "Total number of views", + "priority": 0 } ], "rows": [ @@ -78,6 +183,21 @@ null, null, null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, null ], "object": { @@ -102,6 +222,23 @@ null, null, null, + [ + "timeseries" + ], + 40, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 100, + null, + null, null ], "object": { @@ -127,6 +264,24 @@ null, null, null, + [ + "timeseries", + "table" + ], + 25, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 50, + null, + null, null ], "object": {