IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

[search] grouping - support indexing dashboard tags, filtering by tag, listing tags (#95670)

Browse Source 
[search] grouping
This commit is contained in:
Scott Lepper 2024-10-31 13:09:17 -04:00 committed by GitHub
parent f539a70d6d
commit f8a5813573
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 1227 additions and 910 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
pkg/storage/unified/resource/index.go
View File

@ -151,13 +151,7 @@ func (i *Index) Init(ctx context.Context) error {
logger.Info("indexing batch", "kind", rt.Key.Resource, "count", len(list.Items))
//add changes to batches for shards with changes in the List
tenants, err := i.AddToBatches(ctx, list)
if err != nil {
return err
}
// Index the batches for tenants with changes if the batch is large enough
err = i.IndexBatches(ctx, i.opts.BatchSize, tenants)
err = i.writeBatch(ctx, list)
if err != nil {
return err
}
@ -187,6 +181,20 @@ func (i *Index) Init(ctx context.Context) error {
return nil
}
func (i *Index) writeBatch(ctx context.Context, list *ListResponse) error {
tenants, err := i.AddToBatches(ctx, list)
if err != nil {
return err
}
// Index the batches for tenants with changes if the batch is large enough
err = i.IndexBatches(ctx, i.opts.BatchSize, tenants)
if err != nil {
return err
}
return nil
}
func (i *Index) Index(ctx context.Context, data *Data) error {
ctx, span := i.tracer.Start(ctx, tracingPrexfixIndex+"Index")
defer span.End()
@ -235,15 +243,15 @@ func (i *Index) Delete(ctx context.Context, uid string, key *ResourceKey) error
return nil
}
func (i *Index) Search(ctx context.Context, tenant string, query string, limit int, offset int) ([]IndexedResource, error) {
func (i *Index) Search(ctx context.Context, request *SearchRequest) (*IndexResults, error) {
ctx, span := i.tracer.Start(ctx, tracingPrexfixIndex+"Search")
defer span.End()
logger := i.log.FromContext(ctx)
if tenant == "" {
tenant = "default"
if request.Tenant == "" {
request.Tenant = "default"
}
shard, err := i.getShard(tenant)
shard, err := i.getShard(request.Tenant)
if err != nil {
return nil, err
}
@ -251,23 +259,30 @@ func (i *Index) Search(ctx context.Context, tenant string, query string, limit i
if err != nil {
return nil, err
}
logger.Info("got index for tenant", "tenant", tenant, "docCount", docCount)
logger.Info("got index for tenant", "tenant", request.Tenant, "docCount", docCount)
fields, _ := shard.index.Fields()
logger.Debug("indexed fields", "fields", fields)
// use 10 as a default limit for now
if limit <= 0 {
limit = 10
if request.Limit <= 0 {
request.Limit = 10
}
req := bleve.NewSearchRequest(bleve.NewQueryStringQuery(query))
req.From = offset
req.Size = limit
query := bleve.NewQueryStringQuery(request.Query)
req := bleve.NewSearchRequest(query)
for _, group := range request.GroupBy {
facet := bleve.NewFacetRequest("Spec."+group.Name, int(group.Limit))
req.AddFacet(group.Name+"_facet", facet)
}
req.From = int(request.Offset)
req.Size = int(request.Size)
req.Fields = []string{"*"} // return all indexed fields in search results
logger.Info("searching index", "query", query, "tenant", tenant)
logger.Info("searching index", "query", request.Query, "tenant", request.Tenant)
res, err := shard.index.Search(req)
if err != nil {
return nil, err
@ -282,7 +297,15 @@ func (i *Index) Search(ctx context.Context, tenant string, query string, limit i
results[resKey] = ir
}
return results, nil
groups := []*Group{}
for _, group := range request.GroupBy {
groupByFacet := res.Facets[group.Name+"_facet"]
for _, term := range groupByFacet.Terms.Terms() {
groups = append(groups, &Group{Name: term.Term, Count: int64(term.Count)})
}
}
return &IndexResults{Values: results, Groups: groups}, nil
}
func (i *Index) Count() (uint64, error) {

11
pkg/storage/unified/resource/index_mapping.go
View File

@ -25,6 +25,11 @@ type IndexedResource struct {
Spec any
}
type IndexResults struct {
Values []IndexedResource
Groups []*Group
}
func (ir IndexedResource) FromSearchHit(hit *search.DocumentMatch) IndexedResource {
ir.Uid = hit.Fields["Uid"].(string)
ir.Kind = hit.Fields["Kind"].(string)
@ -178,6 +183,10 @@ func getSpecObjectMappings() map[string][]SpecFieldMapping {
Field: "description",
Type: "string",
},
{
Field: "tags",
Type: "string[]",
},
},
}
@ -198,7 +207,7 @@ func createSpecObjectMapping(kind string) *mapping.DocumentMapping {
// Create a field mapping based on field type
switch fieldType {
case "string":
case "string", "string[]":
specMapping.AddFieldMappingsAt(fieldName, bleve.NewTextFieldMapping())
case "int", "int64", "float64":
specMapping.AddFieldMappingsAt(fieldName, bleve.NewNumericFieldMapping())

5
pkg/storage/unified/resource/index_server.go
View File

@ -28,17 +28,18 @@ func (is *IndexServer) Search(ctx context.Context, req *SearchRequest) (*SearchR
ctx, span := is.tracer.Start(ctx, tracingPrefixIndexServer+"Search")
defer span.End()
results, err := is.index.Search(ctx, req.Tenant, req.Query, int(req.Limit), int(req.Offset))
results, err := is.index.Search(ctx, req)
if err != nil {
return nil, err
}
res := &SearchResponse{}
for _, r := range results {
for _, r := range results.Values {
resJsonBytes, err := json.Marshal(r)
if err != nil {
return nil, err
}
res.Items = append(res.Items, &ResourceWrapper{Value: resJsonBytes})
res.Groups = results.Groups
}
return res, nil
}

58
pkg/storage/unified/resource/index_test.go
View File

@ -21,22 +21,35 @@ const testTenant = "default"
var testContext = context.Background()
func TestIndexDashboard(t *testing.T) {
data, err := os.ReadFile("./testdata/dashboard-resource.json")
require.NoError(t, err)
data := readTestData(t, "dashboard-resource.json")
list := &ListResponse{Items: []*ResourceWrapper{{Value: data}}}
index := newTestIndex(t)
_, err = index.AddToBatches(testContext, list)
index := newTestIndex(t, 1)
err := index.writeBatch(testContext, list)
require.NoError(t, err)
err = index.IndexBatches(testContext, 1, []string{testTenant})
require.NoError(t, err)
assertCountEquals(t, index, 1)
assertSearchCountEquals(t, index, 1)
assertSearchCountEquals(t, index, "*", 1)
}
func TestIndexDashboardWithTags(t *testing.T) {
data := readTestData(t, "dashboard-tagged-resource.json")
data2 := readTestData(t, "dashboard-tagged-resource2.json")
list := &ListResponse{Items: []*ResourceWrapper{{Value: data}, {Value: data2}}}
index := newTestIndex(t, 2)
err := index.writeBatch(testContext, list)
require.NoError(t, err)
assertCountEquals(t, index, 2)
assertSearchCountEquals(t, index, "tag1", 2)
assertSearchCountEquals(t, index, "tag4", 1)
assertSearchGroupCountEquals(t, index, "*", "tags", 4)
assertSearchGroupCountEquals(t, index, "tag4", "tags", 3)
}
func TestIndexBatch(t *testing.T) {
index := newTestIndex(t)
index := newTestIndex(t, 1000)
startAll := time.Now()
ns := namespaces()
@ -105,7 +118,7 @@ func namespaces() []string {
return ns
}
func newTestIndex(t *testing.T) *Index {
func newTestIndex(t *testing.T, batchSize int) *Index {
tracingCfg := tracing.NewEmptyTracingConfig()
trace, err := tracing.ProvideService(tracingCfg)
require.NoError(t, err)
@ -117,7 +130,7 @@ func newTestIndex(t *testing.T) *Index {
opts: Opts{
ListLimit: 5000,
Workers: 10,
BatchSize: 1000,
BatchSize: batchSize,
},
}
}
@ -128,8 +141,25 @@ func assertCountEquals(t *testing.T, index *Index, expected uint64) {
assert.Equal(t, expected, total)
}
func assertSearchCountEquals(t *testing.T, index *Index, expected int) {
results, err := index.Search(testContext, testTenant, "*", expected+1, 0)
func assertSearchCountEquals(t *testing.T, index *Index, search string, expected int64) {
req := &SearchRequest{Query: search, Tenant: testTenant, Limit: expected + 1, Offset: 0, Size: expected + 1}
results, err := index.Search(testContext, req)
require.NoError(t, err)
assert.Equal(t, expected, len(results))
assert.Equal(t, expected, int64(len(results.Values)))
}
func assertSearchGroupCountEquals(t *testing.T, index *Index, search string, group string, expected int64) {
groupBy := []*GroupBy{{Name: group, Limit: 100}}
req := &SearchRequest{Query: search, Tenant: testTenant, Limit: expected + 1, Offset: 0, Size: expected + 1, GroupBy: groupBy}
results, err := index.Search(testContext, req)
require.NoError(t, err)
assert.Equal(t, expected, int64(len(results.Groups)))
}
func readTestData(t *testing.T, name string) []byte {
// We can ignore the gosec G304 because this is only for tests
// nolint:gosec
data, err := os.ReadFile("./testdata/" + name)
require.NoError(t, err)
return data
}

1239
pkg/storage/unified/resource/resource.pb.go
View File

File diff suppressed because it is too large Load Diff

15
pkg/storage/unified/resource/resource.proto
View File

@ -334,10 +334,25 @@ message SearchRequest {
// pagination support
int64 limit = 5;
int64 offset = 6;
// batch size (optional)
int64 size = 7;
// grouping (optional)
repeated GroupBy groupBy = 8;
}
message GroupBy {
string name = 1;
int64 limit = 2;
}
message Group {
string name = 1;
int64 count = 2;
}
message SearchResponse {
repeated ResourceWrapper items = 1;
repeated Group groups = 2;
}
message HistoryRequest {

284
pkg/storage/unified/resource/testdata/dashboard-tagged-resource.json vendored Normal file
View File

@ -0,0 +1,284 @@
{
"kind": "Dashboard",
"apiVersion": "dashboard.grafana.app/v0alpha1",
"metadata": {
"name": "adg5xd8",
"namespace": "default",
"uid": "86ab200a-e8b0-47ce-bbc1-8c2e078b0956",
"creationTimestamp": "2024-10-30T20:24:07Z",
"annotations": {
"grafana.app/createdBy": "user:be2g71ke8yoe8b",
"grafana.app/originHash": "Grafana v9.2.0 (NA)",
"grafana.app/originName": "UI",
"grafana.app/originPath": "/dashboard/new"
},
"managedFields": [
{
"manager": "Mozilla",
"operation": "Update",
"apiVersion": "dashboard.grafana.app/v0alpha1",
"time": "2024-10-30T20:24:07Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:metadata": {
"f:annotations": {
".": {},
"f:grafana.app/originHash": {},
"f:grafana.app/originName": {},
"f:grafana.app/originPath": {}
},
"f:generateName": {}
},
"f:spec": {
"f:annotations": {
".": {},
"f:list": {}
},
"f:description": {},
"f:editable": {},
"f:fiscalYearStartMonth": {},
"f:graphTooltip": {},
"f:id": {},
"f:links": {},
"f:panels": {},
"f:preload": {},
"f:schemaVersion": {},
"f:tags": {},
"f:templating": {
".": {},
"f:list": {}
},
"f:timepicker": {},
"f:timezone": {},
"f:title": {},
"f:uid": {},
"f:version": {},
"f:weekStart": {}
}
}
}
]
},
"spec": {
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations \u0026 Alerts",
"type": "dashboard"
}
]
},
"description": "A dashboard with tags",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.2.0",
"targets": [
{
"refId": "A"
}
],
"title": "Panel 2",
"type": "timeseries"
},
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.2.0",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"refId": "A"
}
],
"title": "Another Panel",
"type": "timeseries"
}
],
"preload": false,
"schemaVersion": 40,
"tags": [
"tag1",
"tag2",
"tag3"
],
"templating": {
"list": []
},
"timepicker": {},
"timezone": "browser",
"title": "Dashboard with Tags",
"uid": "",
"version": 0,
"weekStart": ""
}
}

284
pkg/storage/unified/resource/testdata/dashboard-tagged-resource2.json vendored Normal file
View File

@ -0,0 +1,284 @@
{
"kind": "Dashboard",
"apiVersion": "dashboard.grafana.app/v0alpha1",
"metadata": {
"name": "foo",
"namespace": "default",
"uid": "86ab200a-e8b0-47ce-bbc1-8c2e078b0956-2",
"creationTimestamp": "2024-10-30T20:24:07Z",
"annotations": {
"grafana.app/createdBy": "user:be2g71ke8yoe8b",
"grafana.app/originHash": "Grafana v9.2.0 (NA)",
"grafana.app/originName": "UI",
"grafana.app/originPath": "/dashboard/new"
},
"managedFields": [
{
"manager": "Mozilla",
"operation": "Update",
"apiVersion": "dashboard.grafana.app/v0alpha1",
"time": "2024-10-30T20:24:07Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:metadata": {
"f:annotations": {
".": {},
"f:grafana.app/originHash": {},
"f:grafana.app/originName": {},
"f:grafana.app/originPath": {}
},
"f:generateName": {}
},
"f:spec": {
"f:annotations": {
".": {},
"f:list": {}
},
"f:description": {},
"f:editable": {},
"f:fiscalYearStartMonth": {},
"f:graphTooltip": {},
"f:id": {},
"f:links": {},
"f:panels": {},
"f:preload": {},
"f:schemaVersion": {},
"f:tags": {},
"f:templating": {
".": {},
"f:list": {}
},
"f:timepicker": {},
"f:timezone": {},
"f:title": {},
"f:uid": {},
"f:version": {},
"f:weekStart": {}
}
}
}
]
},
"spec": {
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations \u0026 Alerts",
"type": "dashboard"
}
]
},
"description": "Another dashboard with tags",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.2.0",
"targets": [
{
"refId": "A"
}
],
"title": "Panel 2",
"type": "timeseries"
},
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.2.0",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"refId": "A"
}
],
"title": "Another Panel",
"type": "timeseries"
}
],
"preload": false,
"schemaVersion": 40,
"tags": [
"tag1",
"tag2",
"tag4"
],
"templating": {
"list": []
},
"timepicker": {},
"timezone": "browser",
"title": "Dashboard with Tags",
"uid": "",
"version": 0,
"weekStart": ""
}
}