IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

[search] minimal dashboard indexing (#95631)

Browse Source 
[search] minimal dashboard indexing
This commit is contained in:
Scott Lepper 2024-10-31 07:46:00 -04:00 committed by GitHub
parent b1157dbd7a
commit 7619d9cb11
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 258 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/storage/unified/resource/index_mapping.go
View File

@ -169,6 +169,16 @@ func getSpecObjectMappings() map[string][]SpecFieldMapping {
Type: "string", Type: "string",
}, },
}, },
"Dashboard": {
{
Field: "title",
Type: "string",
},
{
Field: "description",
Type: "string",
},
},
} }
return mappings return mappings

89
pkg/storage/unified/resource/index_test.go
View File

@ -3,6 +3,7 @@ package resource
import ( import (
"context" "context"
"fmt" "fmt"
"os"
"strconv" "strconv"
"strings" "strings"
"testing" "testing"
@ -11,60 +12,53 @@ import (
"github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/infra/tracing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/exp/rand" "golang.org/x/exp/rand"
) )
const testTenant = "default"
var testContext = context.Background()
func TestIndexDashboard(t *testing.T) {
data, err := os.ReadFile("./testdata/dashboard-resource.json")
require.NoError(t, err)
list := &ListResponse{Items: []*ResourceWrapper{{Value: data}}}
index := newTestIndex(t)
_, err = index.AddToBatches(testContext, list)
require.NoError(t, err)
err = index.IndexBatches(testContext, 1, []string{testTenant})
require.NoError(t, err)
assertCountEquals(t, index, 1)
assertSearchCountEquals(t, index, 1)
}
func TestIndexBatch(t *testing.T) { func TestIndexBatch(t *testing.T) {
tracingCfg := tracing.NewEmptyTracingConfig() index := newTestIndex(t)
trace, err := tracing.ProvideService(tracingCfg)
if err != nil {
t.Fatal(err)
}
index := &Index{
tracer: trace,
shards: make(map[string]Shard),
log: log.New("unifiedstorage.search.index"),
opts: Opts{
ListLimit: 5000,
Workers: 10,
BatchSize: 1000,
},
}
ctx := context.Background()
startAll := time.Now() startAll := time.Now()
ns := namespaces() ns := namespaces()
// simulate 10 List calls // simulate 10 List calls
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
list := &ListResponse{Items: loadTestItems(strconv.Itoa(i), ns)} list := &ListResponse{Items: loadTestItems(strconv.Itoa(i), ns)}
start := time.Now() start := time.Now()
_, err = index.AddToBatches(ctx, list) _, err := index.AddToBatches(testContext, list)
if err != nil { require.NoError(t, err)
t.Fatal(err)
}
elapsed := time.Since(start) elapsed := time.Since(start)
fmt.Println("Time elapsed:", elapsed) fmt.Println("Time elapsed:", elapsed)
} }
// index all batches for each shard/tenant // index all batches for each shard/tenant
err = index.IndexBatches(ctx, 1, ns) err := index.IndexBatches(testContext, 1, ns)
if err != nil { require.NoError(t, err)
t.Fatal(err)
}
elapsed := time.Since(startAll) elapsed := time.Since(startAll)
fmt.Println("Total Time elapsed:", elapsed) fmt.Println("Total Time elapsed:", elapsed)
assert.Equal(t, len(ns), len(index.shards)) assert.Equal(t, len(ns), len(index.shards))
assertCountEquals(t, index, 100000)
total, err := index.Count()
if err != nil {
t.Fatal(err)
}
assert.Equal(t, uint64(100000), total)
} }
func loadTestItems(uid string, tenants []string) []*ResourceWrapper { func loadTestItems(uid string, tenants []string) []*ResourceWrapper {
@ -110,3 +104,32 @@ func namespaces() []string {
} }
return ns return ns
} }
func newTestIndex(t *testing.T) *Index {
tracingCfg := tracing.NewEmptyTracingConfig()
trace, err := tracing.ProvideService(tracingCfg)
require.NoError(t, err)
return &Index{
tracer: trace,
shards: make(map[string]Shard),
log: log.New("unifiedstorage.search.index"),
opts: Opts{
ListLimit: 5000,
Workers: 10,
BatchSize: 1000,
},
}
}
func assertCountEquals(t *testing.T, index *Index, expected uint64) {
total, err := index.Count()
require.NoError(t, err)
assert.Equal(t, expected, total)
}
func assertSearchCountEquals(t *testing.T, index *Index, expected int) {
results, err := index.Search(testContext, testTenant, "*", expected+1, 0)
require.NoError(t, err)
assert.Equal(t, expected, len(results))
}

192
pkg/storage/unified/resource/testdata/dashboard-resource.json vendored Normal file
View File

@ -0,0 +1,192 @@
{
"kind": "Dashboard",
"apiVersion": "dashboard.grafana.app/v0alpha1",
"metadata": {
"name": "adfbg6f",
"namespace": "default",
"uid": "b396894e-56bf-4a01-837b-64157912ca00",
"creationTimestamp": "2024-10-30T18:30:54Z",
"annotations": {
"grafana.app/createdBy": "user:be2g71ke8yoe8b",
"grafana.app/originHash": "Grafana v9.2.0 (NA)",
"grafana.app/originName": "UI",
"grafana.app/originPath": "/dashboard/new"
},
"managedFields": [
{
"manager": "Mozilla",
"operation": "Update",
"apiVersion": "dashboard.grafana.app/v0alpha1",
"time": "2024-10-30T18:30:54Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:metadata": {
"f:annotations": {
".": {},
"f:grafana.app/originHash": {},
"f:grafana.app/originName": {},
"f:grafana.app/originPath": {}
},
"f:generateName": {}
},
"f:spec": {
"f:annotations": {
".": {},
"f:list": {}
},
"f:description": {},
"f:editable": {},
"f:fiscalYearStartMonth": {},
"f:graphTooltip": {},
"f:id": {},
"f:links": {},
"f:panels": {},
"f:preload": {},
"f:schemaVersion": {},
"f:tags": {},
"f:templating": {
".": {},
"f:list": {}
},
"f:timepicker": {},
"f:timezone": {},
"f:title": {},
"f:uid": {},
"f:version": {},
"f:weekStart": {}
}
}
}
]
},
"spec": {
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations \u0026 Alerts",
"type": "dashboard"
}
]
},
"description": "",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.2.0",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"refId": "A"
}
],
"title": "Panel Title",
"type": "timeseries"
}
],
"preload": false,
"schemaVersion": 40,
"tags": [],
"templating": {
"list": []
},
"timepicker": {},
"timezone": "browser",
"title": "K8Test",
"uid": "",
"version": 0,
"weekStart": ""
}
}