[search] minimal dashboard indexing (#95631)

[search] minimal dashboard indexing
This commit is contained in:
Scott Lepper 2024-10-31 07:46:00 -04:00 committed by GitHub
parent b1157dbd7a
commit 7619d9cb11
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 258 additions and 33 deletions

View File

@ -169,6 +169,16 @@ func getSpecObjectMappings() map[string][]SpecFieldMapping {
Type: "string",
},
},
"Dashboard": {
{
Field: "title",
Type: "string",
},
{
Field: "description",
Type: "string",
},
},
}
return mappings

View File

@ -3,6 +3,7 @@ package resource
import (
"context"
"fmt"
"os"
"strconv"
"strings"
"testing"
@ -11,60 +12,53 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/exp/rand"
)
const testTenant = "default"
var testContext = context.Background()
func TestIndexDashboard(t *testing.T) {
data, err := os.ReadFile("./testdata/dashboard-resource.json")
require.NoError(t, err)
list := &ListResponse{Items: []*ResourceWrapper{{Value: data}}}
index := newTestIndex(t)
_, err = index.AddToBatches(testContext, list)
require.NoError(t, err)
err = index.IndexBatches(testContext, 1, []string{testTenant})
require.NoError(t, err)
assertCountEquals(t, index, 1)
assertSearchCountEquals(t, index, 1)
}
func TestIndexBatch(t *testing.T) {
tracingCfg := tracing.NewEmptyTracingConfig()
trace, err := tracing.ProvideService(tracingCfg)
if err != nil {
t.Fatal(err)
}
index := newTestIndex(t)
index := &Index{
tracer: trace,
shards: make(map[string]Shard),
log: log.New("unifiedstorage.search.index"),
opts: Opts{
ListLimit: 5000,
Workers: 10,
BatchSize: 1000,
},
}
ctx := context.Background()
startAll := time.Now()
ns := namespaces()
// simulate 10 List calls
for i := 0; i < 10; i++ {
list := &ListResponse{Items: loadTestItems(strconv.Itoa(i), ns)}
start := time.Now()
_, err = index.AddToBatches(ctx, list)
if err != nil {
t.Fatal(err)
}
_, err := index.AddToBatches(testContext, list)
require.NoError(t, err)
elapsed := time.Since(start)
fmt.Println("Time elapsed:", elapsed)
}
// index all batches for each shard/tenant
err = index.IndexBatches(ctx, 1, ns)
if err != nil {
t.Fatal(err)
}
err := index.IndexBatches(testContext, 1, ns)
require.NoError(t, err)
elapsed := time.Since(startAll)
fmt.Println("Total Time elapsed:", elapsed)
assert.Equal(t, len(ns), len(index.shards))
total, err := index.Count()
if err != nil {
t.Fatal(err)
}
assert.Equal(t, uint64(100000), total)
assertCountEquals(t, index, 100000)
}
func loadTestItems(uid string, tenants []string) []*ResourceWrapper {
@ -110,3 +104,32 @@ func namespaces() []string {
}
return ns
}
func newTestIndex(t *testing.T) *Index {
tracingCfg := tracing.NewEmptyTracingConfig()
trace, err := tracing.ProvideService(tracingCfg)
require.NoError(t, err)
return &Index{
tracer: trace,
shards: make(map[string]Shard),
log: log.New("unifiedstorage.search.index"),
opts: Opts{
ListLimit: 5000,
Workers: 10,
BatchSize: 1000,
},
}
}
func assertCountEquals(t *testing.T, index *Index, expected uint64) {
total, err := index.Count()
require.NoError(t, err)
assert.Equal(t, expected, total)
}
func assertSearchCountEquals(t *testing.T, index *Index, expected int) {
results, err := index.Search(testContext, testTenant, "*", expected+1, 0)
require.NoError(t, err)
assert.Equal(t, expected, len(results))
}

View File

@ -0,0 +1,192 @@
{
"kind": "Dashboard",
"apiVersion": "dashboard.grafana.app/v0alpha1",
"metadata": {
"name": "adfbg6f",
"namespace": "default",
"uid": "b396894e-56bf-4a01-837b-64157912ca00",
"creationTimestamp": "2024-10-30T18:30:54Z",
"annotations": {
"grafana.app/createdBy": "user:be2g71ke8yoe8b",
"grafana.app/originHash": "Grafana v9.2.0 (NA)",
"grafana.app/originName": "UI",
"grafana.app/originPath": "/dashboard/new"
},
"managedFields": [
{
"manager": "Mozilla",
"operation": "Update",
"apiVersion": "dashboard.grafana.app/v0alpha1",
"time": "2024-10-30T18:30:54Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:metadata": {
"f:annotations": {
".": {},
"f:grafana.app/originHash": {},
"f:grafana.app/originName": {},
"f:grafana.app/originPath": {}
},
"f:generateName": {}
},
"f:spec": {
"f:annotations": {
".": {},
"f:list": {}
},
"f:description": {},
"f:editable": {},
"f:fiscalYearStartMonth": {},
"f:graphTooltip": {},
"f:id": {},
"f:links": {},
"f:panels": {},
"f:preload": {},
"f:schemaVersion": {},
"f:tags": {},
"f:templating": {
".": {},
"f:list": {}
},
"f:timepicker": {},
"f:timezone": {},
"f:title": {},
"f:uid": {},
"f:version": {},
"f:weekStart": {}
}
}
}
]
},
"spec": {
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations \u0026 Alerts",
"type": "dashboard"
}
]
},
"description": "",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.2.0",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"refId": "A"
}
],
"title": "Panel Title",
"type": "timeseries"
}
],
"preload": false,
"schemaVersion": 40,
"tags": [],
"templating": {
"list": []
},
"timepicker": {},
"timezone": "browser",
"title": "K8Test",
"uid": "",
"version": 0,
"weekStart": ""
}
}