mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
[search] minimal dashboard indexing (#95631)
[search] minimal dashboard indexing
This commit is contained in:
Scott Lepper
GitHub
parent
b1157dbd7a
commit
7619d9cb11
@ -169,6 +169,16 @@ func getSpecObjectMappings() map[string][]SpecFieldMapping {
|
||||
Type: "string",
|
||||
},
|
||||
},
|
||||
"Dashboard": {
|
||||
{
|
||||
Field: "title",
|
||||
Type: "string",
|
||||
},
|
||||
{
|
||||
Field: "description",
|
||||
Type: "string",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return mappings
|
||||
|
||||
@ -3,6 +3,7 @@ package resource
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@ -11,60 +12,53 @@ import (
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/infra/tracing"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/exp/rand"
|
||||
)
|
||||
|
||||
const testTenant = "default"
|
||||
|
||||
var testContext = context.Background()
|
||||
|
||||
func TestIndexDashboard(t *testing.T) {
|
||||
data, err := os.ReadFile("./testdata/dashboard-resource.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
list := &ListResponse{Items: []*ResourceWrapper{{Value: data}}}
|
||||
index := newTestIndex(t)
|
||||
_, err = index.AddToBatches(testContext, list)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = index.IndexBatches(testContext, 1, []string{testTenant})
|
||||
require.NoError(t, err)
|
||||
assertCountEquals(t, index, 1)
|
||||
assertSearchCountEquals(t, index, 1)
|
||||
}
|
||||
|
||||
func TestIndexBatch(t *testing.T) {
|
||||
tracingCfg := tracing.NewEmptyTracingConfig()
|
||||
trace, err := tracing.ProvideService(tracingCfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
index := newTestIndex(t)
|
||||
|
||||
index := &Index{
|
||||
tracer: trace,
|
||||
shards: make(map[string]Shard),
|
||||
log: log.New("unifiedstorage.search.index"),
|
||||
opts: Opts{
|
||||
ListLimit: 5000,
|
||||
Workers: 10,
|
||||
BatchSize: 1000,
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
startAll := time.Now()
|
||||
|
||||
ns := namespaces()
|
||||
// simulate 10 List calls
|
||||
for i := 0; i < 10; i++ {
|
||||
list := &ListResponse{Items: loadTestItems(strconv.Itoa(i), ns)}
|
||||
start := time.Now()
|
||||
_, err = index.AddToBatches(ctx, list)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := index.AddToBatches(testContext, list)
|
||||
require.NoError(t, err)
|
||||
elapsed := time.Since(start)
|
||||
fmt.Println("Time elapsed:", elapsed)
|
||||
}
|
||||
|
||||
// index all batches for each shard/tenant
|
||||
err = index.IndexBatches(ctx, 1, ns)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := index.IndexBatches(testContext, 1, ns)
|
||||
require.NoError(t, err)
|
||||
|
||||
elapsed := time.Since(startAll)
|
||||
fmt.Println("Total Time elapsed:", elapsed)
|
||||
|
||||
assert.Equal(t, len(ns), len(index.shards))
|
||||
|
||||
total, err := index.Count()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
assert.Equal(t, uint64(100000), total)
|
||||
assertCountEquals(t, index, 100000)
|
||||
}
|
||||
|
||||
func loadTestItems(uid string, tenants []string) []*ResourceWrapper {
|
||||
@ -110,3 +104,32 @@ func namespaces() []string {
|
||||
}
|
||||
return ns
|
||||
}
|
||||
|
||||
func newTestIndex(t *testing.T) *Index {
|
||||
tracingCfg := tracing.NewEmptyTracingConfig()
|
||||
trace, err := tracing.ProvideService(tracingCfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
return &Index{
|
||||
tracer: trace,
|
||||
shards: make(map[string]Shard),
|
||||
log: log.New("unifiedstorage.search.index"),
|
||||
opts: Opts{
|
||||
ListLimit: 5000,
|
||||
Workers: 10,
|
||||
BatchSize: 1000,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func assertCountEquals(t *testing.T, index *Index, expected uint64) {
|
||||
total, err := index.Count()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expected, total)
|
||||
}
|
||||
|
||||
func assertSearchCountEquals(t *testing.T, index *Index, expected int) {
|
||||
results, err := index.Search(testContext, testTenant, "*", expected+1, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expected, len(results))
|
||||
}
|
||||
|
||||
192
pkg/storage/unified/resource/testdata/dashboard-resource.json
vendored
Normal file
192
pkg/storage/unified/resource/testdata/dashboard-resource.json
vendored
Normal file
@ -0,0 +1,192 @@
|
||||
{
|
||||
"kind": "Dashboard",
|
||||
"apiVersion": "dashboard.grafana.app/v0alpha1",
|
||||
"metadata": {
|
||||
"name": "adfbg6f",
|
||||
"namespace": "default",
|
||||
"uid": "b396894e-56bf-4a01-837b-64157912ca00",
|
||||
"creationTimestamp": "2024-10-30T18:30:54Z",
|
||||
"annotations": {
|
||||
"grafana.app/createdBy": "user:be2g71ke8yoe8b",
|
||||
"grafana.app/originHash": "Grafana v9.2.0 (NA)",
|
||||
"grafana.app/originName": "UI",
|
||||
"grafana.app/originPath": "/dashboard/new"
|
||||
},
|
||||
"managedFields": [
|
||||
{
|
||||
"manager": "Mozilla",
|
||||
"operation": "Update",
|
||||
"apiVersion": "dashboard.grafana.app/v0alpha1",
|
||||
"time": "2024-10-30T18:30:54Z",
|
||||
"fieldsType": "FieldsV1",
|
||||
"fieldsV1": {
|
||||
"f:metadata": {
|
||||
"f:annotations": {
|
||||
".": {},
|
||||
"f:grafana.app/originHash": {},
|
||||
"f:grafana.app/originName": {},
|
||||
"f:grafana.app/originPath": {}
|
||||
},
|
||||
"f:generateName": {}
|
||||
},
|
||||
"f:spec": {
|
||||
"f:annotations": {
|
||||
".": {},
|
||||
"f:list": {}
|
||||
},
|
||||
"f:description": {},
|
||||
"f:editable": {},
|
||||
"f:fiscalYearStartMonth": {},
|
||||
"f:graphTooltip": {},
|
||||
"f:id": {},
|
||||
"f:links": {},
|
||||
"f:panels": {},
|
||||
"f:preload": {},
|
||||
"f:schemaVersion": {},
|
||||
"f:tags": {},
|
||||
"f:templating": {
|
||||
".": {},
|
||||
"f:list": {}
|
||||
},
|
||||
"f:timepicker": {},
|
||||
"f:timezone": {},
|
||||
"f:title": {},
|
||||
"f:uid": {},
|
||||
"f:version": {},
|
||||
"f:weekStart": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"spec": {
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations \u0026 Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "datasource",
|
||||
"uid": "grafana"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "9.2.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "datasource",
|
||||
"uid": "grafana"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Panel Title",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"preload": false,
|
||||
"schemaVersion": 40,
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "K8Test",
|
||||
"uid": "",
|
||||
"version": 0,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user