SearchV2: instant local updates, folder events (#50001)

This commit is contained in:
Alexander Emelin
2022-06-03 13:11:32 -07:00
committed by GitHub
parent 49d93fb67e
commit d2868a1ce7
9 changed files with 728 additions and 35 deletions
+64 -4
View File
@@ -105,7 +105,10 @@ func initIndex(dashboards []dashboard, logger log.Logger, extendDoc ExtendDashbo
}
// Index each panel in dashboard.
location += "/" + dash.uid
if location != "" {
location += "/"
}
location += dash.uid
docs := getDashboardPanelDocs(dash, location)
for _, panelDoc := range docs {
batch.Insert(panelDoc)
@@ -194,7 +197,6 @@ func getDashboardPanelDocs(dash dashboard, location string) []*bluge.Document {
purl := fmt.Sprintf("%s?viewPanel=%d", url, panel.ID)
doc := newSearchDocument(uid, panel.Title, panel.Description, purl).
AddField(bluge.NewKeywordField(documentFieldDSUID, dash.uid).StoreValue()).
AddField(bluge.NewKeywordField(documentFieldPanelType, panel.Type).Aggregatable().StoreValue()).
AddField(bluge.NewKeywordField(documentFieldLocation, location).Aggregatable().StoreValue()).
AddField(bluge.NewKeywordField(documentFieldKind, string(entityKindPanel)).Aggregatable().StoreValue()) // likely want independent index for this
@@ -263,10 +265,10 @@ func newSearchDocument(uid string, name string, descr string, url string) *bluge
return doc
}
func getDashboardPanelIDs(reader *bluge.Reader, dashboardUID string) ([]string, error) {
func getDashboardPanelIDs(reader *bluge.Reader, panelLocation string) ([]string, error) {
var panelIDs []string
fullQuery := bluge.NewBooleanQuery()
fullQuery.AddMust(bluge.NewTermQuery(dashboardUID).SetField(documentFieldDSUID))
fullQuery.AddMust(bluge.NewTermQuery(panelLocation).SetField(documentFieldLocation))
fullQuery.AddMust(bluge.NewTermQuery(string(entityKindPanel)).SetField(documentFieldKind))
req := bluge.NewAllMatches(fullQuery)
documentMatchIterator, err := reader.Search(context.Background(), req)
@@ -291,6 +293,64 @@ func getDashboardPanelIDs(reader *bluge.Reader, dashboardUID string) ([]string,
return panelIDs, err
}
func getDocsIDsByLocationPrefix(reader *bluge.Reader, prefix string) ([]string, error) {
var ids []string
fullQuery := bluge.NewBooleanQuery()
fullQuery.AddMust(bluge.NewPrefixQuery(prefix).SetField(documentFieldLocation))
req := bluge.NewAllMatches(fullQuery)
documentMatchIterator, err := reader.Search(context.Background(), req)
if err != nil {
return nil, err
}
match, err := documentMatchIterator.Next()
for err == nil && match != nil {
// load the identifier for this match
err = match.VisitStoredFields(func(field string, value []byte) bool {
if field == documentFieldUID {
ids = append(ids, string(value))
}
return true
})
if err != nil {
return nil, err
}
// load the next document match
match, err = documentMatchIterator.Next()
}
return ids, err
}
func getDashboardLocation(reader *bluge.Reader, dashboardUID string) (string, bool, error) {
var dashboardLocation string
var found bool
fullQuery := bluge.NewBooleanQuery()
fullQuery.AddMust(bluge.NewTermQuery(dashboardUID).SetField(documentFieldUID))
fullQuery.AddMust(bluge.NewTermQuery(string(entityKindDashboard)).SetField(documentFieldKind))
req := bluge.NewAllMatches(fullQuery)
documentMatchIterator, err := reader.Search(context.Background(), req)
if err != nil {
return "", false, err
}
match, err := documentMatchIterator.Next()
for err == nil && match != nil {
// load the identifier for this match
err = match.VisitStoredFields(func(field string, value []byte) bool {
if field == documentFieldLocation {
dashboardLocation = string(value)
found = true
return false
}
return true
})
if err != nil {
return "", false, err
}
// load the next document match
match, err = documentMatchIterator.Next()
}
return dashboardLocation, found, err
}
//nolint: gocyclo
func doSearchQuery(
ctx context.Context,
+58 -16
View File
@@ -30,6 +30,7 @@ type dashboardLoader interface {
}
type eventStore interface {
OnEvent(handler store.EventHandler)
GetLastEvent(ctx context.Context) (*store.EntityEvent, error)
GetAllEventsAfter(ctx context.Context, id int64) ([]*store.EntityEvent, error)
}
@@ -95,6 +96,8 @@ func (i *dashboardIndex) run(ctx context.Context) error {
return fmt.Errorf("can't build initial dashboard search index: %w", err)
}
i.eventStore.OnEvent(i.applyEventOnIndex)
for {
select {
case <-partialUpdateTicker.C:
@@ -293,7 +296,6 @@ func (i *dashboardIndex) applyIndexUpdates(ctx context.Context, lastEventID int6
}
started := time.Now()
for _, e := range events {
i.logger.Debug("processing event", "event", e)
err := i.applyEventOnIndex(ctx, e)
if err != nil {
i.logger.Error("can't apply event", "error", err)
@@ -306,31 +308,30 @@ func (i *dashboardIndex) applyIndexUpdates(ctx context.Context, lastEventID int6
}
func (i *dashboardIndex) applyEventOnIndex(ctx context.Context, e *store.EntityEvent) error {
i.logger.Debug("processing event", "event", e)
if !strings.HasPrefix(e.EntityId, "database/") {
i.logger.Warn("unknown storage", "entityId", e.EntityId)
return nil
}
parts := strings.Split(strings.TrimPrefix(e.EntityId, "database/"), "/")
// database/org/entityType/path*
parts := strings.SplitN(strings.TrimPrefix(e.EntityId, "database/"), "/", 3)
if len(parts) != 3 {
i.logger.Error("can't parse entityId", "entityId", e.EntityId)
return nil
}
orgIDStr := parts[0]
kind := parts[1]
dashboardUID := parts[2]
if kind != "dashboard" {
i.logger.Error("unknown kind in entityId", "entityId", e.EntityId)
return nil
}
orgID, err := strconv.Atoi(orgIDStr)
orgID, err := strconv.ParseInt(orgIDStr, 10, 64)
if err != nil {
i.logger.Error("can't extract org ID", "entityId", e.EntityId)
return nil
}
return i.applyDashboardEvent(ctx, int64(orgID), dashboardUID, e.EventType)
kind := store.EntityType(parts[1])
uid := parts[2]
return i.applyEvent(ctx, orgID, kind, uid, e.EventType)
}
func (i *dashboardIndex) applyDashboardEvent(ctx context.Context, orgID int64, dashboardUID string, _ store.EntityEventType) error {
func (i *dashboardIndex) applyEvent(ctx context.Context, orgID int64, kind store.EntityType, uid string, _ store.EntityEventType) error {
i.mu.Lock()
_, ok := i.perOrgWriter[orgID]
if !ok {
@@ -340,7 +341,8 @@ func (i *dashboardIndex) applyDashboardEvent(ctx context.Context, orgID int64, d
}
i.mu.Unlock()
dbDashboards, err := i.loader.LoadDashboards(ctx, orgID, dashboardUID)
// Both dashboard and folder share same DB table.
dbDashboards, err := i.loader.LoadDashboards(ctx, orgID, uid)
if err != nil {
return err
}
@@ -364,7 +366,14 @@ func (i *dashboardIndex) applyDashboardEvent(ctx context.Context, orgID int64, d
// In the future we can rely on operation types to reduce work here.
if len(dbDashboards) == 0 {
newReader, err = i.removeDashboard(ctx, writer, reader, dashboardUID)
switch kind {
case store.EntityTypeDashboard:
newReader, err = i.removeDashboard(ctx, writer, reader, uid)
case store.EntityTypeFolder:
newReader, err = i.removeFolder(ctx, writer, reader, uid)
default:
return nil
}
} else {
newReader, err = i.updateDashboard(ctx, orgID, writer, reader, dbDashboards[0])
}
@@ -377,8 +386,21 @@ func (i *dashboardIndex) applyDashboardEvent(ctx context.Context, orgID int64, d
}
func (i *dashboardIndex) removeDashboard(_ context.Context, writer *bluge.Writer, reader *bluge.Reader, dashboardUID string) (*bluge.Reader, error) {
dashboardLocation, ok, err := getDashboardLocation(reader, dashboardUID)
if err != nil {
return nil, err
}
if !ok {
// No dashboard, nothing to remove.
return reader, nil
}
// Find all panel docs to remove with dashboard.
panelIDs, err := getDashboardPanelIDs(reader, dashboardUID)
panelLocation := dashboardUID
if dashboardLocation != "" {
panelLocation = dashboardLocation + "/" + dashboardUID
}
panelIDs, err := getDocsIDsByLocationPrefix(reader, panelLocation)
if err != nil {
return nil, err
}
@@ -397,6 +419,23 @@ func (i *dashboardIndex) removeDashboard(_ context.Context, writer *bluge.Writer
return writer.Reader()
}
func (i *dashboardIndex) removeFolder(_ context.Context, writer *bluge.Writer, reader *bluge.Reader, folderUID string) (*bluge.Reader, error) {
ids, err := getDocsIDsByLocationPrefix(reader, folderUID)
if err != nil {
return nil, err
}
batch := bluge.NewBatch()
batch.Delete(bluge.NewDocument(folderUID).ID())
for _, id := range ids {
batch.Delete(bluge.NewDocument(id).ID())
}
err = writer.Batch(batch)
if err != nil {
return nil, err
}
return writer.Reader()
}
func stringInSlice(str string, slice []string) bool {
for _, s := range slice {
if s == str {
@@ -437,14 +476,17 @@ func (i *dashboardIndex) updateDashboard(ctx context.Context, orgID int64, write
var actualPanelIDs []string
location += "/" + dash.uid
if location != "" {
location += "/"
}
location += dash.uid
panelDocs := getDashboardPanelDocs(dash, location)
for _, panelDoc := range panelDocs {
actualPanelIDs = append(actualPanelIDs, string(panelDoc.ID().Term()))
batch.Update(panelDoc.ID(), panelDoc)
}
indexedPanelIDs, err := getDashboardPanelIDs(reader, dash.uid)
indexedPanelIDs, err := getDashboardPanelIDs(reader, location)
if err != nil {
return nil, err
}
+142 -1
View File
@@ -2,15 +2,17 @@ package searchV2
import (
"context"
"fmt"
"path/filepath"
"testing"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/searchV2/extract"
"github.com/grafana/grafana/pkg/services/store"
"github.com/blugelabs/bluge"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/experimental"
"github.com/stretchr/testify/require"
)
@@ -370,3 +372,142 @@ func TestDashboardIndex_MultipleTokensScattered(t *testing.T) {
)
})
}
var dashboardsWithFolders = []dashboard{
{
id: 1,
uid: "1",
isFolder: true,
info: &extract.DashboardInfo{
Title: "My folder",
},
},
{
id: 2,
uid: "2",
folderID: 1,
info: &extract.DashboardInfo{
Title: "Dashboard in folder 1",
Panels: []extract.PanelInfo{
{
ID: 1,
Title: "Panel 1",
},
{
ID: 2,
Title: "Panel 2",
},
},
},
},
{
id: 3,
uid: "3",
folderID: 1,
info: &extract.DashboardInfo{
Title: "Dashboard in folder 2",
Panels: []extract.PanelInfo{
{
ID: 3,
Title: "Panel 3",
},
},
},
},
{
id: 4,
uid: "4",
info: &extract.DashboardInfo{
Title: "One more dash",
Panels: []extract.PanelInfo{
{
ID: 3,
Title: "Panel 4",
},
},
},
},
}
func TestDashboardIndex_Folders(t *testing.T) {
t.Run("folders-indexed", func(t *testing.T) {
_, reader, _ := initTestIndexFromDashes(t, dashboardsWithFolders)
checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
DashboardQuery{Query: "My folder", Kind: []string{string(entityKindFolder)}},
)
})
t.Run("folders-dashboard-has-folder", func(t *testing.T) {
_, reader, _ := initTestIndexFromDashes(t, dashboardsWithFolders)
// TODO: golden file compare does not work here.
resp := doSearchQuery(context.Background(), testLogger, reader, testAllowAllFilter,
DashboardQuery{Query: "Dashboard in folder", Kind: []string{string(entityKindDashboard)}},
&NoopQueryExtender{}, "")
custom, ok := resp.Frames[0].Meta.Custom.(*customMeta)
require.Equal(t, uint64(2), custom.Count)
require.True(t, ok, fmt.Sprintf("actual type: %T", resp.Frames[0].Meta.Custom))
require.Equal(t, "/dashboards/f/1/", custom.Locations["1"].URL)
})
t.Run("folders-dashboard-removed-on-folder-removed", func(t *testing.T) {
index, reader, writer := initTestIndexFromDashes(t, dashboardsWithFolders)
newReader, err := index.removeFolder(context.Background(), writer, reader, "1")
require.NoError(t, err)
// In response we expect one dashboard which does not belong to removed folder.
checkSearchResponse(t, filepath.Base(t.Name()), newReader, testAllowAllFilter,
DashboardQuery{Query: "dash", Kind: []string{string(entityKindDashboard)}},
)
})
t.Run("folders-panels-removed-on-folder-removed", func(t *testing.T) {
index, reader, writer := initTestIndexFromDashes(t, dashboardsWithFolders)
newReader, err := index.removeFolder(context.Background(), writer, reader, "1")
require.NoError(t, err)
resp := doSearchQuery(context.Background(), testLogger, newReader, testAllowAllFilter,
DashboardQuery{Query: "Panel", Kind: []string{string(entityKindPanel)}},
&NoopQueryExtender{}, "")
custom, ok := resp.Frames[0].Meta.Custom.(*customMeta)
require.True(t, ok)
require.Equal(t, uint64(1), custom.Count) // 1 panel which does not belong to dashboards in removed folder.
})
}
var dashboardsWithPanels = []dashboard{
{
id: 1,
uid: "1",
info: &extract.DashboardInfo{
Title: "My Dash",
Panels: []extract.PanelInfo{
{
ID: 1,
Title: "Panel 1",
},
{
ID: 2,
Title: "Panel 2",
},
},
},
},
}
func TestDashboardIndex_Panels(t *testing.T) {
t.Run("panels-indexed", func(t *testing.T) {
_, reader, _ := initTestIndexFromDashes(t, dashboardsWithPanels)
// TODO: golden file compare does not work here.
resp := doSearchQuery(
context.Background(), testLogger, reader, testAllowAllFilter,
DashboardQuery{Query: "Panel", Kind: []string{string(entityKindPanel)}},
&NoopQueryExtender{}, "")
custom, ok := resp.Frames[0].Meta.Custom.(*customMeta)
require.True(t, ok, fmt.Sprintf("actual type: %T", resp.Frames[0].Meta.Custom))
require.Equal(t, uint64(2), custom.Count)
require.Equal(t, "/d/1/", custom.Locations["1"].URL)
})
t.Run("panels-panel-removed-on-dashboard-removed", func(t *testing.T) {
index, reader, writer := initTestIndexFromDashes(t, dashboardsWithPanels)
newReader, err := index.removeDashboard(context.Background(), writer, reader, "1")
require.NoError(t, err)
checkSearchResponse(t, filepath.Base(t.Name()), newReader, testAllowAllFilter,
DashboardQuery{Query: "Panel", Kind: []string{string(entityKindPanel)}},
)
})
}
@@ -0,0 +1,131 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0] {
// "type": "search-results",
// "custom": {
// "count": 1
// }
// }
// Name: Query results
// Dimensions: 8 Fields by 1 Rows
// +----------------+----------------+----------------+------------------+----------------+--------------------------+--------------------------+----------------+
// | Name: kind | Name: uid | Name: name | Name: panel_type | Name: url | Name: tags | Name: ds_uid | Name: location |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []string | Type: []string | Type: []string | Type: []string | Type: []string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []string |
// +----------------+----------------+----------------+------------------+----------------+--------------------------+--------------------------+----------------+
// | dashboard | 4 | One more dash | | /pfix/d/4/ | null | null | |
// +----------------+----------------+----------------+------------------+----------------+--------------------------+--------------------------+----------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"frames": [
{
"schema": {
"name": "Query results",
"meta": {
"type": "search-results",
"custom": {
"count": 1
}
},
"fields": [
{
"name": "kind",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "uid",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "name",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "panel_type",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "url",
"type": "string",
"typeInfo": {
"frame": "string"
},
"config": {
"links": [
{
"title": "link",
"url": "${__value.text}"
}
]
}
},
{
"name": "tags",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
}
},
{
"name": "ds_uid",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
}
},
{
"name": "location",
"type": "string",
"typeInfo": {
"frame": "string"
}
}
]
},
"data": {
"values": [
[
"dashboard"
],
[
"4"
],
[
"One more dash"
],
[
""
],
[
"/pfix/d/4/"
],
[
null
],
[
null
],
[
""
]
]
}
}
]
}
+131
View File
@@ -0,0 +1,131 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0] {
// "type": "search-results",
// "custom": {
// "count": 1
// }
// }
// Name: Query results
// Dimensions: 8 Fields by 1 Rows
// +----------------+----------------+----------------+------------------+-----------------------+--------------------------+--------------------------+----------------+
// | Name: kind | Name: uid | Name: name | Name: panel_type | Name: url | Name: tags | Name: ds_uid | Name: location |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []string | Type: []string | Type: []string | Type: []string | Type: []string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []string |
// +----------------+----------------+----------------+------------------+-----------------------+--------------------------+--------------------------+----------------+
// | folder | 1 | My folder | | /pfix/dashboards/f/1/ | null | null | |
// +----------------+----------------+----------------+------------------+-----------------------+--------------------------+--------------------------+----------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"frames": [
{
"schema": {
"name": "Query results",
"meta": {
"type": "search-results",
"custom": {
"count": 1
}
},
"fields": [
{
"name": "kind",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "uid",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "name",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "panel_type",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "url",
"type": "string",
"typeInfo": {
"frame": "string"
},
"config": {
"links": [
{
"title": "link",
"url": "${__value.text}"
}
]
}
},
{
"name": "tags",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
}
},
{
"name": "ds_uid",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
}
},
{
"name": "location",
"type": "string",
"typeInfo": {
"frame": "string"
}
}
]
},
"data": {
"values": [
[
"folder"
],
[
"1"
],
[
"My folder"
],
[
""
],
[
"/pfix/dashboards/f/1/"
],
[
null
],
[
null
],
[
""
]
]
}
}
]
}
@@ -0,0 +1,114 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0] {
// "type": "search-results",
// "custom": {
// "count": 0
// }
// }
// Name: Query results
// Dimensions: 8 Fields by 0 Rows
// +----------------+----------------+----------------+------------------+----------------+--------------------------+--------------------------+----------------+
// | Name: kind | Name: uid | Name: name | Name: panel_type | Name: url | Name: tags | Name: ds_uid | Name: location |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []string | Type: []string | Type: []string | Type: []string | Type: []string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []string |
// +----------------+----------------+----------------+------------------+----------------+--------------------------+--------------------------+----------------+
// +----------------+----------------+----------------+------------------+----------------+--------------------------+--------------------------+----------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"frames": [
{
"schema": {
"name": "Query results",
"meta": {
"type": "search-results",
"custom": {
"count": 0
}
},
"fields": [
{
"name": "kind",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "uid",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "name",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "panel_type",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "url",
"type": "string",
"typeInfo": {
"frame": "string"
},
"config": {
"links": [
{
"title": "link",
"url": "${__value.text}"
}
]
}
},
{
"name": "tags",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
}
},
{
"name": "ds_uid",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
}
},
{
"name": "location",
"type": "string",
"typeInfo": {
"frame": "string"
}
}
]
},
"data": {
"values": [
[],
[],
[],
[],
[],
[],
[],
[]
]
}
}
]
}