IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Search poc: Initial index paginates list response (#94582)

Browse Source
This commit is contained in:
owensmallwood 2024-10-11 02:13:10 -06:00 committed by GitHub
parent 4eab10eaa1
commit 3d59d3b40f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 57 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/services/unifiedSearch/service.go
View File

@ -3,7 +3,6 @@ package unifiedSearch
import (
"context"
"errors"
"fmt"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
@ -12,6 +11,7 @@ import (
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/org"
@ -152,14 +152,16 @@ func (s *StandardSearchService) DoQuery(ctx context.Context, user *backend.User,
}
func (s *StandardSearchService) doQuery(ctx context.Context, signedInUser *user.SignedInUser, orgID int64, q Query) *backend.DataResponse {
response := s.doSearchQuery(ctx, q, s.cfg.AppSubURL)
response := s.doSearchQuery(ctx, q, s.cfg.AppSubURL, orgID)
return response
}
func (s *StandardSearchService) doSearchQuery(ctx context.Context, qry Query, _ string) *backend.DataResponse {
func (s *StandardSearchService) doSearchQuery(ctx context.Context, qry Query, _ string, orgID int64) *backend.DataResponse {
response := &backend.DataResponse{}
tenantId := fmt.Sprintf("stacks-%s", s.cfg.StackID)
// will use stack id for cloud and org id for on-prem
tenantId := request.GetNamespaceMapper(s.cfg)(orgID)
req := &resource.SearchRequest{Tenant: tenantId, Query: qry.Query, Limit: int64(qry.Limit), Offset: int64(qry.From)}
res, err := s.resourceClient.Search(ctx, req)
if err != nil {

83
pkg/storage/unified/resource/index.go
View File

@ -39,46 +39,65 @@ func NewIndex(s *server, opts Opts) *Index {
return idx
}
func (i *Index) Init(ctx context.Context) error {
resourceTypes := fetchResourceTypes()
for _, rt := range resourceTypes {
r := &ListRequest{Options: rt}
list, err := i.s.List(ctx, r)
func (i *Index) IndexBatch(list *ListResponse, kind string) error {
for _, obj := range list.Items {
res, err := getResource(obj.Value)
if err != nil {
return err
}
i.log.Info("initial indexing resources", "count", len(list.Items))
for _, obj := range list.Items {
res, err := getResource(obj.Value)
if err != nil {
return err
}
shard, err := i.getShard(tenant(res))
if err != nil {
return err
}
i.log.Info("indexing resource for tenant", "res", res, "tenant", tenant(res))
var jsonDoc interface{}
err = json.Unmarshal(obj.Value, &jsonDoc)
if err != nil {
return err
}
err = shard.batch.Index(res.Metadata.Uid, jsonDoc)
if err != nil {
return err
}
shard, err := i.getShard(tenant(res))
if err != nil {
return err
}
i.log.Debug("initial indexing resources batch", "count", len(list.Items), "kind", kind, "tenant", tenant(res))
for _, shard := range i.shards {
err := shard.index.Batch(shard.batch)
var jsonDoc interface{}
err = json.Unmarshal(obj.Value, &jsonDoc)
if err != nil {
return err
}
err = shard.batch.Index(res.Metadata.Uid, jsonDoc)
if err != nil {
return err
}
}
for _, shard := range i.shards {
err := shard.index.Batch(shard.batch)
if err != nil {
return err
}
shard.batch.Reset()
}
return nil
}
func (i *Index) Init(ctx context.Context) error {
resourceTypes := fetchResourceTypes()
for _, rt := range resourceTypes {
i.log.Info("indexing resource", "kind", rt.Key.Resource)
r := &ListRequest{Options: rt, Limit: 100}
// Paginate through the list of resources and index each page
for {
list, err := i.s.List(ctx, r)
if err != nil {
return err
}
shard.batch.Reset()
// Index current page
err = i.IndexBatch(list, rt.Key.Resource)
if err != nil {
return err
}
if list.NextPageToken == "" {
break
}
r.NextPageToken = list.NextPageToken
}
}
@ -91,7 +110,7 @@ func (i *Index) Index(ctx context.Context, data *Data) error {
return err
}
tenant := tenant(res)
i.log.Info("indexing resource for tenant", "res", res, "tenant", tenant)
i.log.Debug("indexing resource for tenant", "res", res, "tenant", tenant)
shard, err := i.getShard(tenant)
if err != nil {
return err