IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Search: Replace Origin calls with Repository (#98754)

Browse Source
This commit is contained in:
Ryan McKinley 2025-01-10 21:27:10 +03:00 committed by GitHub
parent 3eace5f7c8
commit ed39259461
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 1211 additions and 652 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pkg/registry/apis/dashboard/legacy/client.go
View File

@ -57,9 +57,12 @@ func (d *directResourceClient) List(ctx context.Context, in *resource.ListReques
return d.server.List(ctx, in)
}
// Origin implements ResourceClient.
func (d *directResourceClient) Origin(ctx context.Context, in *resource.OriginRequest, opts ...grpc.CallOption) (*resource.OriginResponse, error) {
return d.server.Origin(ctx, in)
func (d *directResourceClient) ListRepositoryObjects(ctx context.Context, in *resource.ListRepositoryObjectsRequest, opts ...grpc.CallOption) (*resource.ListRepositoryObjectsResponse, error) {
return d.server.ListRepositoryObjects(ctx, in)
}
func (d *directResourceClient) CountRepositoryObjects(ctx context.Context, in *resource.CountRepositoryObjectsRequest, opts ...grpc.CallOption) (*resource.CountRepositoryObjectsResponse, error) {
return d.server.CountRepositoryObjects(ctx, in)
}
// PutBlob implements ResourceClient.

9
pkg/registry/apis/dashboard/legacy/storage.go
View File

@ -330,9 +330,12 @@ func (a *dashboardSqlAccess) History(ctx context.Context, req *resource.HistoryR
return list, err
}
// Used for efficient provisioning
func (a *dashboardSqlAccess) Origin(context.Context, *resource.OriginRequest) (*resource.OriginResponse, error) {
return nil, fmt.Errorf("not yet (origin)")
func (a *dashboardSqlAccess) ListRepositoryObjects(ctx context.Context, req *resource.ListRepositoryObjectsRequest) (*resource.ListRepositoryObjectsResponse, error) {
return nil, fmt.Errorf("not implemented")
}
func (a *dashboardSqlAccess) CountRepositoryObjects(context.Context, *resource.CountRepositoryObjectsRequest) (*resource.CountRepositoryObjectsResponse, error) {
return nil, fmt.Errorf("not implemented")
}
// GetStats implements ResourceServer.

26
pkg/storage/unified/resource/client.go
View File

@ -8,11 +8,12 @@ import (
"github.com/fullstorydev/grpchan"
"github.com/fullstorydev/grpchan/inprocgrpc"
authnlib "github.com/grafana/authlib/authn"
"github.com/grafana/authlib/claims"
grpcAuth "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth"
"google.golang.org/grpc"
authnlib "github.com/grafana/authlib/authn"
"github.com/grafana/authlib/claims"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/authn/grpcutils"
grpcUtils "github.com/grafana/grafana/pkg/storage/unified/resource/grpc"
@ -21,6 +22,7 @@ import (
type ResourceClient interface {
ResourceStoreClient
ResourceIndexClient
RepositoryIndexClient
BlobStoreClient
DiagnosticsClient
}
@ -29,6 +31,7 @@ type ResourceClient interface {
type resourceClient struct {
ResourceStoreClient
ResourceIndexClient
RepositoryIndexClient
BlobStoreClient
DiagnosticsClient
}
@ -36,10 +39,11 @@ type resourceClient struct {
func NewLegacyResourceClient(channel *grpc.ClientConn) ResourceClient {
cc := grpchan.InterceptClientConn(channel, grpcUtils.UnaryClientInterceptor, grpcUtils.StreamClientInterceptor)
return &resourceClient{
ResourceStoreClient: NewResourceStoreClient(cc),
ResourceIndexClient: NewResourceIndexClient(cc),
BlobStoreClient: NewBlobStoreClient(cc),
DiagnosticsClient: NewDiagnosticsClient(cc),
ResourceStoreClient: NewResourceStoreClient(cc),
ResourceIndexClient: NewResourceIndexClient(cc),
RepositoryIndexClient: NewRepositoryIndexClient(cc),
BlobStoreClient: NewBlobStoreClient(cc),
DiagnosticsClient: NewDiagnosticsClient(cc),
}
}
@ -51,6 +55,7 @@ func NewLocalResourceClient(server ResourceServer) ResourceClient {
for _, desc := range []*grpc.ServiceDesc{
&ResourceStore_ServiceDesc,
&ResourceIndex_ServiceDesc,
&RepositoryIndex_ServiceDesc,
&BlobStore_ServiceDesc,
&Diagnostics_ServiceDesc,
} {
@ -72,10 +77,11 @@ func NewLocalResourceClient(server ResourceServer) ResourceClient {
cc := grpchan.InterceptClientConn(channel, clientInt.UnaryClientInterceptor, clientInt.StreamClientInterceptor)
return &resourceClient{
ResourceStoreClient: NewResourceStoreClient(cc),
ResourceIndexClient: NewResourceIndexClient(cc),
BlobStoreClient: NewBlobStoreClient(cc),
DiagnosticsClient: NewDiagnosticsClient(cc),
ResourceStoreClient: NewResourceStoreClient(cc),
ResourceIndexClient: NewResourceIndexClient(cc),
RepositoryIndexClient: NewRepositoryIndexClient(cc),
BlobStoreClient: NewBlobStoreClient(cc),
DiagnosticsClient: NewDiagnosticsClient(cc),
}
}

9
pkg/storage/unified/resource/document.go
View File

@ -102,7 +102,7 @@ type IndexableDocument struct {
References ResourceReferences `json:"reference,omitempty"`
// When the resource is managed by an upstream repository
RepoInfo *utils.ResourceRepositoryInfo `json:"repository,omitempty"`
RepoInfo *utils.ResourceRepositoryInfo `json:"repo,omitempty"`
}
func (m *IndexableDocument) Type() string {
@ -261,8 +261,11 @@ const SEARCH_FIELD_CREATED = "created"
const SEARCH_FIELD_CREATED_BY = "createdBy"
const SEARCH_FIELD_UPDATED = "updated"
const SEARCH_FIELD_UPDATED_BY = "updatedBy"
const SEARCH_FIELD_REPOSITORY = "repository"
const SEARCH_FIELD_REPOSITORY_HASH = "repository_hash"
const SEARCH_FIELD_REPOSITORY_NAME = "repo.name"
const SEARCH_FIELD_REPOSITORY_PATH = "repo.path"
const SEARCH_FIELD_REPOSITORY_HASH = "repo.hash"
const SEARCH_FIELD_REPOSITORY_TIME = "repo.time"
const SEARCH_FIELD_SCORE = "_score" // the match score
const SEARCH_FIELD_EXPLAIN = "_explain" // score explanation as JSON object

2
pkg/storage/unified/resource/document_test.go
View File

@ -40,7 +40,7 @@ func TestStandardDocumentBuilder(t *testing.T) {
"createdBy": "user:ABC",
"updatedBy": "user:XYZ",
"name": "test1",
"repository": {
"repo": {
"name": "SQL",
"path": "15",
"hash": "xyz"

1246
pkg/storage/unified/resource/resource.pb.go
View File

File diff suppressed because it is too large Load Diff

101
pkg/storage/unified/resource/resource.proto
View File

@ -470,54 +470,76 @@ message HistoryResponse {
ErrorResult error = 4;
}
message OriginRequest {
// List items within a resource type & repository name
// Access control is managed above this request
message ListRepositoryObjectsRequest {
// Starting from the requested page (other query parameters must match!)
string next_page_token = 1;
// All results exist within this resource namespace+name
ResourceKey key = 2;
// The name of the repository
string name = 3;
// Maximum number of items to return
int64 limit = 2;
// Resource identifier
ResourceKey key = 3;
// List the deleted values (eg, show trash)
string origin = 4;
int64 limit = 4;
}
message ResourceOriginInfo {
// The resource
ResourceKey key = 1;
message ListRepositoryObjectsResponse {
message Item {
// The resource object key
ResourceKey object = 1;
// Size of the full resource body
int32 resource_size = 2;
// Hash for the resource
string path = 2;
// Verification hash from the origin
string hash = 3;
// Change time from the origin
int64 time = 5;
// Hash for the resource
string resource_hash = 3;
// Title inside the payload
string title = 6;
// The name of the folder in metadata
string folder = 7;
}
// The origin name
string origin = 4;
// Path on the origin
string path = 5;
// Verification hash from the origin
string hash = 6;
// Change time from the origin
int64 timestamp = 7;
}
message OriginResponse {
repeated ResourceOriginInfo items = 1;
// Item iterator
repeated Item items = 1;
// More results exist... pass this in the next request
string next_page_token = 2;
// ResourceVersion of the list response
int64 resource_version = 3;
// Error details
ErrorResult error = 3;
}
// Count the items that exist with
message CountRepositoryObjectsRequest {
// Namespace (tenant)
string namespace = 1;
// The name of the repository
// (eventually empty to count across all repositories)
string repository = 2;
}
// Count the items that exist with
message CountRepositoryObjectsResponse {
message KindCount {
string group = 1;
string resource = 2;
int64 count = 3;
}
// Stats keyed by the repository name
map<string,KindCount> stats = 1;
// Error details
ErrorResult error = 4;
ErrorResult error = 2;
}
message HealthCheckRequest {
@ -609,7 +631,7 @@ message ResourceTableColumnDefinition {
// Defines the column type. In k8s, this will resolve into both the type and format fields
ColumnType type = 2;
// The value is an arry of given type
// The value is an array of given type
bool is_array = 3;
// description is a human readable description of this column.
@ -774,9 +796,16 @@ service ResourceIndex {
// Show resource history (and trash)
rpc History(HistoryRequest) returns (HistoryResponse);
}
// Used for efficient provisioning
rpc Origin(OriginRequest) returns (OriginResponse);
// Query repository info from the search index.
// Results access control is based on access to the repository *not* the items
service RepositoryIndex {
// Describe how many resources of each type exist within a repository
rpc CountRepositoryObjects(CountRepositoryObjectsRequest) returns (CountRepositoryObjectsResponse);
// List the resources of a specific kind within a repository
rpc ListRepositoryObjects(ListRepositoryObjectsRequest) returns (ListRepositoryObjectsResponse);
}
service BlobStore {

173
pkg/storage/unified/resource/resource_grpc.pb.go
View File

@ -389,7 +389,6 @@ const (
ResourceIndex_Search_FullMethodName = "/resource.ResourceIndex/Search"
ResourceIndex_GetStats_FullMethodName = "/resource.ResourceIndex/GetStats"
ResourceIndex_History_FullMethodName = "/resource.ResourceIndex/History"
ResourceIndex_Origin_FullMethodName = "/resource.ResourceIndex/Origin"
)
// ResourceIndexClient is the client API for ResourceIndex service.
@ -404,8 +403,6 @@ type ResourceIndexClient interface {
GetStats(ctx context.Context, in *ResourceStatsRequest, opts ...grpc.CallOption) (*ResourceStatsResponse, error)
// Show resource history (and trash)
History(ctx context.Context, in *HistoryRequest, opts ...grpc.CallOption) (*HistoryResponse, error)
// Used for efficient provisioning
Origin(ctx context.Context, in *OriginRequest, opts ...grpc.CallOption) (*OriginResponse, error)
}
type resourceIndexClient struct {
@ -446,16 +443,6 @@ func (c *resourceIndexClient) History(ctx context.Context, in *HistoryRequest, o
return out, nil
}
func (c *resourceIndexClient) Origin(ctx context.Context, in *OriginRequest, opts ...grpc.CallOption) (*OriginResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(OriginResponse)
err := c.cc.Invoke(ctx, ResourceIndex_Origin_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
// ResourceIndexServer is the server API for ResourceIndex service.
// All implementations should embed UnimplementedResourceIndexServer
// for forward compatibility
@ -468,8 +455,6 @@ type ResourceIndexServer interface {
GetStats(context.Context, *ResourceStatsRequest) (*ResourceStatsResponse, error)
// Show resource history (and trash)
History(context.Context, *HistoryRequest) (*HistoryResponse, error)
// Used for efficient provisioning
Origin(context.Context, *OriginRequest) (*OriginResponse, error)
}
// UnimplementedResourceIndexServer should be embedded to have forward compatible implementations.
@ -485,9 +470,6 @@ func (UnimplementedResourceIndexServer) GetStats(context.Context, *ResourceStats
func (UnimplementedResourceIndexServer) History(context.Context, *HistoryRequest) (*HistoryResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method History not implemented")
}
func (UnimplementedResourceIndexServer) Origin(context.Context, *OriginRequest) (*OriginResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Origin not implemented")
}
// UnsafeResourceIndexServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to ResourceIndexServer will
@ -554,24 +536,6 @@ func _ResourceIndex_History_Handler(srv interface{}, ctx context.Context, dec fu
return interceptor(ctx, in, info, handler)
}
func _ResourceIndex_Origin_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(OriginRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ResourceIndexServer).Origin(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: ResourceIndex_Origin_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ResourceIndexServer).Origin(ctx, req.(*OriginRequest))
}
return interceptor(ctx, in, info, handler)
}
// ResourceIndex_ServiceDesc is the grpc.ServiceDesc for ResourceIndex service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@ -591,9 +555,142 @@ var ResourceIndex_ServiceDesc = grpc.ServiceDesc{
MethodName: "History",
Handler: _ResourceIndex_History_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "resource.proto",
}
const (
RepositoryIndex_CountRepositoryObjects_FullMethodName = "/resource.RepositoryIndex/CountRepositoryObjects"
RepositoryIndex_ListRepositoryObjects_FullMethodName = "/resource.RepositoryIndex/ListRepositoryObjects"
)
// RepositoryIndexClient is the client API for RepositoryIndex service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
//
// Query repository info from the search index.
// Results access control is based on access to the repository *not* the items
type RepositoryIndexClient interface {
// Describe how many resources of each type exist within a repository
CountRepositoryObjects(ctx context.Context, in *CountRepositoryObjectsRequest, opts ...grpc.CallOption) (*CountRepositoryObjectsResponse, error)
// List the resources of a specific kind within a repository
ListRepositoryObjects(ctx context.Context, in *ListRepositoryObjectsRequest, opts ...grpc.CallOption) (*ListRepositoryObjectsResponse, error)
}
type repositoryIndexClient struct {
cc grpc.ClientConnInterface
}
func NewRepositoryIndexClient(cc grpc.ClientConnInterface) RepositoryIndexClient {
return &repositoryIndexClient{cc}
}
func (c *repositoryIndexClient) CountRepositoryObjects(ctx context.Context, in *CountRepositoryObjectsRequest, opts ...grpc.CallOption) (*CountRepositoryObjectsResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(CountRepositoryObjectsResponse)
err := c.cc.Invoke(ctx, RepositoryIndex_CountRepositoryObjects_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *repositoryIndexClient) ListRepositoryObjects(ctx context.Context, in *ListRepositoryObjectsRequest, opts ...grpc.CallOption) (*ListRepositoryObjectsResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ListRepositoryObjectsResponse)
err := c.cc.Invoke(ctx, RepositoryIndex_ListRepositoryObjects_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
// RepositoryIndexServer is the server API for RepositoryIndex service.
// All implementations should embed UnimplementedRepositoryIndexServer
// for forward compatibility
//
// Query repository info from the search index.
// Results access control is based on access to the repository *not* the items
type RepositoryIndexServer interface {
// Describe how many resources of each type exist within a repository
CountRepositoryObjects(context.Context, *CountRepositoryObjectsRequest) (*CountRepositoryObjectsResponse, error)
// List the resources of a specific kind within a repository
ListRepositoryObjects(context.Context, *ListRepositoryObjectsRequest) (*ListRepositoryObjectsResponse, error)
}
// UnimplementedRepositoryIndexServer should be embedded to have forward compatible implementations.
type UnimplementedRepositoryIndexServer struct {
}
func (UnimplementedRepositoryIndexServer) CountRepositoryObjects(context.Context, *CountRepositoryObjectsRequest) (*CountRepositoryObjectsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method CountRepositoryObjects not implemented")
}
func (UnimplementedRepositoryIndexServer) ListRepositoryObjects(context.Context, *ListRepositoryObjectsRequest) (*ListRepositoryObjectsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListRepositoryObjects not implemented")
}
// UnsafeRepositoryIndexServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to RepositoryIndexServer will
// result in compilation errors.
type UnsafeRepositoryIndexServer interface {
mustEmbedUnimplementedRepositoryIndexServer()
}
func RegisterRepositoryIndexServer(s grpc.ServiceRegistrar, srv RepositoryIndexServer) {
s.RegisterService(&RepositoryIndex_ServiceDesc, srv)
}
func _RepositoryIndex_CountRepositoryObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(CountRepositoryObjectsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(RepositoryIndexServer).CountRepositoryObjects(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: RepositoryIndex_CountRepositoryObjects_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(RepositoryIndexServer).CountRepositoryObjects(ctx, req.(*CountRepositoryObjectsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _RepositoryIndex_ListRepositoryObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListRepositoryObjectsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(RepositoryIndexServer).ListRepositoryObjects(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: RepositoryIndex_ListRepositoryObjects_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(RepositoryIndexServer).ListRepositoryObjects(ctx, req.(*ListRepositoryObjectsRequest))
}
return interceptor(ctx, in, info, handler)
}
// RepositoryIndex_ServiceDesc is the grpc.ServiceDesc for RepositoryIndex service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var RepositoryIndex_ServiceDesc = grpc.ServiceDesc{
ServiceName: "resource.RepositoryIndex",
HandlerType: (*RepositoryIndexServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Origin",
Handler: _ResourceIndex_Origin_Handler,
MethodName: "CountRepositoryObjects",
Handler: _RepositoryIndex_CountRepositoryObjects_Handler,
},
{
MethodName: "ListRepositoryObjects",
Handler: _RepositoryIndex_ListRepositoryObjects_Handler,
},
},
Streams: []grpc.StreamDesc{},

32
pkg/storage/unified/resource/search.go
View File

@ -43,9 +43,11 @@ type ResourceIndex interface {
// When working with federated queries, the additional indexes will be passed in explicitly
Search(ctx context.Context, access authz.AccessClient, req *ResourceSearchRequest, federate []ResourceIndex) (*ResourceSearchResponse, error)
// Execute an origin query -- access control is not not checked for each item
// NOTE: this will likely be used for provisioning, or it will be removed
Origin(ctx context.Context, req *OriginRequest) (*OriginResponse, error)
// List within an response
ListRepositoryObjects(ctx context.Context, req *ListRepositoryObjectsRequest) (*ListRepositoryObjectsResponse, error)
// Counts the values in a repo
CountRepositoryObjects(ctx context.Context) (map[string]int64, error)
// Get the number of documents in the index
DocCount(ctx context.Context, folder string) (int64, error)
@ -93,7 +95,8 @@ type searchSupport struct {
}
var (
_ ResourceIndexServer = (*searchSupport)(nil)
_ ResourceIndexServer = (*searchSupport)(nil)
_ RepositoryIndexServer = (*searchSupport)(nil)
)
func newSearchSupport(opts SearchOptions, storage StorageBackend, access authz.AccessClient, blob BlobSupport, tracer trace.Tracer) (support *searchSupport, err error) {
@ -131,12 +134,25 @@ func newSearchSupport(opts SearchOptions, storage StorageBackend, access authz.A
// History implements ResourceIndexServer.
func (s *searchSupport) History(context.Context, *HistoryRequest) (*HistoryResponse, error) {
return nil, fmt.Errorf("not implemented yet... likely should not be the serarch server")
return nil, fmt.Errorf("not implemented yet... likely should not be the search server")
}
// Origin implements ResourceIndexServer.
func (s *searchSupport) Origin(context.Context, *OriginRequest) (*OriginResponse, error) {
return nil, fmt.Errorf("TBD.. rename to repository")
func (s *searchSupport) ListRepositoryObjects(ctx context.Context, req *ListRepositoryObjectsRequest) (*ListRepositoryObjectsResponse, error) {
idx, err := s.getOrCreateIndex(ctx, NamespacedResource{
Group: req.Key.Group,
Namespace: req.Key.Namespace,
Resource: req.Key.Resource,
})
if err != nil {
return &ListRepositoryObjectsResponse{
Error: AsErrorResult(err),
}, nil
}
return idx.ListRepositoryObjects(ctx, req)
}
func (s *searchSupport) CountRepositoryObjects(context.Context, *CountRepositoryObjectsRequest) (*CountRepositoryObjectsResponse, error) {
return nil, fmt.Errorf("not implemented yet... requires iterating kinds")
}
// Search implements ResourceIndexServer.

10
pkg/storage/unified/resource/server.go
View File

@ -29,6 +29,7 @@ import (
type ResourceServer interface {
ResourceStoreServer
ResourceIndexServer
RepositoryIndexServer
BlobStoreServer
DiagnosticsServer
}
@ -1073,9 +1074,12 @@ func (s *server) History(ctx context.Context, req *HistoryRequest) (*HistoryResp
return s.search.History(ctx, req)
}
// Origin implements ResourceServer.
func (s *server) Origin(ctx context.Context, req *OriginRequest) (*OriginResponse, error) {
return s.search.Origin(ctx, req)
func (s *server) ListRepositoryObjects(ctx context.Context, req *ListRepositoryObjectsRequest) (*ListRepositoryObjectsResponse, error) {
return s.search.ListRepositoryObjects(ctx, req)
}
func (s *server) CountRepositoryObjects(ctx context.Context, req *CountRepositoryObjectsRequest) (*CountRepositoryObjectsResponse, error) {
return s.search.CountRepositoryObjects(ctx, req)
}
// IsHealthy implements ResourceServer.

105
pkg/storage/unified/search/bleve.go
View File

@ -245,9 +245,108 @@ func (b *bleveIndex) Flush() (err error) {
return err
}
// Origin implements resource.DocumentIndex.
func (b *bleveIndex) Origin(ctx context.Context, req *resource.OriginRequest) (*resource.OriginResponse, error) {
panic("unimplemented")
func (b *bleveIndex) ListRepositoryObjects(ctx context.Context, req *resource.ListRepositoryObjectsRequest) (*resource.ListRepositoryObjectsResponse, error) {
if req.NextPageToken != "" {
return nil, fmt.Errorf("next page not implemented yet")
}
size := 1000000000 // big number
if req.Limit > 0 {
size = int(req.Limit)
}
found, err := b.index.SearchInContext(ctx, &bleve.SearchRequest{
Query: &query.TermQuery{
Term: req.Name,
FieldVal: resource.SEARCH_FIELD_REPOSITORY_NAME,
},
Fields: []string{
resource.SEARCH_FIELD_TITLE,
resource.SEARCH_FIELD_FOLDER,
resource.SEARCH_FIELD_REPOSITORY_NAME,
resource.SEARCH_FIELD_REPOSITORY_PATH,
resource.SEARCH_FIELD_REPOSITORY_HASH,
resource.SEARCH_FIELD_REPOSITORY_TIME,
},
Sort: search.SortOrder{
&search.SortField{
Field: resource.SEARCH_FIELD_REPOSITORY_PATH,
Type: search.SortFieldAsString,
Desc: false,
},
},
Size: size,
From: 0, // TODO! next page token!!!
})
if err != nil {
return nil, err
}
asString := func(v any) string {
if v == nil {
return ""
}
str, ok := v.(string)
if ok {
return str
}
return fmt.Sprintf("%v", v)
}
asTime := func(v any) int64 {
if v == nil {
return 0
}
intV, ok := v.(int64)
if ok {
return intV
}
str, ok := v.(string)
if ok {
t, _ := time.Parse(time.RFC3339, str)
return t.UnixMilli()
}
return 0
}
rsp := &resource.ListRepositoryObjectsResponse{}
for _, hit := range found.Hits {
item := &resource.ListRepositoryObjectsResponse_Item{
Object: &resource.ResourceKey{},
Hash: asString(hit.Fields[resource.SEARCH_FIELD_REPOSITORY_HASH]),
Path: asString(hit.Fields[resource.SEARCH_FIELD_REPOSITORY_PATH]),
Time: asTime(hit.Fields[resource.SEARCH_FIELD_REPOSITORY_TIME]),
Title: asString(hit.Fields[resource.SEARCH_FIELD_TITLE]),
Folder: asString(hit.Fields[resource.SEARCH_FIELD_FOLDER]),
}
err := item.Object.ReadSearchID(hit.ID)
if err != nil {
return nil, err
}
rsp.Items = append(rsp.Items, item)
}
return rsp, nil
}
func (b *bleveIndex) CountRepositoryObjects(ctx context.Context) (map[string]int64, error) {
found, err := b.index.SearchInContext(ctx, &bleve.SearchRequest{
Query: bleve.NewMatchAllQuery(),
Size: 0,
Facets: bleve.FacetsRequest{
"count": bleve.NewFacetRequest(resource.SEARCH_FIELD_REPOSITORY_NAME, 1000), // typically less then 5
},
})
if err != nil {
return nil, err
}
rsp := make(map[string]int64)
f, ok := found.Facets["count"]
if ok && f.Terms != nil {
for _, v := range f.Terms.Terms() {
rsp[v.Term] = int64(v.Count)
}
}
return rsp, nil
}
// Search implements resource.DocumentIndex.

31
pkg/storage/unified/search/bleve_mappings.go
View File

@ -67,17 +67,38 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
}
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_FOLDER, folderMapping)
repoMapping := &mapping.FieldMapping{
Name: resource.SEARCH_FIELD_REPOSITORY,
// Repositories
repo := bleve.NewDocumentStaticMapping()
repo.AddFieldMappingsAt("name", &mapping.FieldMapping{
Name: "name",
Type: "text",
Analyzer: keyword.Name,
Store: true,
Index: true,
IncludeTermVectors: false,
IncludeInAll: true,
DocValues: true,
}
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_REPOSITORY, repoMapping)
})
repo.AddFieldMappingsAt("path", &mapping.FieldMapping{
Name: "path",
Type: "text",
Analyzer: keyword.Name,
Store: true,
Index: true,
IncludeTermVectors: false,
IncludeInAll: true,
})
repo.AddFieldMappingsAt("hash", &mapping.FieldMapping{
Name: "hash",
Type: "text",
Analyzer: keyword.Name,
Store: true,
Index: true,
IncludeTermVectors: false,
IncludeInAll: true,
})
repo.AddFieldMappingsAt("time", mapping.NewDateTimeFieldMapping())
mapper.AddSubDocumentMapping("repo", repo)
labelMapper := bleve.NewDocumentMapping()
mapper.AddSubDocumentMapping(resource.SEARCH_FIELD_LABELS, labelMapper)

9
pkg/storage/unified/search/bleve_mappings_test.go
View File

@ -26,9 +26,10 @@ func TestDocumentMapping(t *testing.T) {
},
RV: 1234,
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "nnn",
Path: "ppp",
Hash: "hhh",
Name: "nnn",
Path: "ppp",
Hash: "hhh",
Timestamp: asTimePointer(1234),
},
}
@ -42,5 +43,5 @@ func TestDocumentMapping(t *testing.T) {
fmt.Printf("DOC: fields %d\n", len(doc.Fields))
fmt.Printf("DOC: size %d\n", doc.Size())
require.Equal(t, 12, len(doc.Fields))
require.Equal(t, 13, len(doc.Fields))
}

78
pkg/storage/unified/search/bleve_test.go
View File

@ -3,9 +3,11 @@ package search
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -82,6 +84,12 @@ func TestBleveBackend(t *testing.T) {
utils.LabelKeyDeprecatedInternalID: "10", // nolint:staticcheck
},
Tags: []string{"aa", "bb"},
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "repo-1",
Path: "path/to/aaa.json",
Hash: "xyz",
Timestamp: asTimePointer(1609462800000), // 2021
},
})
_ = index.Write(&resource.IndexableDocument{
RV: 2,
@ -105,6 +113,12 @@ func TestBleveBackend(t *testing.T) {
"region": "east",
utils.LabelKeyDeprecatedInternalID: "11", // nolint:staticcheck
},
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "repo-1",
Path: "path/to/bbb.json",
Hash: "hijk",
Timestamp: asTimePointer(1640998800000), // 2022
},
})
_ = index.Write(&resource.IndexableDocument{
RV: 3,
@ -119,7 +133,8 @@ func TestBleveBackend(t *testing.T) {
TitleSort: "ccc (dash)",
Folder: "zzz",
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "r0",
Name: "repo2",
Path: "path/in/repo2.yaml",
},
Fields: map[string]any{
DASHBOARD_LEGACY_ID: 12,
@ -201,6 +216,53 @@ func TestBleveBackend(t *testing.T) {
rsp.Results.Rows[0].Key.Name,
rsp.Results.Rows[1].Key.Name,
})
// Now look for repositories
found, err := index.ListRepositoryObjects(ctx, &resource.ListRepositoryObjectsRequest{
Name: "repo-1",
Limit: 100,
})
require.NoError(t, err)
jj, err := json.MarshalIndent(found, "", " ")
require.NoError(t, err)
fmt.Printf("%s\n", string(jj))
require.JSONEq(t, `{
"items": [
{
"object": {
"namespace": "ns",
"group": "dashboard.grafana.app",
"resource": "dashboards",
"name": "aaa"
},
"path": "path/to/aaa.json",
"hash": "xyz",
"time": 1609462800000,
"title": "aaa (dash)",
"folder": "xxx"
},
{
"object": {
"namespace": "ns",
"group": "dashboard.grafana.app",
"resource": "dashboards",
"name": "bbb"
},
"path": "path/to/bbb.json",
"hash": "hijk",
"time": 1640998800000,
"title": "bbb (dash)",
"folder": "xxx"
}
]
}`, string(jj))
counts, err := index.CountRepositoryObjects(ctx)
require.NoError(t, err)
require.Equal(t, map[string]int64{
"repo-1": 2,
"repo2": 1,
}, counts)
})
t.Run("build folders", func(t *testing.T) {
@ -222,6 +284,12 @@ func TestBleveBackend(t *testing.T) {
},
Title: "zzz (folder)",
TitleSort: "zzz (folder)",
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "repo-1",
Path: "path/to/folder.json",
Hash: "xxxx",
Timestamp: asTimePointer(300),
},
})
_ = index.Write(&resource.IndexableDocument{
RV: 2,
@ -327,3 +395,11 @@ func TestBleveBackend(t *testing.T) {
}`, string(disp))
})
}
func asTimePointer(milli int64) *time.Time {
if milli > 0 {
t := time.UnixMilli(milli)
return &t
}
return nil
}

2
pkg/storage/unified/search/testdata/doc/folder-aaa-out.json vendored
View File

@ -11,7 +11,7 @@
"title_sort": "test-aaa",
"created": 1730490142000,
"createdBy": "user:1",
"repository": {
"repo": {
"name": "SQL"
}
}

2
pkg/storage/unified/search/testdata/doc/folder-bbb-out.json vendored
View File

@ -11,7 +11,7 @@
"title_sort": "test-bbb",
"created": 1730490142000,
"createdBy": "user:1",
"repository": {
"repo": {
"name": "SQL"
}
}

2
pkg/storage/unified/search/testdata/doc/playlist-aaa-out.json vendored
View File

@ -11,7 +11,7 @@
"title_sort": "test aaa",
"created": 1731336353000,
"createdBy": "user:t000000001",
"repository": {
"repo": {
"name": "UI",
"path": "/playlists/new",
"hash": "Grafana v11.4.0-pre (c0de407fee)"

2
pkg/storage/unified/search/testdata/doc/report-aaa-out.json vendored
View File

@ -11,7 +11,7 @@
"title_sort": "test aaa",
"created": 1706690655000,
"createdBy": "user:abc",
"repository": {
"repo": {
"name": "SQL"
}
}

15
pkg/storage/unified/sql/search.go
View File

@ -2,6 +2,7 @@ package sql
import (
"context"
"fmt"
"net/http"
"github.com/grafana/grafana/pkg/storage/unified/resource"
@ -62,14 +63,12 @@ func (b *backend) History(context.Context, *resource.HistoryRequest) (*resource.
}, nil
}
// Origin implements resource.ResourceIndexServer.
func (b *backend) Origin(context.Context, *resource.OriginRequest) (*resource.OriginResponse, error) {
return &resource.OriginResponse{
Error: &resource.ErrorResult{
Code: http.StatusNotImplemented,
Message: "SQL backend does not implement Origin",
},
}, nil
func (b *backend) RepositoryList(ctx context.Context, req *resource.ListRepositoryObjectsRequest) (*resource.ListRepositoryObjectsResponse, error) {
return nil, fmt.Errorf("SQL backend does not implement RepositoryList")
}
func (b *backend) RepositoryStats(context.Context, *resource.CountRepositoryObjectsRequest) (*resource.CountRepositoryObjectsResponse, error) {
return nil, fmt.Errorf("SQL backend does not implement RepositoryStats")
}
// Search implements resource.ResourceIndexServer.