From 8165258a2d1d3b610acdb32ebaf679547208ea7e Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 5 Dec 2024 13:58:13 +0300 Subject: [PATCH] Storage: Query stats within a namespace (#97403) --- .../apis/dashboard/legacy/sql_dashboards.go | 2 +- pkg/storage/unified/resource/cdk_backend.go | 2 +- pkg/storage/unified/resource/search.go | 2 +- pkg/storage/unified/resource/server.go | 3 ++- pkg/storage/unified/sql/backend.go | 3 ++- pkg/storage/unified/sql/data/resource_stats.sql | 4 ++++ pkg/storage/unified/sql/queries.go | 3 ++- pkg/storage/unified/sql/queries_test.go | 8 ++++++++ pkg/storage/unified/sql/test/integration_test.go | 2 +- .../mysql--resource_stats-query-namespace.sql | 14 ++++++++++++++ .../postgres--resource_stats-query-namespace.sql | 14 ++++++++++++++ .../sqlite--resource_stats-query-namespace.sql | 14 ++++++++++++++ 12 files changed, 64 insertions(+), 7 deletions(-) create mode 100755 pkg/storage/unified/sql/testdata/mysql--resource_stats-query-namespace.sql create mode 100755 pkg/storage/unified/sql/testdata/postgres--resource_stats-query-namespace.sql create mode 100755 pkg/storage/unified/sql/testdata/sqlite--resource_stats-query-namespace.sql diff --git a/pkg/registry/apis/dashboard/legacy/sql_dashboards.go b/pkg/registry/apis/dashboard/legacy/sql_dashboards.go index 817f0e49323..bcaf4432089 100644 --- a/pkg/registry/apis/dashboard/legacy/sql_dashboards.go +++ b/pkg/registry/apis/dashboard/legacy/sql_dashboards.go @@ -132,7 +132,7 @@ type rowsWrapper struct { err error } -func (a *dashboardSqlAccess) GetResourceStats(ctx context.Context, minCount int) ([]resource.ResourceStats, error) { +func (a *dashboardSqlAccess) GetResourceStats(ctx context.Context, namespace string, minCount int) ([]resource.ResourceStats, error) { return nil, fmt.Errorf("not implemented") } diff --git a/pkg/storage/unified/resource/cdk_backend.go b/pkg/storage/unified/resource/cdk_backend.go index 34d49fd2fcc..8eb12090f68 100644 --- a/pkg/storage/unified/resource/cdk_backend.go +++ b/pkg/storage/unified/resource/cdk_backend.go @@ -109,7 +109,7 @@ func (s *cdkBackend) getPath(key *ResourceKey, rv int64) string { } // GetResourceStats implements Backend. -func (s *cdkBackend) GetResourceStats(ctx context.Context, minCount int) ([]ResourceStats, error) { +func (s *cdkBackend) GetResourceStats(ctx context.Context, namespace string, minCount int) ([]ResourceStats, error) { return nil, fmt.Errorf("not implemented") } diff --git a/pkg/storage/unified/resource/search.go b/pkg/storage/unified/resource/search.go index e1f2ed8cb22..63c2cf24507 100644 --- a/pkg/storage/unified/resource/search.go +++ b/pkg/storage/unified/resource/search.go @@ -179,7 +179,7 @@ func (s *searchSupport) init(ctx context.Context) error { group := errgroup.Group{} group.SetLimit(s.initWorkers) - stats, err := s.storage.GetResourceStats(ctx, s.initMinSize) + stats, err := s.storage.GetResourceStats(ctx, "", s.initMinSize) if err != nil { return err } diff --git a/pkg/storage/unified/resource/server.go b/pkg/storage/unified/resource/server.go index 614f8534ed7..8339deefbc8 100644 --- a/pkg/storage/unified/resource/server.go +++ b/pkg/storage/unified/resource/server.go @@ -95,7 +95,8 @@ type StorageBackend interface { // For HA setups, this will be more events than the local WriteEvent above! WatchWriteEvents(ctx context.Context) (<-chan *WrittenEvent, error) - GetResourceStats(ctx context.Context, minCount int) ([]ResourceStats, error) + // Get resource stats within the storage backend. When namespace is empty, it will apply to all + GetResourceStats(ctx context.Context, namespace string, minCount int) ([]ResourceStats, error) } type ResourceStats struct { diff --git a/pkg/storage/unified/sql/backend.go b/pkg/storage/unified/sql/backend.go index 4be8773cf67..ae207870164 100644 --- a/pkg/storage/unified/sql/backend.go +++ b/pkg/storage/unified/sql/backend.go @@ -122,12 +122,13 @@ func (b *backend) Stop(_ context.Context) error { } // GetResourceStats implements Backend. -func (b *backend) GetResourceStats(ctx context.Context, minCount int) ([]resource.ResourceStats, error) { +func (b *backend) GetResourceStats(ctx context.Context, namespace string, minCount int) ([]resource.ResourceStats, error) { _, span := b.tracer.Start(ctx, tracePrefix+".GetResourceStats") defer span.End() req := &sqlStatsRequest{ SQLTemplate: sqltemplate.New(b.dialect), + Namespace: namespace, MinCount: minCount, // not used in query... yet? } diff --git a/pkg/storage/unified/sql/data/resource_stats.sql b/pkg/storage/unified/sql/data/resource_stats.sql index c5d8cc69589..3f86cb184f9 100644 --- a/pkg/storage/unified/sql/data/resource_stats.sql +++ b/pkg/storage/unified/sql/data/resource_stats.sql @@ -5,6 +5,10 @@ SELECT COUNT(*), MAX({{ .Ident "resource_version" }}) FROM {{ .Ident "resource" }} +{{ if .Namespace }} +WHERE 1 = 1 + AND {{ .Ident "namespace" }} = {{ .Arg .Namespace }} +{{ end}} GROUP BY {{ .Ident "namespace" }}, {{ .Ident "group" }}, diff --git a/pkg/storage/unified/sql/queries.go b/pkg/storage/unified/sql/queries.go index 27fc77ad55c..3d12a89a32e 100644 --- a/pkg/storage/unified/sql/queries.go +++ b/pkg/storage/unified/sql/queries.go @@ -74,7 +74,8 @@ func (r sqlResourceRequest) Validate() error { type sqlStatsRequest struct { sqltemplate.SQLTemplate - MinCount int + Namespace string + MinCount int } func (r sqlStatsRequest) Validate() error { diff --git a/pkg/storage/unified/sql/queries_test.go b/pkg/storage/unified/sql/queries_test.go index 30fe25eed2f..958f5497ee3 100644 --- a/pkg/storage/unified/sql/queries_test.go +++ b/pkg/storage/unified/sql/queries_test.go @@ -228,6 +228,14 @@ func TestUnifiedStorageQueries(t *testing.T) { MinCount: 10, // Not yet used in query (only response filter) }, }, + { + Name: "query-namespace", + Data: &sqlStatsRequest{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Namespace: "default", + MinCount: 10, // Not yet used in query (only response filter) + }, + }, }, }}) } diff --git a/pkg/storage/unified/sql/test/integration_test.go b/pkg/storage/unified/sql/test/integration_test.go index 2abe4aec365..a4423cd5fd9 100644 --- a/pkg/storage/unified/sql/test/integration_test.go +++ b/pkg/storage/unified/sql/test/integration_test.go @@ -98,7 +98,7 @@ func TestIntegrationBackendHappyPath(t *testing.T) { require.NoError(t, err) require.Greater(t, rv3, rv2) - stats, err := backend.GetResourceStats(ctx, 0) + stats, err := backend.GetResourceStats(ctx, "", 0) require.NoError(t, err) require.Len(t, stats, 1) require.Equal(t, int64(3), stats[0].Count) diff --git a/pkg/storage/unified/sql/testdata/mysql--resource_stats-query-namespace.sql b/pkg/storage/unified/sql/testdata/mysql--resource_stats-query-namespace.sql new file mode 100755 index 00000000000..d2414188349 --- /dev/null +++ b/pkg/storage/unified/sql/testdata/mysql--resource_stats-query-namespace.sql @@ -0,0 +1,14 @@ +SELECT + `namespace`, + `group`, + `resource`, + COUNT(*), + MAX(`resource_version`) +FROM `resource` +WHERE 1 = 1 + AND `namespace` = 'default' +GROUP BY + `namespace`, + `group`, + `resource` +; diff --git a/pkg/storage/unified/sql/testdata/postgres--resource_stats-query-namespace.sql b/pkg/storage/unified/sql/testdata/postgres--resource_stats-query-namespace.sql new file mode 100755 index 00000000000..c0476882345 --- /dev/null +++ b/pkg/storage/unified/sql/testdata/postgres--resource_stats-query-namespace.sql @@ -0,0 +1,14 @@ +SELECT + "namespace", + "group", + "resource", + COUNT(*), + MAX("resource_version") +FROM "resource" +WHERE 1 = 1 + AND "namespace" = 'default' +GROUP BY + "namespace", + "group", + "resource" +; diff --git a/pkg/storage/unified/sql/testdata/sqlite--resource_stats-query-namespace.sql b/pkg/storage/unified/sql/testdata/sqlite--resource_stats-query-namespace.sql new file mode 100755 index 00000000000..c0476882345 --- /dev/null +++ b/pkg/storage/unified/sql/testdata/sqlite--resource_stats-query-namespace.sql @@ -0,0 +1,14 @@ +SELECT + "namespace", + "group", + "resource", + COUNT(*), + MAX("resource_version") +FROM "resource" +WHERE 1 = 1 + AND "namespace" = 'default' +GROUP BY + "namespace", + "group", + "resource" +;