From 67b6be5515f01a88e58fe0fcad8d9eadc7149625 Mon Sep 17 00:00:00 2001 From: Todd Treece <360020+toddtreece@users.noreply.github.com> Date: Thu, 1 Feb 2024 17:27:30 -0500 Subject: [PATCH] K8s: Refactor config/options for aggregation (#81739) --- .github/CODEOWNERS | 3 +- pkg/aggregator/aggregator.go | 628 ------------------ pkg/aggregator/config.go | 57 -- pkg/aggregator/crdRegistrationController.go | 214 ------ pkg/api/frontendsettings_test.go | 2 +- pkg/api/http_server.go | 4 +- pkg/api/metrics.go | 2 +- pkg/api/playlist.go | 4 +- pkg/cmd/grafana/apiserver/aggregator.md | 51 -- pkg/cmd/grafana/apiserver/cmd.go | 109 +-- pkg/cmd/grafana/apiserver/server.go | 13 +- pkg/cmd/grafana/main.go | 12 - pkg/registry/apis/apis.go | 2 + .../apis/dashboard/access/sql_dashboards.go | 4 +- pkg/registry/apis/dashboard/authorizer.go | 2 +- pkg/registry/apis/dashboard/legacy_storage.go | 2 +- pkg/registry/apis/dashboard/register.go | 19 +- pkg/registry/apis/dashboard/sub_access.go | 2 +- pkg/registry/apis/dashboard/sub_versions.go | 2 +- .../apis/dashboard/summary_storage.go | 2 +- pkg/registry/apis/datasource/plugincontext.go | 4 +- pkg/registry/apis/datasource/querier.go | 148 +++++ pkg/registry/apis/datasource/register.go | 11 +- pkg/registry/apis/datasource/standalone.go | 2 +- pkg/registry/apis/example/dummy_storage.go | 4 +- pkg/registry/apis/example/register.go | 17 +- pkg/registry/apis/example/storage.go | 2 +- pkg/registry/apis/example/subresource.go | 2 +- pkg/registry/apis/featuretoggle/features.go | 2 +- pkg/registry/apis/featuretoggle/register.go | 15 +- pkg/registry/apis/featuretoggle/toggles.go | 2 +- pkg/registry/apis/folders/conversions.go | 4 +- pkg/registry/apis/folders/legacy_storage.go | 6 +- pkg/registry/apis/folders/register.go | 17 +- pkg/registry/apis/folders/storage.go | 4 +- pkg/registry/apis/folders/sub_children.go | 2 +- pkg/registry/apis/folders/sub_parents.go | 2 +- pkg/registry/apis/playlist/conversions.go | 4 +- .../apis/playlist/conversions_test.go | 2 +- pkg/registry/apis/playlist/legacy_storage.go | 2 +- pkg/registry/apis/playlist/register.go | 15 +- pkg/registry/apis/playlist/storage.go | 4 +- pkg/registry/apis/query/register.go | 15 +- pkg/registry/apis/service/register.go | 12 +- pkg/registry/apis/service/storage.go | 6 +- pkg/registry/apis/wireset.go | 2 + .../backgroundsvcs/background_services.go | 2 +- pkg/server/wire.go | 2 +- .../README.md | 0 .../apiserver}/aggregator/README.md | 0 .../apiserver/aggregator/aggregator.go | 285 ++++++++ .../aggregator/availableController.go | 0 .../apiserver}/aggregator/resolver.go | 0 .../auth/authorizer/impersonation.go | 0 .../auth/authorizer/impersonation_test.go | 0 .../auth/authorizer/org_id.go | 2 +- .../auth/authorizer/org_role.go | 0 .../auth/authorizer/provider.go | 0 .../auth/authorizer/stack_id.go | 2 +- .../builder}/common.go | 7 +- .../builder}/helper.go | 62 +- .../builder}/openapi.go | 6 +- .../builder}/request_handler.go | 8 +- pkg/services/apiserver/config.go | 48 ++ .../endpoints/request/namespace.go | 0 .../endpoints/request/namespace_test.go | 2 +- pkg/services/apiserver/options/aggregator.go | 112 ++++ pkg/services/apiserver/options/extra.go | 46 ++ .../options}/log.go | 2 +- pkg/services/apiserver/options/options.go | 126 ++++ pkg/services/apiserver/options/storage.go | 51 ++ .../registry/generic/strategy.go | 0 .../rest/dualwriter.go | 0 .../service.go | 235 +++---- .../storage/entity/fieldRequirements.go | 0 .../storage/entity/restoptions.go | 0 .../storage/entity/storage.go | 0 .../storage/entity/utils.go | 2 +- .../storage/entity/utils_test.go | 0 .../storage/file/file.go | 0 .../storage/file/restoptions.go | 0 .../storage/file/util.go | 0 .../storage/file/watchset.go | 0 .../utils/clientConfig.go | 0 .../utils/meta.go | 0 .../utils/meta_test.go | 2 +- .../utils/tableConverter.go | 0 .../utils/tableConverter_test.go | 2 +- .../utils/uids.go | 0 .../wireset.go | 6 +- pkg/services/grafana-apiserver/config.go | 60 -- pkg/services/grafana-apiserver/config_test.go | 40 -- pkg/tests/apis/helper.go | 2 +- 93 files changed, 1104 insertions(+), 1448 deletions(-) delete mode 100644 pkg/aggregator/aggregator.go delete mode 100644 pkg/aggregator/config.go delete mode 100644 pkg/aggregator/crdRegistrationController.go delete mode 100644 pkg/cmd/grafana/apiserver/aggregator.md create mode 100644 pkg/registry/apis/datasource/querier.go rename pkg/services/{grafana-apiserver => apiserver}/README.md (100%) rename pkg/{ => services/apiserver}/aggregator/README.md (100%) create mode 100644 pkg/services/apiserver/aggregator/aggregator.go rename pkg/{ => services/apiserver}/aggregator/availableController.go (100%) rename pkg/{ => services/apiserver}/aggregator/resolver.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/auth/authorizer/impersonation.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/auth/authorizer/impersonation_test.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/auth/authorizer/org_id.go (95%) rename pkg/services/{grafana-apiserver => apiserver}/auth/authorizer/org_role.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/auth/authorizer/provider.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/auth/authorizer/stack_id.go (94%) rename pkg/services/{grafana-apiserver => apiserver/builder}/common.go (94%) rename pkg/services/{grafana-apiserver => apiserver/builder}/helper.go (65%) rename pkg/services/{grafana-apiserver => apiserver/builder}/openapi.go (99%) rename pkg/services/{grafana-apiserver => apiserver/builder}/request_handler.go (96%) create mode 100644 pkg/services/apiserver/config.go rename pkg/services/{grafana-apiserver => apiserver}/endpoints/request/namespace.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/endpoints/request/namespace_test.go (97%) create mode 100644 pkg/services/apiserver/options/aggregator.go create mode 100644 pkg/services/apiserver/options/extra.go rename pkg/services/{grafana-apiserver => apiserver/options}/log.go (97%) create mode 100644 pkg/services/apiserver/options/options.go create mode 100644 pkg/services/apiserver/options/storage.go rename pkg/services/{grafana-apiserver => apiserver}/registry/generic/strategy.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/rest/dualwriter.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/service.go (63%) rename pkg/services/{grafana-apiserver => apiserver}/storage/entity/fieldRequirements.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/entity/restoptions.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/entity/storage.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/entity/utils.go (98%) rename pkg/services/{grafana-apiserver => apiserver}/storage/entity/utils_test.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/file/file.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/file/restoptions.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/file/util.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/file/watchset.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/utils/clientConfig.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/utils/meta.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/utils/meta_test.go (98%) rename pkg/services/{grafana-apiserver => apiserver}/utils/tableConverter.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/utils/tableConverter_test.go (98%) rename pkg/services/{grafana-apiserver => apiserver}/utils/uids.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/wireset.go (63%) delete mode 100644 pkg/services/grafana-apiserver/config.go delete mode 100644 pkg/services/grafana-apiserver/config_test.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1ccee149cb5..6bd4a84f4e4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -108,7 +108,7 @@ /pkg/services/dashboardversion/ @grafana/backend-platform /pkg/services/encryption/ @grafana/backend-platform /pkg/services/folder/ @grafana/backend-platform -/pkg/services/grafana-apiserver @grafana/grafana-app-platform-squad +/pkg/services/apiserver @grafana/grafana-app-platform-squad /pkg/services/hooks/ @grafana/backend-platform /pkg/services/kmsproviders/ @grafana/backend-platform /pkg/services/licensing/ @grafana/backend-platform @@ -275,7 +275,6 @@ /pkg/modules/ @grafana/grafana-app-platform-squad /pkg/kindsysreport/ @grafana/grafana-app-platform-squad /pkg/services/grpcserver/ @grafana/grafana-app-platform-squad -/pkg/aggregator @grafana/grafana-app-platform-squad /pkg/generated @grafana/grafana-app-platform-squad # Alerting diff --git a/pkg/aggregator/aggregator.go b/pkg/aggregator/aggregator.go deleted file mode 100644 index 74bc2118847..00000000000 --- a/pkg/aggregator/aggregator.go +++ /dev/null @@ -1,628 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only -// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/aggregator.go -// Provenance-includes-license: Apache-2.0 -// Provenance-includes-copyright: The Kubernetes Authors. -// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/server.go -// Provenance-includes-license: Apache-2.0 -// Provenance-includes-copyright: The Kubernetes Authors. -// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/pkg/controlplane/apiserver/apiextensions.go -// Provenance-includes-license: Apache-2.0 -// Provenance-includes-copyright: The Kubernetes Authors. - -package aggregator - -import ( - "crypto/tls" - "fmt" - "io" - "net" - "net/http" - "path" - "strings" - "sync" - "time" - - servicev0alpha1 "github.com/grafana/grafana/pkg/apis/service/v0alpha1" - serviceclientset "github.com/grafana/grafana/pkg/generated/clientset/versioned" - informersv0alpha1 "github.com/grafana/grafana/pkg/generated/informers/externalversions" - "github.com/grafana/grafana/pkg/registry/apis/service" - grafanaAPIServer "github.com/grafana/grafana/pkg/services/grafana-apiserver" - filestorage "github.com/grafana/grafana/pkg/services/grafana-apiserver/storage/file" - - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions" - apiextensionsopenapi "k8s.io/apiextensions-apiserver/pkg/generated/openapi" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - utilnet "k8s.io/apimachinery/pkg/util/net" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" - openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" - genericfeatures "k8s.io/apiserver/pkg/features" - genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/healthz" - "k8s.io/apiserver/pkg/server/options" - "k8s.io/apiserver/pkg/server/resourceconfig" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/apiserver/pkg/util/openapi" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" - "k8s.io/klog/v2" - v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" - v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper" - apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" - aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" - aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" - apiregistrationclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" - apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" - apiregistrationInformers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1" - "k8s.io/kube-aggregator/pkg/controllers/autoregister" - apiserver "k8s.io/kube-aggregator/pkg/controllers/status" - aggregatoropenapi "k8s.io/kube-aggregator/pkg/generated/openapi" - "k8s.io/kube-openapi/pkg/common" -) - -// AggregatorServerOptions contains the state for the aggregator apiserver -type AggregatorServerOptions struct { - Builders []grafanaAPIServer.APIGroupBuilder - AlternateDNS []string - Config *Config - serviceResolver ServiceResolver - - sharedInformerFactory informersv0alpha1.SharedInformerFactory - - StdOut io.Writer - StdErr io.Writer -} - -func NewAggregatorServerOptions(out, errOut io.Writer, - options *options.RecommendedOptions, - extraConfig *ExtraConfig, -) (*AggregatorServerOptions, error) { - sharedConfig, err := initSharedConfig(options, aggregatorscheme.Codecs, nil) - if err != nil { - klog.Errorf("Error creating shared config: %s", err) - return nil, err - } - - sharedInformerFactory, err := initSharedInformerFactory(sharedConfig) - if err != nil { - klog.Errorf("Error creating shared informer factory: %s", err) - return nil, err - } - - serviceResolver, err := initServiceResolver(sharedInformerFactory) - if err != nil { - klog.Errorf("Error creating service resolver: %s", err) - return nil, err - } - - fakeInformers := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 10*time.Minute) - builders := []grafanaAPIServer.APIGroupBuilder{ - service.NewServiceAPIBuilder(), - } - - extensionsConfig, err := initApiExtensionsConfig(options, sharedConfig, fakeInformers, serviceResolver, extraConfig.DataPath) - if err != nil { - klog.Errorf("Error creating extensions config: %s", err) - return nil, err - } - - aggregatorConfig, err := initAggregatorConfig(options, sharedConfig, extraConfig, fakeInformers, builders, serviceResolver, extraConfig.DataPath) - if err != nil { - klog.Errorf("Error creating aggregator config: %s", err) - return nil, err - } - - return &AggregatorServerOptions{ - StdOut: out, - StdErr: errOut, - Builders: builders, - sharedInformerFactory: sharedInformerFactory, - serviceResolver: serviceResolver, - Config: &Config{ - Aggregator: aggregatorConfig, - ApiExtensions: extensionsConfig, - - SharedConfig: sharedConfig, - extraConfig: extraConfig, - }, - }, nil -} - -func (o *AggregatorServerOptions) LoadAPIGroupBuilders() error { - // Install schemas - for _, b := range o.Builders { - if err := b.InstallSchema(aggregatorscheme.Scheme); err != nil { - return err - } - } - return nil -} - -func initSharedConfig(options *options.RecommendedOptions, codecs serializer.CodecFactory, alternateDNS []string) (*genericapiserver.RecommendedConfig, error) { - if err := options.SecureServing.MaybeDefaultWithSelfSignedCerts( - "localhost", alternateDNS, []net.IP{net.IPv4(127, 0, 0, 1)}, - ); err != nil { - return nil, fmt.Errorf("error creating self-signed certificates: %v", err) - } - - options.Authentication.RemoteKubeConfigFileOptional = true - options.Authorization.RemoteKubeConfigFileOptional = true - - options.Admission = nil - - if options.CoreAPI.CoreAPIKubeconfigPath == "" { - options.CoreAPI = nil - } - - serverConfig := genericapiserver.NewRecommendedConfig(codecs) - - // NOTE: AggregatedDiscoveryGroupManager in kube-apiserver is set up by controlplane APIServerConfig creation - // Here, we adopt that one line in addition to what recommendedOptions gives us - // Without it, CRDs work on API routes (and are registered in openapi) but not discoverable by kubectl - serverConfig.AggregatedDiscoveryGroupManager = aggregated.NewResourceManager("apis") - - if options.CoreAPI == nil { - if err := modifiedApplyTo(options, serverConfig); err != nil { - return nil, err - } - } else { - if err := options.ApplyTo(serverConfig); err != nil { - return nil, err - } - } - - return serverConfig, nil -} - -// A copy of ApplyTo in recommended.go, but for >= 0.28, server pkg in apiserver does a bit extra causing -// a panic when CoreAPI is set to nil -func modifiedApplyTo(options *options.RecommendedOptions, config *genericapiserver.RecommendedConfig) error { - if err := options.Etcd.ApplyTo(&config.Config); err != nil { - return err - } - if err := options.EgressSelector.ApplyTo(&config.Config); err != nil { - return err - } - if err := options.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil { - return err - } - if err := options.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil { - return err - } - if err := options.Authentication.ApplyTo(&config.Config.Authentication, config.SecureServing, config.OpenAPIConfig); err != nil { - return err - } - if err := options.Authorization.ApplyTo(&config.Config.Authorization); err != nil { - return err - } - if err := options.Audit.ApplyTo(&config.Config); err != nil { - return err - } - - // TODO: determine whether we need flow control (API priority and fairness) - //if err := options.Features.ApplyTo(&config.Config); err != nil { - // return err - //} - - if err := options.CoreAPI.ApplyTo(config); err != nil { - return err - } - - _, err := options.ExtraAdmissionInitializers(config) - if err != nil { - return err - } - return nil -} - -func getMergedOpenAPIDefinitions(builders []grafanaAPIServer.APIGroupBuilder, ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { - // Add OpenAPI specs for each group+version - prerequisiteAPIs := grafanaAPIServer.GetOpenAPIDefinitions(builders)(ref) - aggregatorAPIs := aggregatoropenapi.GetOpenAPIDefinitions(ref) - - for k, v := range prerequisiteAPIs { - aggregatorAPIs[k] = v - } - - return aggregatorAPIs -} - -func initSharedInformerFactory(sharedConfig *genericapiserver.RecommendedConfig) (informersv0alpha1.SharedInformerFactory, error) { - serviceClient, err := serviceclientset.NewForConfig(sharedConfig.LoopbackClientConfig) - if err != nil { - return nil, err - } - return informersv0alpha1.NewSharedInformerFactory( - serviceClient, - 5*time.Minute, // this is effectively used as a refresh interval right now. Might want to do something nicer later on. - ), nil -} - -func initServiceResolver(factory informersv0alpha1.SharedInformerFactory) (apiserver.ServiceResolver, error) { - return NewExternalNameResolver(factory.Service().V0alpha1().ExternalNames().Lister()), nil -} - -func initApiExtensionsConfig(options *options.RecommendedOptions, - sharedConfig *genericapiserver.RecommendedConfig, - fakeInfomers informers.SharedInformerFactory, - serviceResolver apiserver.ServiceResolver, - dataPath string, -) (*apiextensionsapiserver.Config, error) { - // make a shallow copy to let us twiddle a few things - // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the api extensions - genericConfig := sharedConfig.Config - - genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - genericConfig.RESTOptionsGetter = nil - - // copy the etcd options so we don't mutate originals. - // we assume that the etcd options have been completed already. avoid messing with anything outside - // of changes to StorageConfig as that may lead to unexpected behavior when the options are applied. - etcdOptions := *options.Etcd - // this is where the true decodable levels come from. - etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(apiextensionsv1beta1.SchemeGroupVersion, v1.SchemeGroupVersion) - // prefer the more compact serialization (v1beta1) for storage until https://issue.k8s.io/82292 is resolved for objects whose v1 serialization is too big but whose v1beta1 serialization can be stored - etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(apiextensionsv1beta1.SchemeGroupVersion, schema.GroupKind{Group: apiextensionsv1beta1.GroupName}) - etcdOptions.SkipHealthEndpoints = true // avoid double wiring of health checks - if err := etcdOptions.ApplyTo(&genericConfig); err != nil { - return nil, err - } - - restOptionsGetter := filestorage.NewRESTOptionsGetter(path.Join(dataPath, "grafana-apiextensionsserver"), etcdOptions.StorageConfig) - genericConfig.RESTOptionsGetter = restOptionsGetter - - // NOTE: ignoring genericConfig.ResourceTransformers in crdOptionsGetter creation for now - // crdOptionsGetter := apiextensionsoptions.NewCRDRESTOptionsGetter(etcdOptions, genericConfig.ResourceTransformers, ) - // The following is equivalent code to apiextensionsoptions.NewCRDRESTOptionsGetter with lesser dependencies - crdEtcdOptions := etcdOptions - crdEtcdOptions.StorageConfig.Codec = unstructured.UnstructuredJSONScheme - crdEtcdOptions.StorageConfig.StorageObjectCountTracker = genericConfig.StorageObjectCountTracker - crdEtcdOptions.WatchCacheSizes = nil // this control is not provided for custom resources - - // override MergedResourceConfig with apiextensions defaults and registry - mergedResourceConfig, err := resourceconfig.MergeAPIResourceConfigs(apiextensionsapiserver.DefaultAPIResourceConfigSource(), nil, apiextensionsapiserver.Scheme) - if err != nil { - return nil, err - } - genericConfig.MergedResourceConfig = mergedResourceConfig - - genericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(apiextensionsopenapi.GetOpenAPIDefinitions), openapinamer.NewDefinitionNamer(apiextensionsapiserver.Scheme, apiextensionsapiserver.Scheme)) - - apiextensionsConfig := &apiextensionsapiserver.Config{ - GenericConfig: &genericapiserver.RecommendedConfig{ - Config: genericConfig, - SharedInformerFactory: fakeInfomers, - }, - ExtraConfig: apiextensionsapiserver.ExtraConfig{ - CRDRESTOptionsGetter: filestorage.NewRESTOptionsGetter(path.Join(dataPath, "grafana-apiextensionsserver"), crdEtcdOptions.StorageConfig), - // TODO: remove the hardcod when HA story is more developed - MasterCount: 1, - // TODO: leaving AuthResolverWrapper unset doesn't impact basic operation of CRDs - // AuthResolverWrapper: authResolverWrapper, - ServiceResolver: serviceResolver, - }, - } - - // we need to clear the poststarthooks so we don't add them multiple times to all the servers (that fails) - apiextensionsConfig.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - - return apiextensionsConfig, nil -} - -func initAggregatorConfig(options *options.RecommendedOptions, - sharedConfig *genericapiserver.RecommendedConfig, - extra *ExtraConfig, - fakeInformers informers.SharedInformerFactory, - builders []grafanaAPIServer.APIGroupBuilder, - serviceResolver apiserver.ServiceResolver, - dataPath string, -) (*aggregatorapiserver.Config, error) { - // make a shallow copy to let us twiddle a few things - // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the aggregator - genericConfig := sharedConfig.Config - - genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - genericConfig.RESTOptionsGetter = nil - // prevent generic API server from installing the OpenAPI handler. Aggregator server - // has its own customized OpenAPI handler. - genericConfig.SkipOpenAPIInstallation = true - mergedResourceConfig, err := resourceconfig.MergeAPIResourceConfigs(aggregatorapiserver.DefaultAPIResourceConfigSource(), nil, aggregatorscheme.Scheme) - if err != nil { - return nil, err - } - genericConfig.MergedResourceConfig = mergedResourceConfig - - getOpenAPIDefinitionsFunc := func() common.GetOpenAPIDefinitions { - return func(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { - return getMergedOpenAPIDefinitions(builders, ref) - } - } - - namer := openapinamer.NewDefinitionNamer(aggregatorscheme.Scheme, apiextensionsapiserver.Scheme) - genericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(getOpenAPIDefinitionsFunc(), namer) - genericConfig.OpenAPIV3Config.Info.Title = "Kubernetes" - genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(getOpenAPIDefinitionsFunc(), namer) - genericConfig.OpenAPIConfig.Info.Title = "Kubernetes" - - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) && - utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) { - // Add StorageVersionPrecondition handler to aggregator-apiserver. - // The handler will block write requests to built-in resources until the - // target resources' storage versions are up-to-date. - genericConfig.BuildHandlerChainFunc = genericapiserver.BuildHandlerChainWithStorageVersionPrecondition - } - - // copy the etcd options so we don't mutate originals. - // we assume that the etcd options have been completed already. avoid messing with anything outside - // of changes to StorageConfig as that may lead to unexpected behavior when the options are applied. - etcdOptions := *options.Etcd - etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion, - apiregistrationv1beta1.SchemeGroupVersion, - servicev0alpha1.SchemeGroupVersion) - etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1.SchemeGroupVersion, - schema.GroupKind{Group: apiregistrationv1beta1.GroupName}, - schema.GroupKind{Group: servicev0alpha1.GROUP}) - etcdOptions.SkipHealthEndpoints = true // avoid double wiring of health checks - if err := etcdOptions.ApplyTo(&genericConfig); err != nil { - return nil, err - } - genericConfig.RESTOptionsGetter = filestorage.NewRESTOptionsGetter(path.Join(dataPath, "grafana-aggregator"), etcdOptions.StorageConfig) - - genericConfig.DisabledPostStartHooks = genericConfig.DisabledPostStartHooks.Insert("apiservice-status-available-controller") - genericConfig.DisabledPostStartHooks = genericConfig.DisabledPostStartHooks.Insert("start-kube-aggregator-informers") - - aggregatorConfig := &aggregatorapiserver.Config{ - GenericConfig: &genericapiserver.RecommendedConfig{ - Config: genericConfig, - SharedInformerFactory: fakeInformers, - ClientConfig: genericConfig.LoopbackClientConfig, - }, - ExtraConfig: aggregatorapiserver.ExtraConfig{ - ProxyClientCertFile: extra.ProxyClientCertFile, - ProxyClientKeyFile: extra.ProxyClientKeyFile, - // NOTE: while ProxyTransport can be skipped in the configuration, it allows honoring - // DISABLE_HTTP2, HTTPS_PROXY and NO_PROXY env vars as needed - ProxyTransport: createProxyTransport(), - }, - } - - aggregatorConfig.ExtraConfig.ServiceResolver = serviceResolver - - // we need to clear the poststarthooks so we don't add them multiple times to all the servers (that fails) - aggregatorConfig.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - - return aggregatorConfig, nil -} - -func (o *AggregatorServerOptions) CreateAggregatorServer(delegateAPIServer genericapiserver.DelegationTarget, apiExtensionsInformers apiextensionsinformers.SharedInformerFactory) (*aggregatorapiserver.APIAggregator, error) { - completedConfig := o.Config.AggregatorComplete - aggregatorServer, err := completedConfig.NewWithDelegate(delegateAPIServer) - if err != nil { - return nil, err - } - - // create controllers for auto-registration - apiRegistrationClient, err := apiregistrationclient.NewForConfig(completedConfig.GenericConfig.LoopbackClientConfig) - if err != nil { - return nil, err - } - - autoRegistrationController := autoregister.NewAutoRegisterController(aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), apiRegistrationClient) - apiServices := apiServicesToRegister(delegateAPIServer, autoRegistrationController) - - crdRegistrationController := NewCRDRegistrationController( - apiExtensionsInformers.Apiextensions().V1().CustomResourceDefinitions(), - autoRegistrationController) - - // Imbue all builtin group-priorities onto the aggregated discovery - if completedConfig.GenericConfig.AggregatedDiscoveryGroupManager != nil { - for gv, entry := range apiVersionPriorities { - completedConfig.GenericConfig.AggregatedDiscoveryGroupManager.SetGroupVersionPriority(metav1.GroupVersion(gv), int(entry.group), int(entry.version)) - } - } - - err = aggregatorServer.GenericAPIServer.AddPostStartHook("kube-apiserver-autoregistration", func(context genericapiserver.PostStartHookContext) error { - go crdRegistrationController.Run(5, context.StopCh) - go func() { - crdRegistrationController.WaitForInitialSync() - autoRegistrationController.Run(5, context.StopCh) - }() - return nil - }) - if err != nil { - return nil, err - } - - err = aggregatorServer.GenericAPIServer.AddBootSequenceHealthChecks( - makeAPIServiceAvailableHealthCheck( - "autoregister-completion", - apiServices, - aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), - ), - ) - if err != nil { - return nil, err - } - - apiregistrationClient, err := apiregistrationclientset.NewForConfig(completedConfig.GenericConfig.LoopbackClientConfig) - if err != nil { - return nil, err - } - - availableController, err := NewAvailableConditionController( - aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), - o.sharedInformerFactory.Service().V0alpha1().ExternalNames(), - apiregistrationClient.ApiregistrationV1(), - nil, - (func() ([]byte, []byte))(nil), - completedConfig.ExtraConfig.ServiceResolver, - ) - if err != nil { - return nil, err - } - - aggregatorServer.GenericAPIServer.AddPostStartHookOrDie("apiservice-status-override-available-controller", func(context genericapiserver.PostStartHookContext) error { - // if we end up blocking for long periods of time, we may need to increase workers. - go availableController.Run(5, context.StopCh) - return nil - }) - - aggregatorServer.GenericAPIServer.AddPostStartHookOrDie("start-grafana-aggregator-informers", func(context genericapiserver.PostStartHookContext) error { - o.sharedInformerFactory.Start(context.StopCh) - aggregatorServer.APIRegistrationInformers.Start(context.StopCh) - return nil - }) - - // Install the API Group+version - for _, b := range o.Builders { - g, err := b.GetAPIGroupInfo(aggregatorscheme.Scheme, aggregatorscheme.Codecs, completedConfig.GenericConfig.RESTOptionsGetter) - if err != nil { - return nil, err - } - if g == nil || len(g.PrioritizedVersions) < 1 { - continue - } - err = aggregatorServer.GenericAPIServer.InstallAPIGroup(g) - if err != nil { - return nil, err - } - } - - return aggregatorServer, nil -} - -func makeAPIService(gv schema.GroupVersion) *v1.APIService { - apiServicePriority, ok := apiVersionPriorities[gv] - if !ok { - // if we aren't found, then we shouldn't register ourselves because it could result in a CRD group version - // being permanently stuck in the APIServices list. - klog.Infof("Skipping APIService creation for %v", gv) - return nil - } - return &v1.APIService{ - ObjectMeta: metav1.ObjectMeta{Name: gv.Version + "." + gv.Group}, - Spec: v1.APIServiceSpec{ - Group: gv.Group, - Version: gv.Version, - GroupPriorityMinimum: apiServicePriority.group, - VersionPriority: apiServicePriority.version, - }, - } -} - -// makeAPIServiceAvailableHealthCheck returns a healthz check that returns healthy -// once all of the specified services have been observed to be available at least once. -func makeAPIServiceAvailableHealthCheck(name string, apiServices []*v1.APIService, apiServiceInformer apiregistrationInformers.APIServiceInformer) healthz.HealthChecker { - // Track the auto-registered API services that have not been observed to be available yet - pendingServiceNamesLock := &sync.RWMutex{} - pendingServiceNames := sets.NewString() - for _, service := range apiServices { - pendingServiceNames.Insert(service.Name) - } - - // When an APIService in the list is seen as available, remove it from the pending list - handleAPIServiceChange := func(service *v1.APIService) { - pendingServiceNamesLock.Lock() - defer pendingServiceNamesLock.Unlock() - if !pendingServiceNames.Has(service.Name) { - return - } - if v1helper.IsAPIServiceConditionTrue(service, v1.Available) { - pendingServiceNames.Delete(service.Name) - } - } - - // Watch add/update events for APIServices - _, _ = apiServiceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { handleAPIServiceChange(obj.(*v1.APIService)) }, - UpdateFunc: func(old, new interface{}) { handleAPIServiceChange(new.(*v1.APIService)) }, - }) - - // Don't return healthy until the pending list is empty - return healthz.NamedCheck(name, func(r *http.Request) error { - pendingServiceNamesLock.RLock() - defer pendingServiceNamesLock.RUnlock() - if pendingServiceNames.Len() > 0 { - return fmt.Errorf("missing APIService: %v", pendingServiceNames.List()) - } - return nil - }) -} - -// priority defines group priority that is used in discovery. This controls -// group position in the kubectl output. -type priority struct { - // group indicates the order of the group relative to other groups. - group int32 - // version indicates the relative order of the version inside of its group. - version int32 -} - -// The proper way to resolve this letting the aggregator know the desired group and version-within-group order of the underlying servers -// is to refactor the genericapiserver.DelegationTarget to include a list of priorities based on which APIs were installed. -// This requires the APIGroupInfo struct to evolve and include the concept of priorities and to avoid mistakes, the core storage map there needs to be updated. -// That ripples out every bit as far as you'd expect, so for 1.7 we'll include the list here instead of being built up during storage. -var apiVersionPriorities = map[schema.GroupVersion]priority{ - {Group: "", Version: "v1"}: {group: 18000, version: 1}, - // to my knowledge, nothing below here collides - {Group: "admissionregistration.k8s.io", Version: "v1"}: {group: 16700, version: 15}, - {Group: "admissionregistration.k8s.io", Version: "v1beta1"}: {group: 16700, version: 12}, - {Group: "admissionregistration.k8s.io", Version: "v1alpha1"}: {group: 16700, version: 9}, - {Group: "apiextensions.k8s.io", Version: "v1"}: {group: 16700, version: 15}, - // Append a new group to the end of the list if unsure. - // You can use min(existing group)-100 as the initial value for a group. - // Version can be set to 9 (to have space around) for a new group. -} - -func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*v1.APIService { - apiServices := []*v1.APIService{} - - for _, curr := range delegateAPIServer.ListedPaths() { - if curr == "/api/v1" { - apiService := makeAPIService(schema.GroupVersion{Group: "", Version: "v1"}) - registration.AddAPIServiceToSyncOnStart(apiService) - apiServices = append(apiServices, apiService) - continue - } - - if !strings.HasPrefix(curr, "/apis/") { - continue - } - // this comes back in a list that looks like /apis/rbac.authorization.k8s.io/v1alpha1 - tokens := strings.Split(curr, "/") - if len(tokens) != 4 { - continue - } - - apiService := makeAPIService(schema.GroupVersion{Group: tokens[2], Version: tokens[3]}) - if apiService == nil { - continue - } - registration.AddAPIServiceToSyncOnStart(apiService) - apiServices = append(apiServices, apiService) - } - - return apiServices -} - -// NOTE: below function imported from https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/server.go#L197 -// createProxyTransport creates the dialer infrastructure to connect to the api servers. -func createProxyTransport() *http.Transport { - // NOTE: We don't set proxyDialerFn but the below SetTransportDefaults will - // See https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/util/net/http.go#L109 - var proxyDialerFn utilnet.DialFunc - // Proxying to services is IP-based... don't expect to be able to verify the hostname - proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} - proxyTransport := utilnet.SetTransportDefaults(&http.Transport{ - DialContext: proxyDialerFn, - TLSClientConfig: proxyTLSClientConfig, - }) - return proxyTransport -} diff --git a/pkg/aggregator/config.go b/pkg/aggregator/config.go deleted file mode 100644 index e7bcd84e286..00000000000 --- a/pkg/aggregator/config.go +++ /dev/null @@ -1,57 +0,0 @@ -package aggregator - -import ( - "github.com/spf13/pflag" - apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/options" - aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" -) - -type ExtraConfig struct { - ProxyClientCertFile string - ProxyClientKeyFile string - - DataPath string -} - -type Config struct { - Aggregator *aggregatorapiserver.Config - ApiExtensions *apiextensionsapiserver.Config - - AggregatorComplete aggregatorapiserver.CompletedConfig - ApiExtensionsComplete apiextensionsapiserver.CompletedConfig - - recommendedOptions *options.RecommendedOptions - SharedConfig *genericapiserver.RecommendedConfig - extraConfig *ExtraConfig -} - -func (c *Config) AddFlags(fs *pflag.FlagSet) { - if c == nil { - return - } - - c.recommendedOptions.AddFlags(fs) -} - -func (c *Config) Complete() { - if c == nil { - return - } - - c.ApiExtensionsComplete = c.ApiExtensions.Complete() - c.AggregatorComplete = c.Aggregator.Complete() -} - -func (ec *ExtraConfig) AddFlags(fs *pflag.FlagSet) { - if ec == nil { - return - } - - fs.StringVar(&ec.ProxyClientCertFile, "proxy-client-cert-file", ec.ProxyClientCertFile, - "path to proxy client cert file") - - fs.StringVar(&ec.ProxyClientKeyFile, "proxy-client-key-file", ec.ProxyClientKeyFile, - "path to proxy client cert file") -} diff --git a/pkg/aggregator/crdRegistrationController.go b/pkg/aggregator/crdRegistrationController.go deleted file mode 100644 index f3bbdd3c85d..00000000000 --- a/pkg/aggregator/crdRegistrationController.go +++ /dev/null @@ -1,214 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only -// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/pkg/controlplane/controller/crdregistration/crdregistration_controller.go -// Provenance-includes-license: Apache-2.0 -// Provenance-includes-copyright: The Kubernetes Authors. - -package aggregator - -import ( - "fmt" - "time" - - "k8s.io/klog/v2" - - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - crdinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1" - crdlisters "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/workqueue" - v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" -) - -// AutoAPIServiceRegistration is an interface which callers can re-declare locally and properly cast to for -// adding and removing APIServices -type AutoAPIServiceRegistration interface { - // AddAPIServiceToSync adds an API service to auto-register. - AddAPIServiceToSync(in *v1.APIService) - // RemoveAPIServiceToSync removes an API service to auto-register. - RemoveAPIServiceToSync(name string) -} - -type crdRegistrationController struct { - crdLister crdlisters.CustomResourceDefinitionLister - crdSynced cache.InformerSynced - - apiServiceRegistration AutoAPIServiceRegistration - - syncHandler func(groupVersion schema.GroupVersion) error - - syncedInitialSet chan struct{} - - // queue is where incoming work is placed to de-dup and to allow "easy" rate limited requeues on errors - // this is actually keyed by a groupVersion - queue workqueue.RateLimitingInterface -} - -// NewCRDRegistrationController returns a controller which will register CRD GroupVersions with the auto APIService registration -// controller so they automatically stay in sync. -func NewCRDRegistrationController(crdinformer crdinformers.CustomResourceDefinitionInformer, apiServiceRegistration AutoAPIServiceRegistration) *crdRegistrationController { - c := &crdRegistrationController{ - crdLister: crdinformer.Lister(), - crdSynced: crdinformer.Informer().HasSynced, - apiServiceRegistration: apiServiceRegistration, - syncedInitialSet: make(chan struct{}), - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_autoregistration_controller"), - } - c.syncHandler = c.handleVersionUpdate - - _, _ = crdinformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - cast := obj.(*apiextensionsv1.CustomResourceDefinition) - c.enqueueCRD(cast) - }, - UpdateFunc: func(oldObj, newObj interface{}) { - // Enqueue both old and new object to make sure we remove and add appropriate API services. - // The working queue will resolve any duplicates and only changes will stay in the queue. - c.enqueueCRD(oldObj.(*apiextensionsv1.CustomResourceDefinition)) - c.enqueueCRD(newObj.(*apiextensionsv1.CustomResourceDefinition)) - }, - DeleteFunc: func(obj interface{}) { - cast, ok := obj.(*apiextensionsv1.CustomResourceDefinition) - if !ok { - tombstone, ok := obj.(cache.DeletedFinalStateUnknown) - if !ok { - klog.V(2).Infof("Couldn't get object from tombstone %#v", obj) - return - } - cast, ok = tombstone.Obj.(*apiextensionsv1.CustomResourceDefinition) - if !ok { - klog.V(2).Infof("Tombstone contained unexpected object: %#v", obj) - return - } - } - c.enqueueCRD(cast) - }, - }) - - return c -} - -func (c *crdRegistrationController) Run(workers int, stopCh <-chan struct{}) { - defer utilruntime.HandleCrash() - // make sure the work queue is shutdown which will trigger workers to end - defer c.queue.ShutDown() - - klog.Infof("Starting crd-autoregister controller") - defer klog.Infof("Shutting down crd-autoregister controller") - - // wait for your secondary caches to fill before starting your work - if !cache.WaitForNamedCacheSync("crd-autoregister", stopCh, c.crdSynced) { - return - } - - // process each item in the list once - if crds, err := c.crdLister.List(labels.Everything()); err != nil { - utilruntime.HandleError(err) - } else { - for _, crd := range crds { - for _, version := range crd.Spec.Versions { - if err := c.syncHandler(schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name}); err != nil { - utilruntime.HandleError(err) - } - } - } - } - close(c.syncedInitialSet) - - // start up your worker threads based on workers. Some controllers have multiple kinds of workers - for i := 0; i < workers; i++ { - // runWorker will loop until "something bad" happens. The .Until will then rekick the worker - // after one second - go wait.Until(c.runWorker, time.Second, stopCh) - } - - // wait until we're told to stop - <-stopCh -} - -// WaitForInitialSync blocks until the initial set of CRD resources has been processed -func (c *crdRegistrationController) WaitForInitialSync() { - <-c.syncedInitialSet -} - -func (c *crdRegistrationController) runWorker() { - // hot loop until we're told to stop. processNextWorkItem will automatically wait until there's work - // available, so we don't worry about secondary waits - for c.processNextWorkItem() { - } -} - -// processNextWorkItem deals with one key off the queue. It returns false when it's time to quit. -func (c *crdRegistrationController) processNextWorkItem() bool { - // pull the next work item from queue. It should be a key we use to lookup something in a cache - key, quit := c.queue.Get() - if quit { - return false - } - // you always have to indicate to the queue that you've completed a piece of work - defer c.queue.Done(key) - - // do your work on the key. This method will contains your "do stuff" logic - err := c.syncHandler(key.(schema.GroupVersion)) - if err == nil { - // if you had no error, tell the queue to stop tracking history for your key. This will - // reset things like failure counts for per-item rate limiting - c.queue.Forget(key) - return true - } - - // there was a failure so be sure to report it. This method allows for pluggable error handling - // which can be used for things like cluster-monitoring - utilruntime.HandleError(fmt.Errorf("%v failed with : %v", key, err)) - // since we failed, we should requeue the item to work on later. This method will add a backoff - // to avoid hotlooping on particular items (they're probably still not going to work right away) - // and overall controller protection (everything I've done is broken, this controller needs to - // calm down or it can starve other useful work) cases. - c.queue.AddRateLimited(key) - - return true -} - -func (c *crdRegistrationController) enqueueCRD(crd *apiextensionsv1.CustomResourceDefinition) { - for _, version := range crd.Spec.Versions { - c.queue.Add(schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name}) - } -} - -func (c *crdRegistrationController) handleVersionUpdate(groupVersion schema.GroupVersion) error { - apiServiceName := groupVersion.Version + "." + groupVersion.Group - - // check all CRDs. There shouldn't that many, but if we have problems later we can index them - crds, err := c.crdLister.List(labels.Everything()) - if err != nil { - return err - } - for _, crd := range crds { - if crd.Spec.Group != groupVersion.Group { - continue - } - for _, version := range crd.Spec.Versions { - if version.Name != groupVersion.Version || !version.Served { - continue - } - - c.apiServiceRegistration.AddAPIServiceToSync(&v1.APIService{ - ObjectMeta: metav1.ObjectMeta{Name: apiServiceName}, - Spec: v1.APIServiceSpec{ - Group: groupVersion.Group, - Version: groupVersion.Version, - GroupPriorityMinimum: 1000, // CRDs should have relatively low priority - VersionPriority: 100, // CRDs will be sorted by kube-like versions like any other APIService with the same VersionPriority - }, - }) - return nil - } - } - - c.apiServiceRegistration.RemoveAPIServiceToSync(apiServiceName) - return nil -} diff --git a/pkg/api/frontendsettings_test.go b/pkg/api/frontendsettings_test.go index 53a3a322377..faaf564c4da 100644 --- a/pkg/api/frontendsettings_test.go +++ b/pkg/api/frontendsettings_test.go @@ -19,8 +19,8 @@ import ( "github.com/grafana/grafana/pkg/plugins/config" "github.com/grafana/grafana/pkg/plugins/pluginscdn" accesscontrolmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/licensing" "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginsettings" "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore" diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index a6895f7a6ce..c9a4536f2a1 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -25,8 +25,8 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/grafana/grafana/pkg/services/anonymous" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + grafanaapiserver "github.com/grafana/grafana/pkg/services/apiserver" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/api/avatar" "github.com/grafana/grafana/pkg/api/routing" diff --git a/pkg/api/metrics.go b/pkg/api/metrics.go index 4bbe662c081..df798af8ca8 100644 --- a/pkg/api/metrics.go +++ b/pkg/api/metrics.go @@ -13,10 +13,10 @@ import ( "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/middleware/requestmeta" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/util/errutil/errhttp" "github.com/grafana/grafana/pkg/web" ) diff --git a/pkg/api/playlist.go b/pkg/api/playlist.go index 4a49bc59708..9f8b2acb85f 100644 --- a/pkg/api/playlist.go +++ b/pkg/api/playlist.go @@ -15,10 +15,10 @@ import ( "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" "github.com/grafana/grafana/pkg/middleware" internalplaylist "github.com/grafana/grafana/pkg/registry/apis/playlist" + grafanaapiserver "github.com/grafana/grafana/pkg/services/apiserver" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/playlist" "github.com/grafana/grafana/pkg/util/errutil/errhttp" "github.com/grafana/grafana/pkg/web" diff --git a/pkg/cmd/grafana/apiserver/aggregator.md b/pkg/cmd/grafana/apiserver/aggregator.md deleted file mode 100644 index 0c8ae3fae9a..00000000000 --- a/pkg/cmd/grafana/apiserver/aggregator.md +++ /dev/null @@ -1,51 +0,0 @@ -# grafana aggregator - -The `aggregator` command in this binary is our equivalent of what kube-apiserver does for aggregation using -the `kube-aggregator` pkg. Here, we enable only select controllers that are useful for aggregation in a Grafana -cloud context. In future, Grafana microservices (and even plugins) will run as separate API servers -hosting each their own APIs (with specific Group/Versions). The `aggregator` component here shall act similar to what -`kube-apiserver` does: doing healthchecks for `APIService` objects registered against it and acting as a proxy for -the specified `GroupVersion` therein. - -## How to get started - -1. Generate the PKI using `openssl` (for development purposes, we will use the CN of `system:masters`): - ```shell - ./hack/make-aggregator-pki.sh - ``` -2. Start the aggregator: - ```shell - # This will generate the kubeconfig which you can use in the extension apiservers for - # enforcing delegate authnz under $PWD/data/grafana-apiserver/aggregator.kubeconfig - go run ./pkg/cmd/grafana aggregator --secure-port 8443 \ - --proxy-client-cert-file $PWD/data/grafana-aggregator/client.crt \ - --proxy-client-key-file $PWD/data/grafana-aggregator/client.key - ``` -3. Apply the manifests: - ```shell - export KUBECONFIG=$PWD/data/grafana-apiserver/aggregator.kubeconfig - kubectl apply -k ./pkg/cmd/grafana/apiserver/deploy/aggregator-test - # SAMPLE OUTPUT - # apiservice.apiregistration.k8s.io/v0alpha1.example.grafana.app created - # externalname.service.grafana.app/example-apiserver created - - kubectl get apiservice - # SAMPLE OUTPUT - # NAME SERVICE AVAILABLE AGE - # v0alpha1.example.grafana.app grafana/example-apiserver False (FailedDiscoveryCheck) 29m - ``` -4. In another tab, start the example microservice that will be aggregated by the parent apiserver: - ```shell - go run ./pkg/cmd/grafana apiserver example.grafana.app \ - --kubeconfig $PWD/data/grafana-aggregator/aggregator.kubeconfig \ - --secure-port 7443 \ - --client-ca-file=$PWD/data/grafana-aggregator/ca.crt - ``` -5. Check `APIService` again: - ```shell - export KUBECONFIG=$PWD/data/grafana-apiserver/aggregator.kubeconfig - kubectl get apiservice - # SAMPLE OUTPUT - # NAME SERVICE AVAILABLE AGE - # v0alpha1.example.grafana.app grafana/example-apiserver True 30m - ``` diff --git a/pkg/cmd/grafana/apiserver/cmd.go b/pkg/cmd/grafana/apiserver/cmd.go index 338c1ca9662..d24829dba19 100644 --- a/pkg/cmd/grafana/apiserver/cmd.go +++ b/pkg/cmd/grafana/apiserver/cmd.go @@ -2,26 +2,13 @@ package apiserver import ( "os" - "path" "github.com/spf13/cobra" - genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/options" - "k8s.io/apiserver/pkg/util/notfoundhandler" - "k8s.io/client-go/tools/clientcmd" "k8s.io/component-base/cli" - "k8s.io/klog/v2" - aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" - "github.com/grafana/grafana/pkg/aggregator" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" -) - -const ( - aggregatorDataPath = "data" - defaultAggregatorEtcdPathPrefix = "/registry/grafana.aggregator" + grafanaapiserver "github.com/grafana/grafana/pkg/services/apiserver" ) func newCommandStartExampleAPIServer(o *APIServerOptions, stopCh <-chan struct{}) *cobra.Command { @@ -74,97 +61,3 @@ func RunCLI() int { return cli.Run(cmd) } - -func newCommandStartAggregator() *cobra.Command { - devAcknowledgementNotice := "The aggregator command is in heavy development. The entire setup is subject to change without notice" - - cwd, err := os.Getwd() - if err != nil { - panic("could not determine current directory") - } - - extraConfig := &aggregator.ExtraConfig{ - DataPath: path.Join(cwd, aggregatorDataPath), - } - - // Register standard k8s flags with the command line - recommendedOptions := options.NewRecommendedOptions( - defaultAggregatorEtcdPathPrefix, - aggregatorscheme.Codecs.LegacyCodec(), // codec is passed to etcd and hence not used - ) - - cmd := &cobra.Command{ - Use: "aggregator", - Short: "Run the grafana aggregator", - Long: "Run a standalone kubernetes based aggregator server. " + - devAcknowledgementNotice, - Example: "grafana aggregator", - RunE: func(c *cobra.Command, args []string) error { - serverOptions, err := aggregator.NewAggregatorServerOptions(os.Stdout, os.Stderr, recommendedOptions, extraConfig) - serverOptions.Config.Complete() - - if err != nil { - klog.Errorf("Could not create aggregator server options: %s", err) - os.Exit(1) - } - - return run(serverOptions) - }, - } - - recommendedOptions.AddFlags(cmd.Flags()) - extraConfig.AddFlags(cmd.Flags()) - - return cmd -} - -func run(serverOptions *aggregator.AggregatorServerOptions) error { - if err := serverOptions.LoadAPIGroupBuilders(); err != nil { - klog.Errorf("Error loading prerequisite APIs: %s", err) - return err - } - - notFoundHandler := notfoundhandler.New(serverOptions.Config.SharedConfig.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey) - apiExtensionsServer, err := serverOptions.Config.ApiExtensionsComplete.New(genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler)) - if err != nil { - return err - } - - aggregator, err := serverOptions.CreateAggregatorServer(apiExtensionsServer.GenericAPIServer, apiExtensionsServer.Informers) - if err != nil { - klog.Errorf("Error creating aggregator server: %s", err) - return err - } - - // Install the API Group+version - err = grafanaapiserver.InstallAPIs(aggregator.GenericAPIServer, serverOptions.Config.Aggregator.GenericConfig.RESTOptionsGetter, serverOptions.Builders) - if err != nil { - klog.Errorf("Error installing apis: %s", err) - return err - } - - if err := clientcmd.WriteToFile( - utils.FormatKubeConfig(aggregator.GenericAPIServer.LoopbackClientConfig), - path.Join(aggregatorDataPath, "grafana-aggregator", "aggregator.kubeconfig"), - ); err != nil { - klog.Errorf("Error persisting aggregator.kubeconfig: %s", err) - return err - } - - prepared, err := aggregator.PrepareRun() - if err != nil { - return err - } - - stopCh := genericapiserver.SetupSignalHandler() - if err := prepared.Run(stopCh); err != nil { - return err - } - return nil -} - -func RunCobraWrapper() int { - cmd := newCommandStartAggregator() - - return cli.Run(cmd) -} diff --git a/pkg/cmd/grafana/apiserver/server.go b/pkg/cmd/grafana/apiserver/server.go index d40f89d615b..a8f5fffb3e7 100644 --- a/pkg/cmd/grafana/apiserver/server.go +++ b/pkg/cmd/grafana/apiserver/server.go @@ -17,9 +17,10 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/query" "github.com/grafana/grafana/pkg/registry/apis/query/runner" "github.com/grafana/grafana/pkg/server" + grafanaAPIServer "github.com/grafana/grafana/pkg/services/apiserver" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaAPIServer "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/setting" ) @@ -30,7 +31,7 @@ const ( // APIServerOptions contains the state for the apiserver type APIServerOptions struct { - builders []grafanaAPIServer.APIGroupBuilder + builders []builder.APIGroupBuilder RecommendedOptions *options.RecommendedOptions AlternateDNS []string @@ -46,7 +47,7 @@ func newAPIServerOptions(out, errOut io.Writer) *APIServerOptions { } func (o *APIServerOptions) loadAPIGroupBuilders(args []string) error { - o.builders = []grafanaAPIServer.APIGroupBuilder{} + o.builders = []builder.APIGroupBuilder{} for _, g := range args { switch g { // No dependencies for testing @@ -171,7 +172,7 @@ func (o *APIServerOptions) Config() (*genericapiserver.RecommendedConfig, error) serverConfig.DisabledPostStartHooks = serverConfig.DisabledPostStartHooks.Insert("priority-and-fairness-config-consumer") // Add OpenAPI specs for each group+version - err := grafanaAPIServer.SetupConfig(serverConfig, o.builders) + err := builder.SetupConfig(grafanaAPIServer.Scheme, serverConfig, o.builders) return serverConfig, err } @@ -199,7 +200,7 @@ func (o *APIServerOptions) RunAPIServer(config *genericapiserver.RecommendedConf } // Install the API Group+version - err = grafanaAPIServer.InstallAPIs(server, config.RESTOptionsGetter, o.builders) + err = builder.InstallAPIs(grafanaAPIServer.Scheme, grafanaAPIServer.Codecs, server, config.RESTOptionsGetter, o.builders, true) if err != nil { return err } diff --git a/pkg/cmd/grafana/main.go b/pkg/cmd/grafana/main.go index d86e66c3a21..696198e40b9 100644 --- a/pkg/cmd/grafana/main.go +++ b/pkg/cmd/grafana/main.go @@ -46,18 +46,6 @@ func main() { }, }, gsrv.ServerCommand(version, commit, enterpriseCommit, buildBranch, buildstamp), - { - // The kube-aggregator inspired grafana aggregator - Name: "aggregator", - Usage: "run grafana aggregator (experimental)", - // Skip parsing flags because the command line is actually managed by cobra - SkipFlagParsing: true, - Action: func(context *cli.Context) error { - // exit here because apiserver handles its own error output - os.Exit(apiserver.RunCobraWrapper()) - return nil - }, - }, }, CommandNotFound: cmdNotFound, EnableBashCompletion: true, diff --git a/pkg/registry/apis/apis.go b/pkg/registry/apis/apis.go index f83ac018bda..86f757eb8ba 100644 --- a/pkg/registry/apis/apis.go +++ b/pkg/registry/apis/apis.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/folders" "github.com/grafana/grafana/pkg/registry/apis/playlist" "github.com/grafana/grafana/pkg/registry/apis/query" + "github.com/grafana/grafana/pkg/registry/apis/service" ) var ( @@ -28,6 +29,7 @@ func ProvideRegistryServiceSink( _ *featuretoggle.FeatureFlagAPIBuilder, _ *datasource.DataSourceAPIBuilder, _ *folders.FolderAPIBuilder, + _ *service.ServiceAPIBuilder, _ *query.QueryAPIBuilder, ) *Service { return &Service{} diff --git a/pkg/registry/apis/dashboard/access/sql_dashboards.go b/pkg/registry/apis/dashboard/access/sql_dashboards.go index b0e8304fa6f..4064a1fa7a3 100644 --- a/pkg/registry/apis/dashboard/access/sql_dashboards.go +++ b/pkg/registry/apis/dashboard/access/sql_dashboards.go @@ -14,9 +14,9 @@ import ( "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/dashboards" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/services/sqlstore/session" ) diff --git a/pkg/registry/apis/dashboard/authorizer.go b/pkg/registry/apis/dashboard/authorizer.go index 9aa8d2b0f59..54a772591d5 100644 --- a/pkg/registry/apis/dashboard/authorizer.go +++ b/pkg/registry/apis/dashboard/authorizer.go @@ -7,8 +7,8 @@ import ( "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/dashboards" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/guardian" ) diff --git a/pkg/registry/apis/dashboard/legacy_storage.go b/pkg/registry/apis/dashboard/legacy_storage.go index 73062163836..e5a3a5ede65 100644 --- a/pkg/registry/apis/dashboard/legacy_storage.go +++ b/pkg/registry/apis/dashboard/legacy_storage.go @@ -14,7 +14,7 @@ import ( common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/dashboard/access" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" ) var ( diff --git a/pkg/registry/apis/dashboard/register.go b/pkg/registry/apis/dashboard/register.go index 7acf642e0d1..e6434240b15 100644 --- a/pkg/registry/apis/dashboard/register.go +++ b/pkg/registry/apis/dashboard/register.go @@ -19,19 +19,19 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/registry/apis/dashboard/access" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic" + grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/dashboards" dashver "github.com/grafana/grafana/pkg/services/dashboardversion" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic" - grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/setting" ) -var _ grafanaapiserver.APIGroupBuilder = (*DashboardsAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*DashboardsAPIBuilder)(nil) // This is used just so wire has something unique to return type DashboardsAPIBuilder struct { @@ -47,7 +47,7 @@ type DashboardsAPIBuilder struct { } func RegisterAPIService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, - apiregistration grafanaapiserver.APIRegistrar, + apiregistration builder.APIRegistrar, dashboardService dashboards.DashboardService, dashboardVersionService dashver.Service, accessControl accesscontrol.AccessControl, @@ -113,6 +113,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, + dualWrite bool, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs) @@ -171,7 +172,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo( } // Dual writes if a RESTOptionsGetter is provided - if optsGetter != nil { + if dualWrite && optsGetter != nil { options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: grafanaregistry.GetAttrs} if err := store.CompleteWithOptions(options); err != nil { return nil, err @@ -195,6 +196,6 @@ func (b *DashboardsAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefiniti return v0alpha1.GetOpenAPIDefinitions } -func (b *DashboardsAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *DashboardsAPIBuilder) GetAPIRoutes() *builder.APIRoutes { return nil // no custom API routes } diff --git a/pkg/registry/apis/dashboard/sub_access.go b/pkg/registry/apis/dashboard/sub_access.go index c184cc8adce..ca2c783b5e2 100644 --- a/pkg/registry/apis/dashboard/sub_access.go +++ b/pkg/registry/apis/dashboard/sub_access.go @@ -11,9 +11,9 @@ import ( "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/auth/identity" dashboardssvc "github.com/grafana/grafana/pkg/services/dashboards" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/guardian" ) diff --git a/pkg/registry/apis/dashboard/sub_versions.go b/pkg/registry/apis/dashboard/sub_versions.go index ca4c06caba1..a38f9947d84 100644 --- a/pkg/registry/apis/dashboard/sub_versions.go +++ b/pkg/registry/apis/dashboard/sub_versions.go @@ -13,8 +13,8 @@ import ( common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" dashver "github.com/grafana/grafana/pkg/services/dashboardversion" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" ) type VersionsREST struct { diff --git a/pkg/registry/apis/dashboard/summary_storage.go b/pkg/registry/apis/dashboard/summary_storage.go index ef9cf3578ec..17311be1372 100644 --- a/pkg/registry/apis/dashboard/summary_storage.go +++ b/pkg/registry/apis/dashboard/summary_storage.go @@ -10,7 +10,7 @@ import ( common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/dashboard/access" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" ) var ( diff --git a/pkg/registry/apis/datasource/plugincontext.go b/pkg/registry/apis/datasource/plugincontext.go index d4c9e01f5b3..1c39aaa9e26 100644 --- a/pkg/registry/apis/datasource/plugincontext.go +++ b/pkg/registry/apis/datasource/plugincontext.go @@ -9,9 +9,9 @@ import ( "github.com/grafana/grafana/pkg/apis/datasource/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/datasources" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/services/pluginsintegration/plugincontext" ) diff --git a/pkg/registry/apis/datasource/querier.go b/pkg/registry/apis/datasource/querier.go new file mode 100644 index 00000000000..9d5b9ff4702 --- /dev/null +++ b/pkg/registry/apis/datasource/querier.go @@ -0,0 +1,148 @@ +package datasource + +import ( + "context" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" + "github.com/grafana/grafana/pkg/apis/datasource/v0alpha1" + "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/datasources" +) + +type QuerierFactoryFunc func(ctx context.Context, ri common.ResourceInfo, pj plugins.JSONData) (Querier, error) + +type QuerierProvider interface { + Querier(ctx context.Context, ri common.ResourceInfo, pj plugins.JSONData) (Querier, error) +} + +type DefaultQuerierProvider struct { + factory QuerierFactoryFunc +} + +func ProvideDefaultQuerierProvider(pluginClient plugins.Client, dsService datasources.DataSourceService, + dsCache datasources.CacheService) *DefaultQuerierProvider { + return NewQuerierProvider(func(ctx context.Context, ri common.ResourceInfo, pj plugins.JSONData) (Querier, error) { + return NewDefaultQuerier(ri, pj, pluginClient, dsService, dsCache), nil + }) +} + +func NewQuerierProvider(factory QuerierFactoryFunc) *DefaultQuerierProvider { + return &DefaultQuerierProvider{ + factory: factory, + } +} + +func (p *DefaultQuerierProvider) Querier(ctx context.Context, ri common.ResourceInfo, pj plugins.JSONData) (Querier, error) { + return p.factory(ctx, ri, pj) +} + +// Querier is the interface that wraps the Query method. +type Querier interface { + // Query runs the query on behalf of the user in context. + Query(ctx context.Context, query *backend.QueryDataRequest) (*backend.QueryDataResponse, error) + // Health checks the health of the plugin. + Health(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) + // Resource gets a resource plugin. + Resource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error + // Datasource gets all data source plugins (with elevated permissions). + Datasource(ctx context.Context, name string) (*v0alpha1.DataSourceConnection, error) + // Datasources lists all data sources (with elevated permissions). + Datasources(ctx context.Context) (*v0alpha1.DataSourceConnectionList, error) +} + +type DefaultQuerier struct { + connectionResourceInfo common.ResourceInfo + pluginJSON plugins.JSONData + pluginClient plugins.Client + dsService datasources.DataSourceService + dsCache datasources.CacheService +} + +func NewDefaultQuerier( + connectionResourceInfo common.ResourceInfo, + pluginJSON plugins.JSONData, + pluginClient plugins.Client, + dsService datasources.DataSourceService, + dsCache datasources.CacheService, +) *DefaultQuerier { + return &DefaultQuerier{ + connectionResourceInfo: connectionResourceInfo, + pluginJSON: pluginJSON, + pluginClient: pluginClient, + dsService: dsService, + dsCache: dsCache, + } +} + +func (q *DefaultQuerier) Query(ctx context.Context, query *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { + _, err := request.NamespaceInfoFrom(ctx, true) + if err != nil { + return nil, err + } + return q.pluginClient.QueryData(ctx, query) +} + +func (q *DefaultQuerier) Resource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { + _, err := request.NamespaceInfoFrom(ctx, true) + if err != nil { + return err + } + return q.pluginClient.CallResource(ctx, req, sender) +} + +func (q *DefaultQuerier) Health(ctx context.Context, query *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { + _, err := request.NamespaceInfoFrom(ctx, true) + if err != nil { + return nil, err + } + return q.pluginClient.CheckHealth(ctx, query) +} + +func (q *DefaultQuerier) Datasource(ctx context.Context, name string) (*v0alpha1.DataSourceConnection, error) { + info, err := request.NamespaceInfoFrom(ctx, true) + if err != nil { + return nil, err + } + user, err := appcontext.User(ctx) + if err != nil { + return nil, err + } + ds, err := q.dsCache.GetDatasourceByUID(ctx, name, user, false) + if err != nil { + return nil, err + } + return asConnection(ds, info.Value) +} + +func (q *DefaultQuerier) Datasources(ctx context.Context) (*v0alpha1.DataSourceConnectionList, error) { + info, err := request.NamespaceInfoFrom(ctx, true) + if err != nil { + return nil, err + } + + ds, err := q.dsService.GetDataSourcesByType(ctx, &datasources.GetDataSourcesByTypeQuery{ + OrgID: info.OrgID, + Type: q.pluginJSON.ID, + }) + if err != nil { + return nil, err + } + return asConnectionList(q.connectionResourceInfo.TypeMeta(), ds, info.Value) +} + +func asConnectionList(typeMeta metav1.TypeMeta, dss []*datasources.DataSource, ns string) (*v0alpha1.DataSourceConnectionList, error) { + result := &v0alpha1.DataSourceConnectionList{ + Items: []v0alpha1.DataSourceConnection{}, + } + for _, ds := range dss { + v, _ := asConnection(ds, ns) + result.Items = append(result.Items, *v) + } + + return result, nil +} diff --git a/pkg/registry/apis/datasource/register.go b/pkg/registry/apis/datasource/register.go index 9fc43b7bc38..b5069a13cef 100644 --- a/pkg/registry/apis/datasource/register.go +++ b/pkg/registry/apis/datasource/register.go @@ -23,13 +23,13 @@ import ( query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore" ) -var _ grafanaapiserver.APIGroupBuilder = (*DataSourceAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*DataSourceAPIBuilder)(nil) // DataSourceAPIBuilder is used just so wire has something unique to return type DataSourceAPIBuilder struct { @@ -44,7 +44,7 @@ type DataSourceAPIBuilder struct { func RegisterAPIService( features featuremgmt.FeatureToggles, - apiRegistrar grafanaapiserver.APIRegistrar, + apiRegistrar builder.APIRegistrar, pluginClient plugins.Client, // access to everything datasources PluginDatasourceProvider, contextProvider PluginContextWrapper, @@ -151,6 +151,7 @@ func (b *DataSourceAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? _ generic.RESTOptionsGetter, + _ bool, ) (*genericapiserver.APIGroupInfo, error) { storage := map[string]rest.Storage{} @@ -213,6 +214,6 @@ func (b *DataSourceAPIBuilder) GetOpenAPIDefinitions() openapi.GetOpenAPIDefinit } // Register additional routes with the server -func (b *DataSourceAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *DataSourceAPIBuilder) GetAPIRoutes() *builder.APIRoutes { return nil } diff --git a/pkg/registry/apis/datasource/standalone.go b/pkg/registry/apis/datasource/standalone.go index 34324bd11b8..fed03c18e13 100644 --- a/pkg/registry/apis/datasource/standalone.go +++ b/pkg/registry/apis/datasource/standalone.go @@ -10,8 +10,8 @@ import ( "github.com/grafana/grafana/pkg/apis/datasource/v0alpha1" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/accesscontrol/actest" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/setting" testdatasource "github.com/grafana/grafana/pkg/tsdb/grafana-testdata-datasource" ) diff --git a/pkg/registry/apis/example/dummy_storage.go b/pkg/registry/apis/example/dummy_storage.go index 290623771cb..ac4f823dd13 100644 --- a/pkg/registry/apis/example/dummy_storage.go +++ b/pkg/registry/apis/example/dummy_storage.go @@ -13,8 +13,8 @@ import ( "k8s.io/apiserver/pkg/registry/rest" example "github.com/grafana/grafana/pkg/apis/example/v0alpha1" - grafanarequest "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic" + grafanarequest "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic" ) var ( diff --git a/pkg/registry/apis/example/register.go b/pkg/registry/apis/example/register.go index bd1277615a1..69a4add5d58 100644 --- a/pkg/registry/apis/example/register.go +++ b/pkg/registry/apis/example/register.go @@ -22,11 +22,11 @@ import ( example "github.com/grafana/grafana/pkg/apis/example/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" ) -var _ grafanaapiserver.APIGroupBuilder = (*TestingAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*TestingAPIBuilder)(nil) // This is used just so wire has something unique to return type TestingAPIBuilder struct { @@ -40,7 +40,7 @@ func NewTestingAPIBuilder() *TestingAPIBuilder { } } -func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration grafanaapiserver.APIRegistrar) *TestingAPIBuilder { +func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *TestingAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { return nil // skip registration unless opting into experimental apis } @@ -84,7 +84,8 @@ func (b *TestingAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { func (b *TestingAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? - optsGetter generic.RESTOptionsGetter, + _ generic.RESTOptionsGetter, + _ bool, ) (*genericapiserver.APIGroupInfo, error) { b.codecs = codecs apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(b.gv.Group, scheme, metav1.ParameterCodec, codecs) @@ -102,9 +103,9 @@ func (b *TestingAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions } // Register additional routes with the server -func (b *TestingAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { - return &grafanaapiserver.APIRoutes{ - Root: []grafanaapiserver.APIRouteHandler{ +func (b *TestingAPIBuilder) GetAPIRoutes() *builder.APIRoutes { + return &builder.APIRoutes{ + Root: []builder.APIRouteHandler{ { Path: "aaa", Spec: &spec3.PathProps{ @@ -166,7 +167,7 @@ func (b *TestingAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { }, }, }, - Namespace: []grafanaapiserver.APIRouteHandler{ + Namespace: []builder.APIRouteHandler{ { Path: "ccc", Spec: &spec3.PathProps{ diff --git a/pkg/registry/apis/example/storage.go b/pkg/registry/apis/example/storage.go index fbec2ad7316..e2e198fa0be 100644 --- a/pkg/registry/apis/example/storage.go +++ b/pkg/registry/apis/example/storage.go @@ -12,7 +12,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" example "github.com/grafana/grafana/pkg/apis/example/v0alpha1" - grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic" + grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic" "github.com/grafana/grafana/pkg/setting" ) diff --git a/pkg/registry/apis/example/subresource.go b/pkg/registry/apis/example/subresource.go index d7d41186b54..63fdcab5f02 100644 --- a/pkg/registry/apis/example/subresource.go +++ b/pkg/registry/apis/example/subresource.go @@ -10,7 +10,7 @@ import ( example "github.com/grafana/grafana/pkg/apis/example/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" ) type dummySubresourceREST struct{} diff --git a/pkg/registry/apis/featuretoggle/features.go b/pkg/registry/apis/featuretoggle/features.go index b001d51d222..23808272b73 100644 --- a/pkg/registry/apis/featuretoggle/features.go +++ b/pkg/registry/apis/featuretoggle/features.go @@ -12,8 +12,8 @@ import ( common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/apis/featuretoggle/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" ) var ( diff --git a/pkg/registry/apis/featuretoggle/register.go b/pkg/registry/apis/featuretoggle/register.go index 1a77fba7a3a..080bb42f20e 100644 --- a/pkg/registry/apis/featuretoggle/register.go +++ b/pkg/registry/apis/featuretoggle/register.go @@ -14,11 +14,11 @@ import ( "k8s.io/kube-openapi/pkg/validation/spec" "github.com/grafana/grafana/pkg/apis/featuretoggle/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" ) -var _ grafanaapiserver.APIGroupBuilder = (*FeatureFlagAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*FeatureFlagAPIBuilder)(nil) var gv = v0alpha1.SchemeGroupVersion @@ -32,7 +32,7 @@ func NewFeatureFlagAPIBuilder(features *featuremgmt.FeatureManager) *FeatureFlag } func RegisterAPIService(features *featuremgmt.FeatureManager, - apiregistration grafanaapiserver.APIRegistrar, + apiregistration builder.APIRegistrar, ) *FeatureFlagAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { return nil // skip registration unless opting into experimental apis @@ -78,7 +78,8 @@ func (b *FeatureFlagAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { func (b *FeatureFlagAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? - optsGetter generic.RESTOptionsGetter, + _ generic.RESTOptionsGetter, + _ bool, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs) @@ -102,13 +103,13 @@ func (b *FeatureFlagAPIBuilder) GetAuthorizer() authorizer.Authorizer { } // Register additional routes with the server -func (b *FeatureFlagAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *FeatureFlagAPIBuilder) GetAPIRoutes() *builder.APIRoutes { defs := v0alpha1.GetOpenAPIDefinitions(func(path string) spec.Ref { return spec.Ref{} }) stateSchema := defs["github.com/grafana/grafana/pkg/apis/featuretoggle/v0alpha1.ResolvedToggleState"].Schema tags := []string{"Editor"} - return &grafanaapiserver.APIRoutes{ - Root: []grafanaapiserver.APIRouteHandler{ + return &builder.APIRoutes{ + Root: []builder.APIRouteHandler{ { Path: "current", Spec: &spec3.PathProps{ diff --git a/pkg/registry/apis/featuretoggle/toggles.go b/pkg/registry/apis/featuretoggle/toggles.go index b4671104c4a..81f5c092a3e 100644 --- a/pkg/registry/apis/featuretoggle/toggles.go +++ b/pkg/registry/apis/featuretoggle/toggles.go @@ -11,8 +11,8 @@ import ( common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/apis/featuretoggle/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" ) var ( diff --git a/pkg/registry/apis/folders/conversions.go b/pkg/registry/apis/folders/conversions.go index d14febaaaa9..11ec26f28fb 100644 --- a/pkg/registry/apis/folders/conversions.go +++ b/pkg/registry/apis/folders/conversions.go @@ -6,9 +6,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/folder" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" ) func convertToK8sResource(v *folder.Folder, namespacer request.NamespaceMapper) *v0alpha1.Folder { diff --git a/pkg/registry/apis/folders/legacy_storage.go b/pkg/registry/apis/folders/legacy_storage.go index e6515b13beb..ac258217ccc 100644 --- a/pkg/registry/apis/folders/legacy_storage.go +++ b/pkg/registry/apis/folders/legacy_storage.go @@ -13,11 +13,11 @@ import ( "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/storage/entity" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/folder" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/storage/entity" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/util" ) diff --git a/pkg/registry/apis/folders/register.go b/pkg/registry/apis/folders/register.go index 4e385b2bded..15544b9f68b 100644 --- a/pkg/registry/apis/folders/register.go +++ b/pkg/registry/apis/folders/register.go @@ -14,16 +14,16 @@ import ( common "k8s.io/kube-openapi/pkg/common" "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/folder" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/setting" ) -var _ grafanaapiserver.APIGroupBuilder = (*FolderAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*FolderAPIBuilder)(nil) var resourceInfo = v0alpha1.FolderResourceInfo @@ -37,7 +37,7 @@ type FolderAPIBuilder struct { func RegisterAPIService(cfg *setting.Cfg, features *featuremgmt.FeatureManager, - apiregistration grafanaapiserver.APIRegistrar, + apiregistration builder.APIRegistrar, folderSvc folder.Service, ) *FolderAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { @@ -89,6 +89,7 @@ func (b *FolderAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, + dualWrite bool, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs) @@ -122,7 +123,7 @@ func (b *FolderAPIBuilder) GetAPIGroupInfo( storage[resourceInfo.StoragePath("children")] = &subChildrenREST{b.folderSvc} // enable dual writes if a RESTOptionsGetter is provided - if optsGetter != nil { + if dualWrite && optsGetter != nil { store, err := newStorage(scheme, optsGetter, legacyStore) if err != nil { return nil, err @@ -138,7 +139,7 @@ func (b *FolderAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions return v0alpha1.GetOpenAPIDefinitions } -func (b *FolderAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *FolderAPIBuilder) GetAPIRoutes() *builder.APIRoutes { return nil // no custom API routes } diff --git a/pkg/registry/apis/folders/storage.go b/pkg/registry/apis/folders/storage.go index b3f7ee2d073..dc90e9ff792 100644 --- a/pkg/registry/apis/folders/storage.go +++ b/pkg/registry/apis/folders/storage.go @@ -6,8 +6,8 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" - grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic" - grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest" + grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic" + grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest" ) var _ grafanarest.Storage = (*storage)(nil) diff --git a/pkg/registry/apis/folders/sub_children.go b/pkg/registry/apis/folders/sub_children.go index 8e9be8b0813..9d5deb40712 100644 --- a/pkg/registry/apis/folders/sub_children.go +++ b/pkg/registry/apis/folders/sub_children.go @@ -9,8 +9,8 @@ import ( "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/folder" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" ) type subChildrenREST struct { diff --git a/pkg/registry/apis/folders/sub_parents.go b/pkg/registry/apis/folders/sub_parents.go index 95ff32f42a4..36f67032d60 100644 --- a/pkg/registry/apis/folders/sub_parents.go +++ b/pkg/registry/apis/folders/sub_parents.go @@ -8,8 +8,8 @@ import ( "k8s.io/apiserver/pkg/registry/rest" "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/folder" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" ) type subParentsREST struct { diff --git a/pkg/registry/apis/playlist/conversions.go b/pkg/registry/apis/playlist/conversions.go index 43317aac080..c301cf9c04f 100644 --- a/pkg/registry/apis/playlist/conversions.go +++ b/pkg/registry/apis/playlist/conversions.go @@ -11,8 +11,8 @@ import ( "k8s.io/apimachinery/pkg/types" playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/utils" playlistsvc "github.com/grafana/grafana/pkg/services/playlist" ) diff --git a/pkg/registry/apis/playlist/conversions_test.go b/pkg/registry/apis/playlist/conversions_test.go index 3bb8284bb8e..83188572030 100644 --- a/pkg/registry/apis/playlist/conversions_test.go +++ b/pkg/registry/apis/playlist/conversions_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/playlist" ) diff --git a/pkg/registry/apis/playlist/legacy_storage.go b/pkg/registry/apis/playlist/legacy_storage.go index 699d6e5371c..316c0f52462 100644 --- a/pkg/registry/apis/playlist/legacy_storage.go +++ b/pkg/registry/apis/playlist/legacy_storage.go @@ -11,7 +11,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" playlistsvc "github.com/grafana/grafana/pkg/services/playlist" ) diff --git a/pkg/registry/apis/playlist/register.go b/pkg/registry/apis/playlist/register.go index b1dab9754f0..4bb5998ca9b 100644 --- a/pkg/registry/apis/playlist/register.go +++ b/pkg/registry/apis/playlist/register.go @@ -15,15 +15,15 @@ import ( common "k8s.io/kube-openapi/pkg/common" playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest" + "github.com/grafana/grafana/pkg/services/apiserver/utils" playlistsvc "github.com/grafana/grafana/pkg/services/playlist" "github.com/grafana/grafana/pkg/setting" ) -var _ grafanaapiserver.APIGroupBuilder = (*PlaylistAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*PlaylistAPIBuilder)(nil) // This is used just so wire has something unique to return type PlaylistAPIBuilder struct { @@ -33,7 +33,7 @@ type PlaylistAPIBuilder struct { } func RegisterAPIService(p playlistsvc.Service, - apiregistration grafanaapiserver.APIRegistrar, + apiregistration builder.APIRegistrar, cfg *setting.Cfg, ) *PlaylistAPIBuilder { builder := &PlaylistAPIBuilder{ @@ -79,6 +79,7 @@ func (b *PlaylistAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, + dualWrite bool, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(playlist.GROUP, scheme, metav1.ParameterCodec, codecs) storage := map[string]rest.Storage{} @@ -128,7 +129,7 @@ func (b *PlaylistAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinition return playlist.GetOpenAPIDefinitions } -func (b *PlaylistAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *PlaylistAPIBuilder) GetAPIRoutes() *builder.APIRoutes { return nil // no custom API routes } diff --git a/pkg/registry/apis/playlist/storage.go b/pkg/registry/apis/playlist/storage.go index b0f848cc11f..eb7512c717b 100644 --- a/pkg/registry/apis/playlist/storage.go +++ b/pkg/registry/apis/playlist/storage.go @@ -6,8 +6,8 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" - grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic" - grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest" + grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic" + grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest" ) var _ grafanarest.Storage = (*storage)(nil) diff --git a/pkg/registry/apis/query/register.go b/pkg/registry/apis/query/register.go index 9b450c4904d..e1f3fa02cd2 100644 --- a/pkg/registry/apis/query/register.go +++ b/pkg/registry/apis/query/register.go @@ -21,14 +21,14 @@ import ( "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/registry/apis/query/runner" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" "github.com/grafana/grafana/pkg/services/pluginsintegration/plugincontext" "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore" ) -var _ grafanaapiserver.APIGroupBuilder = (*QueryAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*QueryAPIBuilder)(nil) type QueryAPIBuilder struct { log log.Logger @@ -54,7 +54,7 @@ func NewQueryAPIBuilder(features featuremgmt.FeatureToggles, } func RegisterAPIService(features featuremgmt.FeatureToggles, - apiregistration grafanaapiserver.APIRegistrar, + apiregistration builder.APIRegistrar, dataSourcesService datasources.DataSourceService, pluginStore pluginstore.Store, accessControl accesscontrol.AccessControl, @@ -106,6 +106,7 @@ func (b *QueryAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, + _ bool, ) (*genericapiserver.APIGroupInfo, error) { gv := v0alpha1.SchemeGroupVersion apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(gv.Group, scheme, metav1.ParameterCodec, codecs) @@ -124,7 +125,7 @@ func (b *QueryAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions { } // Register additional routes with the server -func (b *QueryAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *QueryAPIBuilder) GetAPIRoutes() *builder.APIRoutes { defs := v0alpha1.GetOpenAPIDefinitions(func(path string) spec.Ref { return spec.Ref{} }) querySchema := defs["github.com/grafana/grafana/pkg/apis/query/v0alpha1.QueryRequest"].Schema responseSchema := defs["github.com/grafana/grafana/pkg/apis/query/v0alpha1.QueryDataResponse"].Schema @@ -167,9 +168,9 @@ func (b *QueryAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { "to": "1704914981544" }`), &randomWalkTable) - return &grafanaapiserver.APIRoutes{ - Root: []grafanaapiserver.APIRouteHandler{}, - Namespace: []grafanaapiserver.APIRouteHandler{ + return &builder.APIRoutes{ + Root: []builder.APIRouteHandler{}, + Namespace: []builder.APIRouteHandler{ { Path: "query", Spec: &spec3.PathProps{ diff --git a/pkg/registry/apis/service/register.go b/pkg/registry/apis/service/register.go index c2cc3e57584..f7366b3badf 100644 --- a/pkg/registry/apis/service/register.go +++ b/pkg/registry/apis/service/register.go @@ -12,11 +12,11 @@ import ( "k8s.io/kube-openapi/pkg/common" service "github.com/grafana/grafana/pkg/apis/service/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" ) -var _ grafanaapiserver.APIGroupBuilder = (*ServiceAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*ServiceAPIBuilder)(nil) // This is used just so wire has something unique to return type ServiceAPIBuilder struct{} @@ -25,10 +25,7 @@ func NewServiceAPIBuilder() *ServiceAPIBuilder { return &ServiceAPIBuilder{} } -func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration grafanaapiserver.APIRegistrar) *ServiceAPIBuilder { - if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { - return nil // skip registration unless opting into experimental apis - } +func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *ServiceAPIBuilder { builder := NewServiceAPIBuilder() apiregistration.RegisterAPI(NewServiceAPIBuilder()) return builder @@ -64,6 +61,7 @@ func (b *ServiceAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, optsGetter generic.RESTOptionsGetter, + _ bool, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(service.GROUP, scheme, metav1.ParameterCodec, codecs) @@ -83,6 +81,6 @@ func (b *ServiceAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions } // Register additional routes with the server -func (b *ServiceAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *ServiceAPIBuilder) GetAPIRoutes() *builder.APIRoutes { return nil } diff --git a/pkg/registry/apis/service/storage.go b/pkg/registry/apis/service/storage.go index db4dc56daf3..bf0d8551b80 100644 --- a/pkg/registry/apis/service/storage.go +++ b/pkg/registry/apis/service/storage.go @@ -10,9 +10,9 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" service "github.com/grafana/grafana/pkg/apis/service/v0alpha1" - grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic" - grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" + grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic" + grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest" + "github.com/grafana/grafana/pkg/services/apiserver/utils" ) var _ grafanarest.Storage = (*storage)(nil) diff --git a/pkg/registry/apis/wireset.go b/pkg/registry/apis/wireset.go index c5da75bf909..e093f7a175e 100644 --- a/pkg/registry/apis/wireset.go +++ b/pkg/registry/apis/wireset.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/folders" "github.com/grafana/grafana/pkg/registry/apis/playlist" "github.com/grafana/grafana/pkg/registry/apis/query" + "github.com/grafana/grafana/pkg/registry/apis/service" "github.com/grafana/grafana/pkg/services/pluginsintegration/plugincontext" ) @@ -28,5 +29,6 @@ var WireSet = wire.NewSet( featuretoggle.RegisterAPIService, datasource.RegisterAPIService, folders.RegisterAPIService, + service.RegisterAPIService, query.RegisterAPIService, ) diff --git a/pkg/registry/backgroundsvcs/background_services.go b/pkg/registry/backgroundsvcs/background_services.go index 7c8a0821833..3f323ab09ae 100644 --- a/pkg/registry/backgroundsvcs/background_services.go +++ b/pkg/registry/backgroundsvcs/background_services.go @@ -11,10 +11,10 @@ import ( apiregistry "github.com/grafana/grafana/pkg/registry/apis" "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/anonymous/anonimpl" + grafanaapiserver "github.com/grafana/grafana/pkg/services/apiserver" "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/cleanup" "github.com/grafana/grafana/pkg/services/dashboardsnapshots" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" "github.com/grafana/grafana/pkg/services/grpcserver" "github.com/grafana/grafana/pkg/services/guardian" ldapapi "github.com/grafana/grafana/pkg/services/ldap/api" diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 65458a41d2f..6e7c1629129 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -44,6 +44,7 @@ import ( "github.com/grafana/grafana/pkg/services/annotations/annotationsimpl" "github.com/grafana/grafana/pkg/services/anonymous/anonimpl/anonstore" "github.com/grafana/grafana/pkg/services/apikey/apikeyimpl" + grafanaapiserver "github.com/grafana/grafana/pkg/services/apiserver" "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/auth/idimpl" "github.com/grafana/grafana/pkg/services/auth/jwt" @@ -72,7 +73,6 @@ import ( "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/folder" "github.com/grafana/grafana/pkg/services/folder/folderimpl" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" "github.com/grafana/grafana/pkg/services/grpcserver" grpccontext "github.com/grafana/grafana/pkg/services/grpcserver/context" "github.com/grafana/grafana/pkg/services/grpcserver/interceptors" diff --git a/pkg/services/grafana-apiserver/README.md b/pkg/services/apiserver/README.md similarity index 100% rename from pkg/services/grafana-apiserver/README.md rename to pkg/services/apiserver/README.md diff --git a/pkg/aggregator/README.md b/pkg/services/apiserver/aggregator/README.md similarity index 100% rename from pkg/aggregator/README.md rename to pkg/services/apiserver/aggregator/README.md diff --git a/pkg/services/apiserver/aggregator/aggregator.go b/pkg/services/apiserver/aggregator/aggregator.go new file mode 100644 index 00000000000..b62e106446e --- /dev/null +++ b/pkg/services/apiserver/aggregator/aggregator.go @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/aggregator.go +// Provenance-includes-license: Apache-2.0 +// Provenance-includes-copyright: The Kubernetes Authors. +// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/server.go +// Provenance-includes-license: Apache-2.0 +// Provenance-includes-copyright: The Kubernetes Authors. +// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/pkg/controlplane/apiserver/apiextensions.go +// Provenance-includes-license: Apache-2.0 +// Provenance-includes-copyright: The Kubernetes Authors. + +package aggregator + +import ( + "crypto/tls" + "fmt" + "net/http" + "strings" + "sync" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/sets" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/healthz" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper" + aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" + apiregistrationclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" + apiregistrationInformers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1" + "k8s.io/kube-aggregator/pkg/controllers/autoregister" + + serviceclientset "github.com/grafana/grafana/pkg/generated/clientset/versioned" + informersv0alpha1 "github.com/grafana/grafana/pkg/generated/informers/externalversions" + "github.com/grafana/grafana/pkg/services/apiserver/options" +) + +func CreateAggregatorConfig(commandOptions *options.Options, sharedConfig genericapiserver.RecommendedConfig) (*aggregatorapiserver.Config, informersv0alpha1.SharedInformerFactory, error) { + // Create a fake clientset and informers for the k8s v1 API group. + // These are not used in grafana's aggregator because v1 APIs are not available. + fakev1Informers := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 10*time.Minute) + + serviceClient, err := serviceclientset.NewForConfig(sharedConfig.LoopbackClientConfig) + if err != nil { + return nil, nil, err + } + sharedInformerFactory := informersv0alpha1.NewSharedInformerFactory( + serviceClient, + 5*time.Minute, // this is effectively used as a refresh interval right now. Might want to do something nicer later on. + ) + serviceResolver := NewExternalNameResolver(sharedInformerFactory.Service().V0alpha1().ExternalNames().Lister()) + + aggregatorConfig := &aggregatorapiserver.Config{ + GenericConfig: &genericapiserver.RecommendedConfig{ + Config: sharedConfig.Config, + SharedInformerFactory: fakev1Informers, + ClientConfig: sharedConfig.LoopbackClientConfig, + }, + ExtraConfig: aggregatorapiserver.ExtraConfig{ + ProxyClientCertFile: commandOptions.AggregatorOptions.ProxyClientCertFile, + ProxyClientKeyFile: commandOptions.AggregatorOptions.ProxyClientKeyFile, + // NOTE: while ProxyTransport can be skipped in the configuration, it allows honoring + // DISABLE_HTTP2, HTTPS_PROXY and NO_PROXY env vars as needed + ProxyTransport: createProxyTransport(), + ServiceResolver: serviceResolver, + }, + } + + if err := commandOptions.AggregatorOptions.ApplyTo(aggregatorConfig, commandOptions.RecommendedOptions.Etcd, commandOptions.StorageOptions.DataPath); err != nil { + return nil, nil, err + } + + return aggregatorConfig, sharedInformerFactory, nil +} + +func CreateAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, sharedInformerFactory informersv0alpha1.SharedInformerFactory, delegateAPIServer genericapiserver.DelegationTarget) (*aggregatorapiserver.APIAggregator, error) { + completedConfig := aggregatorConfig.Complete() + aggregatorServer, err := completedConfig.NewWithDelegate(delegateAPIServer) + if err != nil { + return nil, err + } + + // create controllers for auto-registration + apiRegistrationClient, err := apiregistrationclient.NewForConfig(completedConfig.GenericConfig.LoopbackClientConfig) + if err != nil { + return nil, err + } + + autoRegistrationController := autoregister.NewAutoRegisterController(aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), apiRegistrationClient) + apiServices := apiServicesToRegister(delegateAPIServer, autoRegistrationController) + + // Imbue all builtin group-priorities onto the aggregated discovery + if completedConfig.GenericConfig.AggregatedDiscoveryGroupManager != nil { + for gv, entry := range APIVersionPriorities { + completedConfig.GenericConfig.AggregatedDiscoveryGroupManager.SetGroupVersionPriority(metav1.GroupVersion(gv), int(entry.Group), int(entry.Version)) + } + } + + err = aggregatorServer.GenericAPIServer.AddPostStartHook("grafana-apiserver-autoregistration", func(context genericapiserver.PostStartHookContext) error { + go func() { + autoRegistrationController.Run(5, context.StopCh) + }() + return nil + }) + if err != nil { + return nil, err + } + + err = aggregatorServer.GenericAPIServer.AddBootSequenceHealthChecks( + makeAPIServiceAvailableHealthCheck( + "autoregister-completion", + apiServices, + aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), + ), + ) + if err != nil { + return nil, err + } + + apiregistrationClient, err := apiregistrationclientset.NewForConfig(completedConfig.GenericConfig.LoopbackClientConfig) + if err != nil { + return nil, err + } + + availableController, err := NewAvailableConditionController( + aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), + sharedInformerFactory.Service().V0alpha1().ExternalNames(), + apiregistrationClient.ApiregistrationV1(), + nil, + (func() ([]byte, []byte))(nil), + completedConfig.ExtraConfig.ServiceResolver, + ) + if err != nil { + return nil, err + } + + aggregatorServer.GenericAPIServer.AddPostStartHookOrDie("apiservice-status-override-available-controller", func(context genericapiserver.PostStartHookContext) error { + // if we end up blocking for long periods of time, we may need to increase workers. + go availableController.Run(5, context.StopCh) + return nil + }) + + aggregatorServer.GenericAPIServer.AddPostStartHookOrDie("start-grafana-aggregator-informers", func(context genericapiserver.PostStartHookContext) error { + sharedInformerFactory.Start(context.StopCh) + aggregatorServer.APIRegistrationInformers.Start(context.StopCh) + return nil + }) + + return aggregatorServer, nil +} + +func makeAPIService(gv schema.GroupVersion) *v1.APIService { + apiServicePriority, ok := APIVersionPriorities[gv] + if !ok { + // if we aren't found, then we shouldn't register ourselves because it could result in a CRD group version + // being permanently stuck in the APIServices list. + klog.Infof("Skipping APIService creation for %v", gv) + return nil + } + return &v1.APIService{ + ObjectMeta: metav1.ObjectMeta{Name: gv.Version + "." + gv.Group}, + Spec: v1.APIServiceSpec{ + Group: gv.Group, + Version: gv.Version, + GroupPriorityMinimum: apiServicePriority.Group, + VersionPriority: apiServicePriority.Version, + }, + } +} + +// makeAPIServiceAvailableHealthCheck returns a healthz check that returns healthy +// once all of the specified services have been observed to be available at least once. +func makeAPIServiceAvailableHealthCheck(name string, apiServices []*v1.APIService, apiServiceInformer apiregistrationInformers.APIServiceInformer) healthz.HealthChecker { + // Track the auto-registered API services that have not been observed to be available yet + pendingServiceNamesLock := &sync.RWMutex{} + pendingServiceNames := sets.NewString() + for _, service := range apiServices { + pendingServiceNames.Insert(service.Name) + } + + // When an APIService in the list is seen as available, remove it from the pending list + handleAPIServiceChange := func(service *v1.APIService) { + pendingServiceNamesLock.Lock() + defer pendingServiceNamesLock.Unlock() + if !pendingServiceNames.Has(service.Name) { + return + } + if v1helper.IsAPIServiceConditionTrue(service, v1.Available) { + pendingServiceNames.Delete(service.Name) + } + } + + // Watch add/update events for APIServices + _, _ = apiServiceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { handleAPIServiceChange(obj.(*v1.APIService)) }, + UpdateFunc: func(old, new interface{}) { handleAPIServiceChange(new.(*v1.APIService)) }, + }) + + // Don't return healthy until the pending list is empty + return healthz.NamedCheck(name, func(r *http.Request) error { + pendingServiceNamesLock.RLock() + defer pendingServiceNamesLock.RUnlock() + if pendingServiceNames.Len() > 0 { + return fmt.Errorf("missing APIService: %v", pendingServiceNames.List()) + } + return nil + }) +} + +// Priority defines group Priority that is used in discovery. This controls +// group position in the kubectl output. +type Priority struct { + // Group indicates the order of the Group relative to other groups. + Group int32 + // Version indicates the relative order of the Version inside of its group. + Version int32 +} + +// APIVersionPriorities are the proper way to resolve this letting the aggregator know the desired group and version-within-group order of the underlying servers +// is to refactor the genericapiserver.DelegationTarget to include a list of priorities based on which APIs were installed. +// This requires the APIGroupInfo struct to evolve and include the concept of priorities and to avoid mistakes, the core storage map there needs to be updated. +// That ripples out every bit as far as you'd expect, so for 1.7 we'll include the list here instead of being built up during storage. +var APIVersionPriorities = map[schema.GroupVersion]Priority{ + {Group: "", Version: "v1"}: {Group: 18000, Version: 1}, + // to my knowledge, nothing below here collides + {Group: "admissionregistration.k8s.io", Version: "v1"}: {Group: 16700, Version: 15}, + {Group: "admissionregistration.k8s.io", Version: "v1beta1"}: {Group: 16700, Version: 12}, + {Group: "admissionregistration.k8s.io", Version: "v1alpha1"}: {Group: 16700, Version: 9}, + // Append a new group to the end of the list if unsure. + // You can use min(existing group)-100 as the initial value for a group. + // Version can be set to 9 (to have space around) for a new group. +} + +func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*v1.APIService { + apiServices := []*v1.APIService{} + + for _, curr := range delegateAPIServer.ListedPaths() { + if curr == "/api/v1" { + apiService := makeAPIService(schema.GroupVersion{Group: "", Version: "v1"}) + registration.AddAPIServiceToSyncOnStart(apiService) + apiServices = append(apiServices, apiService) + continue + } + + if !strings.HasPrefix(curr, "/apis/") { + continue + } + // this comes back in a list that looks like /apis/rbac.authorization.k8s.io/v1alpha1 + tokens := strings.Split(curr, "/") + if len(tokens) != 4 { + continue + } + + apiService := makeAPIService(schema.GroupVersion{Group: tokens[2], Version: tokens[3]}) + if apiService == nil { + continue + } + registration.AddAPIServiceToSyncOnStart(apiService) + apiServices = append(apiServices, apiService) + } + + return apiServices +} + +// NOTE: below function imported from https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/server.go#L197 +// createProxyTransport creates the dialer infrastructure to connect to the api servers. +func createProxyTransport() *http.Transport { + // NOTE: We don't set proxyDialerFn but the below SetTransportDefaults will + // See https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/util/net/http.go#L109 + var proxyDialerFn utilnet.DialFunc + // Proxying to services is IP-based... don't expect to be able to verify the hostname + proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} + proxyTransport := utilnet.SetTransportDefaults(&http.Transport{ + DialContext: proxyDialerFn, + TLSClientConfig: proxyTLSClientConfig, + }) + return proxyTransport +} diff --git a/pkg/aggregator/availableController.go b/pkg/services/apiserver/aggregator/availableController.go similarity index 100% rename from pkg/aggregator/availableController.go rename to pkg/services/apiserver/aggregator/availableController.go diff --git a/pkg/aggregator/resolver.go b/pkg/services/apiserver/aggregator/resolver.go similarity index 100% rename from pkg/aggregator/resolver.go rename to pkg/services/apiserver/aggregator/resolver.go diff --git a/pkg/services/grafana-apiserver/auth/authorizer/impersonation.go b/pkg/services/apiserver/auth/authorizer/impersonation.go similarity index 100% rename from pkg/services/grafana-apiserver/auth/authorizer/impersonation.go rename to pkg/services/apiserver/auth/authorizer/impersonation.go diff --git a/pkg/services/grafana-apiserver/auth/authorizer/impersonation_test.go b/pkg/services/apiserver/auth/authorizer/impersonation_test.go similarity index 100% rename from pkg/services/grafana-apiserver/auth/authorizer/impersonation_test.go rename to pkg/services/apiserver/auth/authorizer/impersonation_test.go diff --git a/pkg/services/grafana-apiserver/auth/authorizer/org_id.go b/pkg/services/apiserver/auth/authorizer/org_id.go similarity index 95% rename from pkg/services/grafana-apiserver/auth/authorizer/org_id.go rename to pkg/services/apiserver/auth/authorizer/org_id.go index 4d6efdfc88a..e9d4a6786a8 100644 --- a/pkg/services/grafana-apiserver/auth/authorizer/org_id.go +++ b/pkg/services/apiserver/auth/authorizer/org_id.go @@ -8,7 +8,7 @@ import ( "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/infra/log" - grafanarequest "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + grafanarequest "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/org" ) diff --git a/pkg/services/grafana-apiserver/auth/authorizer/org_role.go b/pkg/services/apiserver/auth/authorizer/org_role.go similarity index 100% rename from pkg/services/grafana-apiserver/auth/authorizer/org_role.go rename to pkg/services/apiserver/auth/authorizer/org_role.go diff --git a/pkg/services/grafana-apiserver/auth/authorizer/provider.go b/pkg/services/apiserver/auth/authorizer/provider.go similarity index 100% rename from pkg/services/grafana-apiserver/auth/authorizer/provider.go rename to pkg/services/apiserver/auth/authorizer/provider.go diff --git a/pkg/services/grafana-apiserver/auth/authorizer/stack_id.go b/pkg/services/apiserver/auth/authorizer/stack_id.go similarity index 94% rename from pkg/services/grafana-apiserver/auth/authorizer/stack_id.go rename to pkg/services/apiserver/auth/authorizer/stack_id.go index 32887dd8142..0098c1dfd21 100644 --- a/pkg/services/grafana-apiserver/auth/authorizer/stack_id.go +++ b/pkg/services/apiserver/auth/authorizer/stack_id.go @@ -8,7 +8,7 @@ import ( "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/infra/log" - grafanarequest "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + grafanarequest "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/setting" ) diff --git a/pkg/services/grafana-apiserver/common.go b/pkg/services/apiserver/builder/common.go similarity index 94% rename from pkg/services/grafana-apiserver/common.go rename to pkg/services/apiserver/builder/common.go index 5845bfc3c22..39d2c5dccee 100644 --- a/pkg/services/grafana-apiserver/common.go +++ b/pkg/services/apiserver/builder/common.go @@ -1,4 +1,4 @@ -package grafanaapiserver +package builder import ( "net/http" @@ -27,6 +27,7 @@ type APIGroupBuilder interface { scheme *runtime.Scheme, codecs serializer.CodecFactory, optsGetter generic.RESTOptionsGetter, + dualWrite bool, ) (*genericapiserver.APIGroupInfo, error) // Get OpenAPI definitions @@ -57,3 +58,7 @@ type APIRoutes struct { // Namespace handlers are mounted under the namespace Namespace []APIRouteHandler } + +type APIRegistrar interface { + RegisterAPI(builder APIGroupBuilder) +} diff --git a/pkg/services/grafana-apiserver/helper.go b/pkg/services/apiserver/builder/helper.go similarity index 65% rename from pkg/services/grafana-apiserver/helper.go rename to pkg/services/apiserver/builder/helper.go index 8205715487f..f6987752365 100644 --- a/pkg/services/grafana-apiserver/helper.go +++ b/pkg/services/apiserver/builder/helper.go @@ -1,34 +1,45 @@ -package grafanaapiserver +package builder import ( "fmt" "net/http" goruntime "runtime" + "runtime/debug" + "strconv" "strings" "time" + "golang.org/x/mod/semver" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/version" openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" "k8s.io/apiserver/pkg/registry/generic" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/util/openapi" - "k8s.io/client-go/kubernetes/scheme" + k8sscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/kube-openapi/pkg/common" "github.com/grafana/grafana/pkg/setting" ) -func SetupConfig(serverConfig *genericapiserver.RecommendedConfig, builders []APIGroupBuilder) error { +func SetupConfig( + scheme *runtime.Scheme, + serverConfig *genericapiserver.RecommendedConfig, + builders []APIGroupBuilder, +) error { defsGetter := GetOpenAPIDefinitions(builders) serverConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig( openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(defsGetter), - openapinamer.NewDefinitionNamer(Scheme, scheme.Scheme)) + openapinamer.NewDefinitionNamer(scheme, k8sscheme.Scheme)) serverConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config( openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(defsGetter), - openapinamer.NewDefinitionNamer(Scheme, scheme.Scheme)) + openapinamer.NewDefinitionNamer(scheme, k8sscheme.Scheme)) + // Add the custom routes to service discovery + serverConfig.OpenAPIV3Config.PostProcessSpec = getOpenAPIPostProcessor(builders) serverConfig.OpenAPIV3Config.GetOperationIDAndTagsFromRoute = func(r common.Route) (string, []string, error) { tags := []string{} prop, ok := r.Metadata()["x-kubernetes-group-version-kind"] @@ -41,9 +52,6 @@ func SetupConfig(serverConfig *genericapiserver.RecommendedConfig, builders []AP return r.OperationName(), tags, nil } - // Add the custom routes to service discovery - serverConfig.OpenAPIV3Config.PostProcessSpec = getOpenAPIPostProcessor(builders) - // Set the swagger build versions serverConfig.OpenAPIConfig.Info.Version = setting.BuildVersion serverConfig.OpenAPIV3Config.Info.Version = setting.BuildVersion @@ -82,12 +90,16 @@ func SetupConfig(serverConfig *genericapiserver.RecommendedConfig, builders []AP return nil } -func InstallAPIs(server *genericapiserver.GenericAPIServer, +func InstallAPIs( + scheme *runtime.Scheme, + codecs serializer.CodecFactory, + server *genericapiserver.GenericAPIServer, optsGetter generic.RESTOptionsGetter, builders []APIGroupBuilder, + dualWrite bool, ) error { for _, b := range builders { - g, err := b.GetAPIGroupInfo(Scheme, Codecs, optsGetter) + g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, dualWrite) if err != nil { return err } @@ -101,3 +113,33 @@ func InstallAPIs(server *genericapiserver.GenericAPIServer, } return nil } + +// find the k8s version according to build info +func getK8sApiserverVersion() (string, error) { + bi, ok := debug.ReadBuildInfo() + if !ok { + return "", fmt.Errorf("debug.ReadBuildInfo() failed") + } + + if len(bi.Deps) == 0 { + return "v?.?", nil // this is normal while debugging + } + + for _, dep := range bi.Deps { + if dep.Path == "k8s.io/apiserver" { + if !semver.IsValid(dep.Version) { + return "", fmt.Errorf("invalid semantic version for k8s.io/apiserver") + } + // v0 => v1 + majorVersion := strings.TrimPrefix(semver.Major(dep.Version), "v") + majorInt, err := strconv.Atoi(majorVersion) + if err != nil { + return "", fmt.Errorf("could not convert majorVersion to int. majorVersion: %s", majorVersion) + } + newMajor := fmt.Sprintf("v%d", majorInt+1) + return strings.Replace(dep.Version, semver.Major(dep.Version), newMajor, 1), nil + } + } + + return "", fmt.Errorf("could not find k8s.io/apiserver in build info") +} diff --git a/pkg/services/grafana-apiserver/openapi.go b/pkg/services/apiserver/builder/openapi.go similarity index 99% rename from pkg/services/grafana-apiserver/openapi.go rename to pkg/services/apiserver/builder/openapi.go index f90b6bfdc46..e4d6a2ac149 100644 --- a/pkg/services/grafana-apiserver/openapi.go +++ b/pkg/services/apiserver/builder/openapi.go @@ -1,4 +1,4 @@ -package grafanaapiserver +package builder import ( "maps" @@ -15,8 +15,8 @@ func GetOpenAPIDefinitions(builders []APIGroupBuilder) common.GetOpenAPIDefiniti return func(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { defs := getStandardOpenAPIDefinitions(ref) maps.Copy(defs, v0alpha1.GetOpenAPIDefinitions(ref)) // common grafana apis - for _, builder := range builders { - g := builder.GetOpenAPIDefinitions() + for _, b := range builders { + g := b.GetOpenAPIDefinitions() if g != nil { out := g(ref) maps.Copy(defs, out) diff --git a/pkg/services/grafana-apiserver/request_handler.go b/pkg/services/apiserver/builder/request_handler.go similarity index 96% rename from pkg/services/grafana-apiserver/request_handler.go rename to pkg/services/apiserver/builder/request_handler.go index 48199878a67..b88c221b57f 100644 --- a/pkg/services/grafana-apiserver/request_handler.go +++ b/pkg/services/apiserver/builder/request_handler.go @@ -1,4 +1,4 @@ -package grafanaapiserver +package builder import ( "fmt" @@ -125,9 +125,9 @@ func getOpenAPIPostProcessor(builders []APIGroupBuilder) func(*spec3.OpenAPI) (* if s.Paths == nil { return s, nil } - for _, builder := range builders { - routes := builder.GetAPIRoutes() - gv := builder.GetGroupVersion() + for _, b := range builders { + routes := b.GetAPIRoutes() + gv := b.GetGroupVersion() prefix := "/apis/" + gv.String() + "/" if s.Paths.Paths[prefix] != nil { copy := spec3.OpenAPI{ diff --git a/pkg/services/apiserver/config.go b/pkg/services/apiserver/config.go new file mode 100644 index 00000000000..3cc6fc82d07 --- /dev/null +++ b/pkg/services/apiserver/config.go @@ -0,0 +1,48 @@ +package apiserver + +import ( + "fmt" + "net" + "path/filepath" + "strconv" + + "github.com/grafana/grafana/pkg/services/apiserver/options" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/setting" +) + +func applyGrafanaConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles, o *options.Options) { + defaultLogLevel := 0 + ip := net.ParseIP(cfg.HTTPAddr) + apiURL := cfg.AppURL + port, err := strconv.Atoi(cfg.HTTPPort) + if err != nil { + port = 3000 + } + + if cfg.Env == setting.Dev { + defaultLogLevel = 10 + port = 6443 + ip = net.ParseIP("127.0.0.1") + apiURL = fmt.Sprintf("https://%s:%d", ip, port) + } + + host := fmt.Sprintf("%s:%d", ip, port) + + o.RecommendedOptions.Etcd.StorageConfig.Transport.ServerList = cfg.SectionWithEnvOverrides("grafana-apiserver").Key("etcd_servers").Strings(",") + + o.RecommendedOptions.SecureServing.BindAddress = ip + o.RecommendedOptions.SecureServing.BindPort = port + o.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true + o.RecommendedOptions.Authorization.RemoteKubeConfigFileOptional = true + + o.RecommendedOptions.Admission = nil + o.RecommendedOptions.CoreAPI = nil + + o.StorageOptions.StorageType = options.StorageType(cfg.SectionWithEnvOverrides("grafana-apiserver").Key("storage_type").MustString(string(options.StorageTypeLegacy))) + o.StorageOptions.DataPath = filepath.Join(cfg.DataPath, "grafana-apiserver") + o.ExtraOptions.DevMode = features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerEnsureKubectlAccess) + o.ExtraOptions.ExternalAddress = host + o.ExtraOptions.APIURL = apiURL + o.ExtraOptions.Verbosity = defaultLogLevel +} diff --git a/pkg/services/grafana-apiserver/endpoints/request/namespace.go b/pkg/services/apiserver/endpoints/request/namespace.go similarity index 100% rename from pkg/services/grafana-apiserver/endpoints/request/namespace.go rename to pkg/services/apiserver/endpoints/request/namespace.go diff --git a/pkg/services/grafana-apiserver/endpoints/request/namespace_test.go b/pkg/services/apiserver/endpoints/request/namespace_test.go similarity index 97% rename from pkg/services/grafana-apiserver/endpoints/request/namespace_test.go rename to pkg/services/apiserver/endpoints/request/namespace_test.go index 4f9dc7e9281..cd5a8e58394 100644 --- a/pkg/services/grafana-apiserver/endpoints/request/namespace_test.go +++ b/pkg/services/apiserver/endpoints/request/namespace_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/setting" ) diff --git a/pkg/services/apiserver/options/aggregator.go b/pkg/services/apiserver/options/aggregator.go new file mode 100644 index 00000000000..51c67cea117 --- /dev/null +++ b/pkg/services/apiserver/options/aggregator.go @@ -0,0 +1,112 @@ +package options + +import ( + "github.com/spf13/pflag" + v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" + genericfeatures "k8s.io/apiserver/pkg/features" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/server/resourceconfig" + utilfeature "k8s.io/apiserver/pkg/util/feature" + apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" + aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" + aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" + aggregatoropenapi "k8s.io/kube-aggregator/pkg/generated/openapi" + "k8s.io/kube-openapi/pkg/common" + + servicev0alpha1 "github.com/grafana/grafana/pkg/apis/service/v0alpha1" + filestorage "github.com/grafana/grafana/pkg/services/apiserver/storage/file" +) + +// AggregatorServerOptions contains the state for the aggregator apiserver +type AggregatorServerOptions struct { + AlternateDNS []string + ProxyClientCertFile string + ProxyClientKeyFile string +} + +func NewAggregatorServerOptions() *AggregatorServerOptions { + return &AggregatorServerOptions{} +} + +func (o *AggregatorServerOptions) getMergedOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { + aggregatorAPIs := aggregatoropenapi.GetOpenAPIDefinitions(ref) + return aggregatorAPIs +} + +func (o *AggregatorServerOptions) AddFlags(fs *pflag.FlagSet) { + if o == nil { + return + } + + fs.StringVar(&o.ProxyClientCertFile, "proxy-client-cert-file", o.ProxyClientCertFile, + "path to proxy client cert file") + + fs.StringVar(&o.ProxyClientKeyFile, "proxy-client-key-file", o.ProxyClientKeyFile, + "path to proxy client cert file") +} + +func (o *AggregatorServerOptions) Validate() []error { + if o == nil { + return nil + } + + // TODO: do we need to validate anything here? + return nil +} + +func (o *AggregatorServerOptions) ApplyTo(aggregatorConfig *aggregatorapiserver.Config, etcdOpts *options.EtcdOptions, dataPath string) error { + genericConfig := aggregatorConfig.GenericConfig + + genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} + genericConfig.RESTOptionsGetter = nil + + if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) && + utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) { + // Add StorageVersionPrecondition handler to aggregator-apiserver. + // The handler will block write requests to built-in resources until the + // target resources' storage versions are up-to-date. + genericConfig.BuildHandlerChainFunc = genericapiserver.BuildHandlerChainWithStorageVersionPrecondition + } + + // copy the etcd options so we don't mutate originals. + // we assume that the etcd options have been completed already. avoid messing with anything outside + // of changes to StorageConfig as that may lead to unexpected behavior when the options are applied. + etcdOptions := *etcdOpts + etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion, + apiregistrationv1beta1.SchemeGroupVersion, + servicev0alpha1.SchemeGroupVersion) + etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1.SchemeGroupVersion, + schema.GroupKind{Group: apiregistrationv1beta1.GroupName}, + schema.GroupKind{Group: servicev0alpha1.GROUP}) + etcdOptions.SkipHealthEndpoints = true // avoid double wiring of health checks + if err := etcdOptions.ApplyTo(&genericConfig.Config); err != nil { + return err + } + // override the RESTOptionsGetter to use the file storage options getter + aggregatorConfig.GenericConfig.RESTOptionsGetter = filestorage.NewRESTOptionsGetter(dataPath, etcdOptions.StorageConfig) + + // prevent generic API server from installing the OpenAPI handler. Aggregator server has its own customized OpenAPI handler. + genericConfig.SkipOpenAPIInstallation = true + mergedResourceConfig, err := resourceconfig.MergeAPIResourceConfigs(aggregatorapiserver.DefaultAPIResourceConfigSource(), nil, aggregatorscheme.Scheme) + if err != nil { + return err + } + genericConfig.MergedResourceConfig = mergedResourceConfig + + namer := openapinamer.NewDefinitionNamer(aggregatorscheme.Scheme) + genericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(o.getMergedOpenAPIDefinitions, namer) + genericConfig.OpenAPIV3Config.Info.Title = "Kubernetes" + genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(o.getMergedOpenAPIDefinitions, namer) + genericConfig.OpenAPIConfig.Info.Title = "Kubernetes" + genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} + + // These hooks use v1 informers, which are not available in the grafana aggregator. + genericConfig.DisabledPostStartHooks = genericConfig.DisabledPostStartHooks.Insert("apiservice-status-available-controller") + genericConfig.DisabledPostStartHooks = genericConfig.DisabledPostStartHooks.Insert("start-kube-aggregator-informers") + + return nil +} diff --git a/pkg/services/apiserver/options/extra.go b/pkg/services/apiserver/options/extra.go new file mode 100644 index 00000000000..fdb394d4d28 --- /dev/null +++ b/pkg/services/apiserver/options/extra.go @@ -0,0 +1,46 @@ +package options + +import ( + "strconv" + + "github.com/go-logr/logr" + "github.com/spf13/pflag" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/component-base/logs" + "k8s.io/klog/v2" +) + +type ExtraOptions struct { + DevMode bool + ExternalAddress string + APIURL string + Verbosity int +} + +func NewExtraOptions() *ExtraOptions { + return &ExtraOptions{ + DevMode: false, + Verbosity: 0, + } +} + +func (o *ExtraOptions) AddFlags(fs *pflag.FlagSet) { + fs.BoolVar(&o.DevMode, "grafana-apiserver-dev-mode", o.DevMode, "Enable dev mode") + fs.StringVar(&o.ExternalAddress, "grafana-apiserver-host", o.ExternalAddress, "Host") + fs.StringVar(&o.APIURL, "grafana-apiserver-api-url", o.APIURL, "API URL") + fs.IntVar(&o.Verbosity, "verbosity", o.Verbosity, "Verbosity") +} + +func (o *ExtraOptions) Validate() []error { + return nil +} + +func (o *ExtraOptions) ApplyTo(c *genericapiserver.RecommendedConfig) error { + logger := logr.New(newLogAdapter(o.Verbosity)) + klog.SetLoggerWithOptions(logger, klog.ContextualLogger(true)) + if _, err := logs.GlogSetter(strconv.Itoa(o.Verbosity)); err != nil { + logger.Error(err, "failed to set log level") + } + c.ExternalAddress = o.ExternalAddress + return nil +} diff --git a/pkg/services/grafana-apiserver/log.go b/pkg/services/apiserver/options/log.go similarity index 97% rename from pkg/services/grafana-apiserver/log.go rename to pkg/services/apiserver/options/log.go index edf10c0d3d3..9e32f66e3b3 100644 --- a/pkg/services/grafana-apiserver/log.go +++ b/pkg/services/apiserver/options/log.go @@ -1,4 +1,4 @@ -package grafanaapiserver +package options import ( "strings" diff --git a/pkg/services/apiserver/options/options.go b/pkg/services/apiserver/options/options.go new file mode 100644 index 00000000000..10c320e8925 --- /dev/null +++ b/pkg/services/apiserver/options/options.go @@ -0,0 +1,126 @@ +package options + +import ( + "net" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" + genericapiserver "k8s.io/apiserver/pkg/server" + genericoptions "k8s.io/apiserver/pkg/server/options" +) + +const defaultEtcdPathPrefix = "/registry/grafana.app" + +type Options struct { + RecommendedOptions *genericoptions.RecommendedOptions + AggregatorOptions *AggregatorServerOptions + StorageOptions *StorageOptions + ExtraOptions *ExtraOptions +} + +func NewOptions(codec runtime.Codec) *Options { + return &Options{ + RecommendedOptions: genericoptions.NewRecommendedOptions( + defaultEtcdPathPrefix, + codec, + ), + AggregatorOptions: NewAggregatorServerOptions(), + StorageOptions: NewStorageOptions(), + ExtraOptions: NewExtraOptions(), + } +} + +func (o *Options) AddFlags(fs *pflag.FlagSet) { + o.RecommendedOptions.AddFlags(fs) + o.AggregatorOptions.AddFlags(fs) + o.StorageOptions.AddFlags(fs) + o.ExtraOptions.AddFlags(fs) +} + +func (o *Options) Validate() []error { + if errs := o.ExtraOptions.Validate(); len(errs) != 0 { + return errs + } + + if errs := o.StorageOptions.Validate(); len(errs) != 0 { + return errs + } + + if errs := o.AggregatorOptions.Validate(); len(errs) != 0 { + return errs + } + + if errs := o.RecommendedOptions.SecureServing.Validate(); len(errs) != 0 { + return errs + } + + if errs := o.RecommendedOptions.Authentication.Validate(); len(errs) != 0 { + return errs + } + + if o.StorageOptions.StorageType == StorageTypeEtcd { + if errs := o.RecommendedOptions.Etcd.Validate(); len(errs) != 0 { + return errs + } + } + + return nil +} + +func (o *Options) ApplyTo(serverConfig *genericapiserver.RecommendedConfig) error { + serverConfig.AggregatedDiscoveryGroupManager = aggregated.NewResourceManager("apis") + + if err := o.ExtraOptions.ApplyTo(serverConfig); err != nil { + return err + } + + if !o.ExtraOptions.DevMode { + o.RecommendedOptions.SecureServing.Listener = newFakeListener() + } + + if err := o.RecommendedOptions.SecureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil { + return err + } + + if err := o.RecommendedOptions.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, serverConfig.OpenAPIConfig); err != nil { + return err + } + + if !o.ExtraOptions.DevMode { + if err := serverConfig.SecureServing.Listener.Close(); err != nil { + return err + } + serverConfig.SecureServing = nil + } + + return nil +} + +type fakeListener struct { + server net.Conn + client net.Conn +} + +func newFakeListener() *fakeListener { + server, client := net.Pipe() + return &fakeListener{ + server: server, + client: client, + } +} + +func (f *fakeListener) Accept() (net.Conn, error) { + return f.server, nil +} + +func (f *fakeListener) Close() error { + if err := f.client.Close(); err != nil { + return err + } + return f.server.Close() +} + +func (f *fakeListener) Addr() net.Addr { + return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 3000, Zone: ""} +} diff --git a/pkg/services/apiserver/options/storage.go b/pkg/services/apiserver/options/storage.go new file mode 100644 index 00000000000..51ccb84efe9 --- /dev/null +++ b/pkg/services/apiserver/options/storage.go @@ -0,0 +1,51 @@ +package options + +import ( + "fmt" + + "github.com/spf13/pflag" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/options" +) + +type StorageType string + +const ( + StorageTypeFile StorageType = "file" + StorageTypeEtcd StorageType = "etcd" + StorageTypeLegacy StorageType = "legacy" + StorageTypeUnified StorageType = "unified" + StorageTypeUnifiedGrpc StorageType = "unified-grpc" +) + +type StorageOptions struct { + StorageType StorageType + DataPath string +} + +func NewStorageOptions() *StorageOptions { + return &StorageOptions{ + StorageType: StorageTypeLegacy, + } +} + +func (o *StorageOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar((*string)(&o.StorageType), "grafana-apiserver-storage-type", string(o.StorageType), "Storage type") + fs.StringVar((*string)(&o.StorageType), "grafana-apiserver-storage-path", string(o.StorageType), "Storage path for file storage") +} + +func (o *StorageOptions) Validate() []error { + errs := []error{} + switch o.StorageType { + case StorageTypeFile, StorageTypeEtcd, StorageTypeLegacy, StorageTypeUnified, StorageTypeUnifiedGrpc: + // no-op + default: + errs = append(errs, fmt.Errorf("--grafana-apiserver-storage-type must be one of %s, %s, %s, %s, %s", StorageTypeFile, StorageTypeEtcd, StorageTypeLegacy, StorageTypeUnified, StorageTypeUnifiedGrpc)) + } + return errs +} + +func (o *StorageOptions) ApplyTo(serverConfig *genericapiserver.RecommendedConfig, etcdOptions *options.EtcdOptions) error { + // TODO: move storage setup here + return nil +} diff --git a/pkg/services/grafana-apiserver/registry/generic/strategy.go b/pkg/services/apiserver/registry/generic/strategy.go similarity index 100% rename from pkg/services/grafana-apiserver/registry/generic/strategy.go rename to pkg/services/apiserver/registry/generic/strategy.go diff --git a/pkg/services/grafana-apiserver/rest/dualwriter.go b/pkg/services/apiserver/rest/dualwriter.go similarity index 100% rename from pkg/services/grafana-apiserver/rest/dualwriter.go rename to pkg/services/apiserver/rest/dualwriter.go diff --git a/pkg/services/grafana-apiserver/service.go b/pkg/services/apiserver/service.go similarity index 63% rename from pkg/services/grafana-apiserver/service.go rename to pkg/services/apiserver/service.go index 84447d9db2f..3f47779488c 100644 --- a/pkg/services/grafana-apiserver/service.go +++ b/pkg/services/apiserver/service.go @@ -1,4 +1,4 @@ -package grafanaapiserver +package apiserver import ( "context" @@ -6,13 +6,8 @@ import ( "net/http" "net/http/httptest" "path" - "runtime/debug" - "strconv" - "strings" - "github.com/go-logr/logr" "github.com/grafana/dskit/services" - "golang.org/x/mod/semver" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,15 +16,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apiserver/pkg/endpoints/responsewriter" genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/options" clientrest "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "k8s.io/component-base/logs" - "k8s.io/klog/v2" - - "github.com/grafana/grafana/pkg/services/grafana-apiserver/auth/authorizer" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" - "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/infra/appcontext" @@ -38,26 +26,21 @@ import ( "github.com/grafana/grafana/pkg/middleware" "github.com/grafana/grafana/pkg/modules" "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/services/apiserver/auth/authorizer" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + grafanaapiserveroptions "github.com/grafana/grafana/pkg/services/apiserver/options" + entitystorage "github.com/grafana/grafana/pkg/services/apiserver/storage/entity" + filestorage "github.com/grafana/grafana/pkg/services/apiserver/storage/file" + "github.com/grafana/grafana/pkg/services/apiserver/utils" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/featuremgmt" - entitystorage "github.com/grafana/grafana/pkg/services/grafana-apiserver/storage/entity" - filestorage "github.com/grafana/grafana/pkg/services/grafana-apiserver/storage/file" + "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/store/entity" "github.com/grafana/grafana/pkg/services/store/entity/db/dbimpl" "github.com/grafana/grafana/pkg/services/store/entity/sqlstash" "github.com/grafana/grafana/pkg/setting" ) -type StorageType string - -const ( - StorageTypeFile StorageType = "file" - StorageTypeEtcd StorageType = "etcd" - StorageTypeLegacy StorageType = "legacy" - StorageTypeUnified StorageType = "unified" - StorageTypeUnifiedGrpc StorageType = "unified-grpc" -) - var ( _ Service = (*service)(nil) _ RestConfigProvider = (*service)(nil) @@ -90,10 +73,6 @@ type Service interface { registry.CanBeDisabled } -type APIRegistrar interface { - RegisterAPI(builder APIGroupBuilder) -} - type RestConfigProvider interface { GetRestConfig() *clientrest.Config } @@ -111,7 +90,7 @@ type DirectRestConfigProvider interface { type service struct { *services.BasicService - config *config + options *grafanaapiserveroptions.Options restConfig *clientrest.Config cfg *setting.Cfg @@ -123,7 +102,7 @@ type service struct { db db.DB rr routing.RouteRegister handler http.Handler - builders []APIGroupBuilder + builders []builder.APIGroupBuilder tracing *tracing.TracingService @@ -139,12 +118,11 @@ func ProvideService( db db.DB, ) (*service, error) { s := &service{ - config: newConfig(cfg, features), cfg: cfg, features: features, rr: rr, stopCh: make(chan struct{}), - builders: []APIGroupBuilder{}, + builders: []builder.APIGroupBuilder{}, authorizer: authorizer.NewGrafanaAuthorizer(cfg, orgService), tracing: tracing, db: db, // For Unified storage @@ -182,7 +160,9 @@ func ProvideService( } s.rr.Group("/apis", proxyHandler) - s.rr.Group("/apiserver-metrics", proxyHandler) + s.rr.Group("/livez", proxyHandler) + s.rr.Group("/readyz", proxyHandler) + s.rr.Group("/healthz", proxyHandler) s.rr.Group("/openapi", proxyHandler) return s, nil @@ -193,7 +173,7 @@ func (s *service) GetRestConfig() *clientrest.Config { } func (s *service) IsDisabled() bool { - return !s.config.enabled + return false } // Run is an adapter for the BackgroundService interface. @@ -204,17 +184,11 @@ func (s *service) Run(ctx context.Context) error { return s.running(ctx) } -func (s *service) RegisterAPI(builder APIGroupBuilder) { - s.builders = append(s.builders, builder) +func (s *service) RegisterAPI(b builder.APIGroupBuilder) { + s.builders = append(s.builders, b) } func (s *service) start(ctx context.Context) error { - logger := logr.New(newLogAdapter(s.config.logLevel)) - klog.SetLoggerWithOptions(logger, klog.ContextualLogger(true)) - if _, err := logs.GlogSetter(strconv.Itoa(s.config.logLevel)); err != nil { - logger.Error(err, "failed to set log level") - } - // Get the list of groups the server will support builders := s.builders @@ -226,59 +200,42 @@ func (s *service) start(ctx context.Context) error { return err } - // Optionally register a custom authorizer auth := b.GetAuthorizer() if auth != nil { s.authorizer.Register(b.GetGroupVersion(), auth) } } - o := options.NewRecommendedOptions("/registry/grafana.app", Codecs.LegacyCodec(groupVersions...)) - o.SecureServing.BindAddress = s.config.ip - o.SecureServing.BindPort = s.config.port - o.Authentication.RemoteKubeConfigFileOptional = true - o.Authorization.RemoteKubeConfigFileOptional = true + o := grafanaapiserveroptions.NewOptions(Codecs.LegacyCodec(groupVersions...)) + applyGrafanaConfig(s.cfg, s.features, o) - o.Admission = nil - o.CoreAPI = nil - - serverConfig := genericapiserver.NewRecommendedConfig(Codecs) - serverConfig.ExternalAddress = s.config.host - - if s.config.devMode { - // SecureServingOptions is used when the apiserver needs it's own listener. - // this is not needed in production, but it's useful for development kubectl access. - if err := o.SecureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil { - return err - } - // AuthenticationOptions is needed to authenticate requests from kubectl in dev mode. - if err := o.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, serverConfig.OpenAPIConfig); err != nil { - return err - } - } else { - // In production mode, override ExternalAddress and LoopbackClientConfig. - // In dev mode we want to use the loopback client config - // and address provided by SecureServingOptions. - serverConfig.ExternalAddress = s.config.host - serverConfig.LoopbackClientConfig = &clientrest.Config{ - Host: s.config.apiURL, - TLSClientConfig: clientrest.TLSClientConfig{ - Insecure: true, - }, - } + if errs := o.Validate(); len(errs) != 0 { + // TODO: handle multiple errors + return errs[0] } - switch s.config.storageType { - case StorageTypeEtcd: - o.Etcd.StorageConfig.Transport.ServerList = s.config.etcdServers - if err := o.Etcd.Validate(); len(err) > 0 { + serverConfig := genericapiserver.NewRecommendedConfig(Codecs) + if err := o.ApplyTo(serverConfig); err != nil { + return err + } + serverConfig.Authorization.Authorizer = s.authorizer + serverConfig.TracerProvider = s.tracing.GetTracerProvider() + + // setup loopback transport + transport := &roundTripperFunc{ready: make(chan struct{})} + serverConfig.LoopbackClientConfig.Transport = transport + serverConfig.LoopbackClientConfig.TLSClientConfig = clientrest.TLSClientConfig{} + + switch o.StorageOptions.StorageType { + case grafanaapiserveroptions.StorageTypeEtcd: + if err := o.RecommendedOptions.Etcd.Validate(); len(err) > 0 { return err[0] } - if err := o.Etcd.ApplyTo(&serverConfig.Config); err != nil { + if err := o.RecommendedOptions.Etcd.ApplyTo(&serverConfig.Config); err != nil { return err } - case StorageTypeUnified: + case grafanaapiserveroptions.StorageTypeUnified: if !s.features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorage) { return fmt.Errorf("unified storage requires the unifiedStorage feature flag (and app_mode = development)") } @@ -293,9 +250,9 @@ func (s *service) start(ctx context.Context) error { return err } - serverConfig.Config.RESTOptionsGetter = entitystorage.NewRESTOptionsGetter(s.cfg, store, o.Etcd.StorageConfig.Codec) + serverConfig.Config.RESTOptionsGetter = entitystorage.NewRESTOptionsGetter(s.cfg, store, o.RecommendedOptions.Etcd.StorageConfig.Codec) - case StorageTypeUnifiedGrpc: + case grafanaapiserveroptions.StorageTypeUnifiedGrpc: // Create a connection to the gRPC server // TODO: support configuring the gRPC server address conn, err := grpc.Dial("localhost:10000", grpc.WithTransportCredentials(insecure.NewCredentials())) @@ -309,20 +266,16 @@ func (s *service) start(ctx context.Context) error { // Create a client instance store := entity.NewEntityStoreClientWrapper(conn) - serverConfig.Config.RESTOptionsGetter = entitystorage.NewRESTOptionsGetter(s.cfg, store, o.Etcd.StorageConfig.Codec) + serverConfig.Config.RESTOptionsGetter = entitystorage.NewRESTOptionsGetter(s.cfg, store, o.RecommendedOptions.Etcd.StorageConfig.Codec) - case StorageTypeFile: - serverConfig.RESTOptionsGetter = filestorage.NewRESTOptionsGetter(s.config.dataPath, o.Etcd.StorageConfig) - - case StorageTypeLegacy: - // do nothing? + case grafanaapiserveroptions.StorageTypeLegacy: + fallthrough + case grafanaapiserveroptions.StorageTypeFile: + serverConfig.RESTOptionsGetter = filestorage.NewRESTOptionsGetter(o.StorageOptions.DataPath, o.RecommendedOptions.Etcd.StorageConfig) } - serverConfig.Authorization.Authorizer = s.authorizer - serverConfig.TracerProvider = s.tracing.GetTracerProvider() - // Add OpenAPI specs for each group+version - err := SetupConfig(serverConfig, builders) + err := builder.SetupConfig(Scheme, serverConfig, builders) if err != nil { return err } @@ -339,28 +292,37 @@ func (s *service) start(ctx context.Context) error { return err } + dualWriteEnabled := o.StorageOptions.StorageType != grafanaapiserveroptions.StorageTypeLegacy + // Install the API Group+version - err = InstallAPIs(server, serverConfig.RESTOptionsGetter, builders) + err = builder.InstallAPIs(Scheme, Codecs, server, serverConfig.RESTOptionsGetter, builders, dualWriteEnabled) if err != nil { return err } + // set the transport function and signal that it's ready + transport.fn = func(req *http.Request) (*http.Response, error) { + w := newWrappedResponseWriter() + resp := responsewriter.WrapForHTTP1Or2(w) + server.Handler.ServeHTTP(resp, req) + return w.Result(), nil + } + close(transport.ready) + + // only write kubeconfig in dev mode + if o.ExtraOptions.DevMode { + if err := ensureKubeConfig(server.LoopbackClientConfig, o.StorageOptions.DataPath); err != nil { + return err + } + } + // Used by the proxy wrapper registered in ProvideService s.handler = server.Handler s.restConfig = server.LoopbackClientConfig + s.options = o prepared := server.PrepareRun() - // When running in production, do not start a standalone https server - if !s.config.devMode { - return nil - } - - // only write kubeconfig in dev mode - if err := s.ensureKubeConfig(); err != nil { - return err - } - go func() { s.stoppedCh <- prepared.Run(s.stopCh) }() @@ -385,12 +347,6 @@ func (s *service) DirectlyServeHTTP(w http.ResponseWriter, r *http.Request) { } func (s *service) running(ctx context.Context) error { - // skip waiting for the server in prod mode - if !s.config.devMode { - <-ctx.Done() - return nil - } - select { case err := <-s.stoppedCh: if err != nil { @@ -402,47 +358,42 @@ func (s *service) running(ctx context.Context) error { return nil } -func (s *service) ensureKubeConfig() error { +func ensureKubeConfig(restConfig *clientrest.Config, dir string) error { return clientcmd.WriteToFile( - utils.FormatKubeConfig(s.restConfig), - path.Join(s.config.dataPath, "grafana.kubeconfig"), + utils.FormatKubeConfig(restConfig), + path.Join(dir, "grafana.kubeconfig"), ) } type roundTripperFunc struct { - fn func(req *http.Request) (*http.Response, error) + ready chan struct{} + fn func(req *http.Request) (*http.Response, error) } func (f *roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { + if f.fn == nil { + <-f.ready + } return f.fn(req) } -// find the k8s version according to build info -func getK8sApiserverVersion() (string, error) { - bi, ok := debug.ReadBuildInfo() - if !ok { - return "", fmt.Errorf("debug.ReadBuildInfo() failed") - } +var _ http.ResponseWriter = (*wrappedResponseWriter)(nil) +var _ responsewriter.UserProvidedDecorator = (*wrappedResponseWriter)(nil) - if len(bi.Deps) == 0 { - return "v?.?", nil // this is normal while debugging - } - - for _, dep := range bi.Deps { - if dep.Path == "k8s.io/apiserver" { - if !semver.IsValid(dep.Version) { - return "", fmt.Errorf("invalid semantic version for k8s.io/apiserver") - } - // v0 => v1 - majorVersion := strings.TrimPrefix(semver.Major(dep.Version), "v") - majorInt, err := strconv.Atoi(majorVersion) - if err != nil { - return "", fmt.Errorf("could not convert majorVersion to int. majorVersion: %s", majorVersion) - } - newMajor := fmt.Sprintf("v%d", majorInt+1) - return strings.Replace(dep.Version, semver.Major(dep.Version), newMajor, 1), nil - } - } - - return "", fmt.Errorf("could not find k8s.io/apiserver in build info") +type wrappedResponseWriter struct { + *httptest.ResponseRecorder +} + +func newWrappedResponseWriter() *wrappedResponseWriter { + w := httptest.NewRecorder() + return &wrappedResponseWriter{w} +} + +func (w *wrappedResponseWriter) Unwrap() http.ResponseWriter { + return w.ResponseRecorder +} + +func (w *wrappedResponseWriter) CloseNotify() <-chan bool { + // TODO: this is probably not the right thing to do here + return make(<-chan bool) } diff --git a/pkg/services/grafana-apiserver/storage/entity/fieldRequirements.go b/pkg/services/apiserver/storage/entity/fieldRequirements.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/entity/fieldRequirements.go rename to pkg/services/apiserver/storage/entity/fieldRequirements.go diff --git a/pkg/services/grafana-apiserver/storage/entity/restoptions.go b/pkg/services/apiserver/storage/entity/restoptions.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/entity/restoptions.go rename to pkg/services/apiserver/storage/entity/restoptions.go diff --git a/pkg/services/grafana-apiserver/storage/entity/storage.go b/pkg/services/apiserver/storage/entity/storage.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/entity/storage.go rename to pkg/services/apiserver/storage/entity/storage.go diff --git a/pkg/services/grafana-apiserver/storage/entity/utils.go b/pkg/services/apiserver/storage/entity/utils.go similarity index 98% rename from pkg/services/grafana-apiserver/storage/entity/utils.go rename to pkg/services/apiserver/storage/entity/utils.go index b20e40c9ebe..22b122e43e6 100644 --- a/pkg/services/grafana-apiserver/storage/entity/utils.go +++ b/pkg/services/apiserver/storage/entity/utils.go @@ -15,7 +15,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/endpoints/request" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" + "github.com/grafana/grafana/pkg/services/apiserver/utils" entityStore "github.com/grafana/grafana/pkg/services/store/entity" ) diff --git a/pkg/services/grafana-apiserver/storage/entity/utils_test.go b/pkg/services/apiserver/storage/entity/utils_test.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/entity/utils_test.go rename to pkg/services/apiserver/storage/entity/utils_test.go diff --git a/pkg/services/grafana-apiserver/storage/file/file.go b/pkg/services/apiserver/storage/file/file.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/file/file.go rename to pkg/services/apiserver/storage/file/file.go diff --git a/pkg/services/grafana-apiserver/storage/file/restoptions.go b/pkg/services/apiserver/storage/file/restoptions.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/file/restoptions.go rename to pkg/services/apiserver/storage/file/restoptions.go diff --git a/pkg/services/grafana-apiserver/storage/file/util.go b/pkg/services/apiserver/storage/file/util.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/file/util.go rename to pkg/services/apiserver/storage/file/util.go diff --git a/pkg/services/grafana-apiserver/storage/file/watchset.go b/pkg/services/apiserver/storage/file/watchset.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/file/watchset.go rename to pkg/services/apiserver/storage/file/watchset.go diff --git a/pkg/services/grafana-apiserver/utils/clientConfig.go b/pkg/services/apiserver/utils/clientConfig.go similarity index 100% rename from pkg/services/grafana-apiserver/utils/clientConfig.go rename to pkg/services/apiserver/utils/clientConfig.go diff --git a/pkg/services/grafana-apiserver/utils/meta.go b/pkg/services/apiserver/utils/meta.go similarity index 100% rename from pkg/services/grafana-apiserver/utils/meta.go rename to pkg/services/apiserver/utils/meta.go diff --git a/pkg/services/grafana-apiserver/utils/meta_test.go b/pkg/services/apiserver/utils/meta_test.go similarity index 98% rename from pkg/services/grafana-apiserver/utils/meta_test.go rename to pkg/services/apiserver/utils/meta_test.go index 9d9cfedd3ce..e8555bdff6e 100644 --- a/pkg/services/grafana-apiserver/utils/meta_test.go +++ b/pkg/services/apiserver/utils/meta_test.go @@ -8,7 +8,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" + "github.com/grafana/grafana/pkg/services/apiserver/utils" ) type TestResource struct { diff --git a/pkg/services/grafana-apiserver/utils/tableConverter.go b/pkg/services/apiserver/utils/tableConverter.go similarity index 100% rename from pkg/services/grafana-apiserver/utils/tableConverter.go rename to pkg/services/apiserver/utils/tableConverter.go diff --git a/pkg/services/grafana-apiserver/utils/tableConverter_test.go b/pkg/services/apiserver/utils/tableConverter_test.go similarity index 98% rename from pkg/services/grafana-apiserver/utils/tableConverter_test.go rename to pkg/services/apiserver/utils/tableConverter_test.go index ec1310475c9..7c7618d4f0d 100644 --- a/pkg/services/grafana-apiserver/utils/tableConverter_test.go +++ b/pkg/services/apiserver/utils/tableConverter_test.go @@ -11,7 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" + "github.com/grafana/grafana/pkg/services/apiserver/utils" ) func TestTableConverter(t *testing.T) { diff --git a/pkg/services/grafana-apiserver/utils/uids.go b/pkg/services/apiserver/utils/uids.go similarity index 100% rename from pkg/services/grafana-apiserver/utils/uids.go rename to pkg/services/apiserver/utils/uids.go diff --git a/pkg/services/grafana-apiserver/wireset.go b/pkg/services/apiserver/wireset.go similarity index 63% rename from pkg/services/grafana-apiserver/wireset.go rename to pkg/services/apiserver/wireset.go index 25a02e898ef..f7163f1ad6f 100644 --- a/pkg/services/grafana-apiserver/wireset.go +++ b/pkg/services/apiserver/wireset.go @@ -1,13 +1,15 @@ -package grafanaapiserver +package apiserver import ( "github.com/google/wire" + + "github.com/grafana/grafana/pkg/services/apiserver/builder" ) var WireSet = wire.NewSet( ProvideService, wire.Bind(new(RestConfigProvider), new(*service)), wire.Bind(new(Service), new(*service)), - wire.Bind(new(APIRegistrar), new(*service)), wire.Bind(new(DirectRestConfigProvider), new(*service)), + wire.Bind(new(builder.APIRegistrar), new(*service)), ) diff --git a/pkg/services/grafana-apiserver/config.go b/pkg/services/grafana-apiserver/config.go deleted file mode 100644 index df30eb12eb9..00000000000 --- a/pkg/services/grafana-apiserver/config.go +++ /dev/null @@ -1,60 +0,0 @@ -package grafanaapiserver - -import ( - "fmt" - "net" - "path/filepath" - "strconv" - - "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/setting" -) - -type config struct { - enabled bool - devMode bool - - ip net.IP - port int - host string - apiURL string - - storageType StorageType - - etcdServers []string - dataPath string - - logLevel int -} - -func newConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles) *config { - defaultLogLevel := 0 - ip := net.ParseIP(cfg.HTTPAddr) - apiURL := cfg.AppURL - port, err := strconv.Atoi(cfg.HTTPPort) - if err != nil { - port = 3000 - } - - if cfg.Env == setting.Dev { - defaultLogLevel = 10 - port = 6443 - ip = net.ParseIP("127.0.0.1") - apiURL = fmt.Sprintf("https://%s:%d", ip, port) - } - - host := fmt.Sprintf("%s:%d", ip, port) - - return &config{ - enabled: true, - devMode: features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerEnsureKubectlAccess), - dataPath: filepath.Join(cfg.DataPath, "grafana-apiserver"), - ip: ip, - port: port, - host: host, - logLevel: cfg.SectionWithEnvOverrides("grafana-apiserver").Key("log_level").MustInt(defaultLogLevel), - etcdServers: cfg.SectionWithEnvOverrides("grafana-apiserver").Key("etcd_servers").Strings(","), - storageType: StorageType(cfg.SectionWithEnvOverrides("grafana-apiserver").Key("storage_type").MustString(string(StorageTypeLegacy))), - apiURL: apiURL, - } -} diff --git a/pkg/services/grafana-apiserver/config_test.go b/pkg/services/grafana-apiserver/config_test.go deleted file mode 100644 index e76fe5e7f93..00000000000 --- a/pkg/services/grafana-apiserver/config_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package grafanaapiserver - -import ( - "net" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/setting" -) - -func TestNewConfig(t *testing.T) { - cfg := setting.NewCfg() - cfg.Env = setting.Prod - cfg.DataPath = "/tmp/grafana" - cfg.HTTPAddr = "10.0.0.1" - cfg.HTTPPort = "4000" - cfg.AppURL = "http://test:4000" - - section := cfg.Raw.Section("grafana-apiserver") - section.Key("log_level").SetValue("5") - section.Key("etcd_servers").SetValue("http://localhost:2379") - - actual := newConfig(cfg, featuremgmt.WithFeatures()) - - expected := &config{ - enabled: true, - devMode: false, - storageType: StorageTypeLegacy, - etcdServers: []string{"http://localhost:2379"}, - apiURL: "http://test:4000", - ip: net.ParseIP("10.0.0.1"), - port: 4000, - host: "10.0.0.1:4000", - dataPath: "/tmp/grafana/grafana-apiserver", - logLevel: 5, - } - require.Equal(t, expected, actual) -} diff --git a/pkg/tests/apis/helper.go b/pkg/tests/apis/helper.go index 3d1e233174d..8faebfdfda7 100644 --- a/pkg/tests/apis/helper.go +++ b/pkg/tests/apis/helper.go @@ -25,9 +25,9 @@ import ( "github.com/grafana/grafana/pkg/infra/localcache" "github.com/grafana/grafana/pkg/server" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/auth/identity" "github.com/grafana/grafana/pkg/services/datasources" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/org/orgimpl" "github.com/grafana/grafana/pkg/services/quota/quotaimpl"