K8s: Add Aggregation to Backend Service (#81591)

Co-authored-by: Charandas Batra <charandas.batra@grafana.com>

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -173,6 +173,7 @@ Experimental features might be changed or removed without prior notice.
 | `promQLScope`                               | In-development feature that will allow injection of labels into prometheus queries.                                                                                                                                                                                               |
 | `nodeGraphDotLayout`                        | Changed the layout algorithm for the node graph                                                                                                                                                                                                                                   |
 | `newPDFRendering`                           | New implementation for the dashboard to PDF rendering                                                                                                                                                                                                                             |
+| `kubernetesAggregator`                      | Enable grafana aggregator                                                                                                                                                                                                                                                         |
 
 ## Development feature toggles
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -176,4 +176,5 @@ export interface FeatureToggles {
   nodeGraphDotLayout?: boolean;
   groupToNestedTableTransformation?: boolean;
   newPDFRendering?: boolean;
+  kubernetesAggregator?: boolean;
 }

pkg/registry/apis/service/register.go

@@ -26,6 +26,10 @@ func NewServiceAPIBuilder() *ServiceAPIBuilder {
 }
 
 func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *ServiceAPIBuilder {
+	if !features.IsEnabledGlobally(featuremgmt.FlagKubernetesAggregator) {
+		return nil // skip registration unless opting into aggregator mode
+	}
+
 	builder := NewServiceAPIBuilder()
 	apiregistration.RegisterAPI(NewServiceAPIBuilder())
 	return builder

pkg/services/apiserver/README.md

@@ -4,7 +4,6 @@
 
 ```ini
 [feature_toggles]
-grafanaAPIServer = true
 kubernetesPlaylists = true
 ```
 
@@ -51,6 +50,10 @@ data/grafana-apiserver
             └── hi.json
 ```
 
+## Enable aggregation
+
+See [aggregator/README.md](./aggregator/README.md) for more information.
+
 ### `kubectl` access
 
 For kubectl to work, grafana needs to run over https.  To simplify development, you can use:
@@ -59,7 +62,6 @@ For kubectl to work, grafana needs to run over https.  To simplify development,
 app_mode = development
 
 [feature_toggles]
-grafanaAPIServer = true
 grafanaAPIServerEnsureKubectlAccess = true 
 kubernetesPlaylists = true
 ```

pkg/services/apiserver/aggregator/README.md

@@ -15,10 +15,13 @@ roll out features for each service without downtime.
 To read more about the concept, see 
 [here](https://kubernetes.io/docs/tasks/extend-kubernetes/setup-extension-api-server/).
 
-Note that, this aggregation will be a totally internal detail to Grafana. External fully functional APIServers that
-may themselves act as parent API Servers to Grafana will never be made aware of them. Any of the `APIService` 
-related to Grafana Groups registered in a real K8s environment will take the address of Grafana's 
-parent server (which will bundle grafana-aggregator).
+Note that this aggregation will be a totally internal detail to Grafana. External fully functional API Servers that
+may themselves act as parent API Servers to Grafana will never be made aware of internal Grafana API Servers.
+Thus, any `APIService` objects corresponding to Grafana's API groups will take the address of 
+Grafana's main API Server (the one that bundles grafana-aggregator).
+
+Also, note that the single binary OSS offering of Grafana doesn't make use of the aggregator component at all, instead
+opting for local installation of all the Grafana API groups.
 
 ### kube-aggregator versus grafana-aggregator
 
@@ -41,7 +44,65 @@ live under that instead.
 
 ### Gotchas (Pay Attention)
 
-1. `grafana-aggregator` uses file storage under `data/grafana-aggregator` (`apiregistration.k8s.io`,
-`service.grafana.app`) and `data/grafana-apiextensions` (`apiextensions.k8s.io`).
-2. Since `grafana-aggregator` outputs configuration (TLS and kubeconfig) that is used in the invocation of aggregated
-  servers, ensure you start the aggregated service after launching the aggregator during local development.
+1. `grafana-aggregator` uses file storage under `data/grafana-apiserver` (`apiregistration.k8s.io`,
+`service.grafana.app`). Thus, any restarts will still have any prior configured aggregation in effect.
+2. During local development, ensure you start the aggregated service after launching the aggregator. This is
+so you have TLS and kubeconfig available for use with example aggregated api servers.
+3. Ensure you have `grafanaAPIServerWithExperimentalAPIs = false` in your custom.ini. Otherwise, the example
+service the following guide uses for the aggregation test is bundled as a `Local` `APIService` and will cause
+configuration overwrites on startup.
+
+## Testing aggregation locally
+
+1. Generate the PKI using `openssl` (for development purposes, we will use the CN of `system:masters`):
+  ```shell
+  ./hack/make-aggregator-pki.sh
+  ```
+2. Configure the aggregator:
+  ```ini
+  [feature_toggles]
+  grafanaAPIServerEnsureKubectlAccess = true
+  ; disable the experimental APIs flag to disable bundling of the example service locally
+  grafanaAPIServerWithExperimentalAPIs = false
+  kubernetesAggregator = true
+
+  [grafana-apiserver]
+  proxy_client_cert_file = ./data/grafana-aggregator/client.crt
+  proxy_client_key_file = ./data/grafana-aggregator/client.key
+  ```
+3. Start the server
+  ```shell
+  make run
+  ```
+4. In another tab, apply the manifests: 
+  ```shell
+  export KUBECONFIG=$PWD/data/grafana-apiserver/grafana.kubeconfig
+  kubectl apply -f ./pkg/services/apiserver/aggregator/examples/
+  # SAMPLE OUTPUT
+  # apiservice.apiregistration.k8s.io/v0alpha1.example.grafana.app created
+  # externalname.service.grafana.app/example-apiserver created
+  
+  kubectl get apiservice
+  # SAMPLE OUTPUT
+  # NAME                           SERVICE                     AVAILABLE                      AGE
+  # v0alpha1.example.grafana.app   grafana/example-apiserver   False (FailedDiscoveryCheck)   29m
+  ```
+5. In another tab, start the example microservice that will be aggregated by the parent apiserver:
+  ```shell
+  go run ./pkg/cmd/grafana apiserver \
+    --runtime-config=example.grafana.app/v0alpha1=true \
+    --secure-port 7443 \
+    --client-ca-file=$PWD/data/grafana-aggregator/ca.crt
+  ```
+6. After 10 seconds, check `APIService` again. It should report as available.
+  ```shell
+  export KUBECONFIG=$PWD/data/grafana-apiserver/grafana.kubeconfig
+  kubectl get apiservice
+  # SAMPLE OUTPUT
+  # NAME                           SERVICE                     AVAILABLE      AGE
+  # v0alpha1.example.grafana.app   grafana/example-apiserver   True           30m
+  ```
+7. For tear down of the above test:
+  ```shell
+  kubectl delete -f ./pkg/services/apiserver/aggregator/examples/
+  ```

pkg/services/apiserver/aggregator/aggregator.go

@@ -104,9 +104,7 @@ func CreateAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, shared
 	}
 
 	err = aggregatorServer.GenericAPIServer.AddPostStartHook("grafana-apiserver-autoregistration", func(context genericapiserver.PostStartHookContext) error {
-		go func() {
-			autoRegistrationController.Run(5, context.StopCh)
-		}()
+		go autoRegistrationController.Run(5, context.StopCh)
 		return nil
 	})
 	if err != nil {
@@ -198,16 +196,20 @@ func makeAPIServiceAvailableHealthCheck(name string, apiServices []*v1.APIServic
 	}
 
 	// Watch add/update events for APIServices
-	_, _ = apiServiceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+	_, err := apiServiceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc:    func(obj interface{}) { handleAPIServiceChange(obj.(*v1.APIService)) },
 		UpdateFunc: func(old, new interface{}) { handleAPIServiceChange(new.(*v1.APIService)) },
 	})
+	if err != nil {
+		klog.Errorf("Failed to watch APIServices for health check: %v", err)
+	}
 
 	// Don't return healthy until the pending list is empty
 	return healthz.NamedCheck(name, func(r *http.Request) error {
 		pendingServiceNamesLock.RLock()
 		defer pendingServiceNamesLock.RUnlock()
 		if pendingServiceNames.Len() > 0 {
+			klog.Error("APIServices not yet available", "services", pendingServiceNames.List())
 			return fmt.Errorf("missing APIService: %v", pendingServiceNames.List())
 		}
 		return nil

pkg/services/apiserver/aggregator/examples/apiservice.yaml

@@ -0,0 +1,14 @@
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v0alpha1.example.grafana.app
+spec:
+  version: v0alpha1
+  insecureSkipTLSVerify: true
+  group: example.grafana.app
+  groupPriorityMinimum: 1000
+  versionPriority: 15
+  service:
+    name: example-apiserver
+    namespace: grafana
+    port: 7443

pkg/services/apiserver/aggregator/examples/externalname.yaml

@@ -0,0 +1,7 @@
+apiVersion: service.grafana.app/v0alpha1
+kind: ExternalName
+metadata:
+  name: example-apiserver
+  namespace: grafana
+spec:
+  host: localhost

pkg/services/apiserver/auth/authorizer/provider.go

@@ -4,7 +4,9 @@ import (
 	"context"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	k8suser "k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	"k8s.io/apiserver/pkg/authorization/union"
 
 	orgsvc "github.com/grafana/grafana/pkg/services/org"
@@ -21,6 +23,7 @@ type GrafanaAuthorizer struct {
 func NewGrafanaAuthorizer(cfg *setting.Cfg, orgService orgsvc.Service) *GrafanaAuthorizer {
 	authorizers := []authorizer.Authorizer{
 		&impersonationAuthorizer{},
+		authorizerfactory.NewPrivilegedGroups(k8suser.SystemPrivilegedGroup),
 	}
 
 	// In Hosted grafana, the StackID replaces the orgID as a valid namespace

pkg/services/apiserver/config.go

@@ -29,20 +29,25 @@ func applyGrafanaConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles, o
 
 	host := fmt.Sprintf("%s:%d", ip, port)
 
-	o.RecommendedOptions.Etcd.StorageConfig.Transport.ServerList = cfg.SectionWithEnvOverrides("grafana-apiserver").Key("etcd_servers").Strings(",")
+	apiserverCfg := cfg.SectionWithEnvOverrides("grafana-apiserver")
+
+	o.RecommendedOptions.Etcd.StorageConfig.Transport.ServerList = apiserverCfg.Key("etcd_servers").Strings(",")
 
 	o.RecommendedOptions.SecureServing.BindAddress = ip
 	o.RecommendedOptions.SecureServing.BindPort = port
 	o.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true
 	o.RecommendedOptions.Authorization.RemoteKubeConfigFileOptional = true
 
+	o.AggregatorOptions.ProxyClientCertFile = apiserverCfg.Key("proxy_client_cert_file").MustString("")
+	o.AggregatorOptions.ProxyClientKeyFile = apiserverCfg.Key("proxy_client_key_file").MustString("")
+
 	o.RecommendedOptions.Admission = nil
 	o.RecommendedOptions.CoreAPI = nil
 
-	o.StorageOptions.StorageType = options.StorageType(cfg.SectionWithEnvOverrides("grafana-apiserver").Key("storage_type").MustString(string(options.StorageTypeLegacy)))
+	o.StorageOptions.StorageType = options.StorageType(apiserverCfg.Key("storage_type").MustString(string(options.StorageTypeLegacy)))
 	o.StorageOptions.DataPath = filepath.Join(cfg.DataPath, "grafana-apiserver")
 	o.ExtraOptions.DevMode = features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerEnsureKubectlAccess)
 	o.ExtraOptions.ExternalAddress = host
 	o.ExtraOptions.APIURL = apiURL
-	o.ExtraOptions.Verbosity = defaultLogLevel
+	o.ExtraOptions.Verbosity = apiserverCfg.Key("log_level").MustInt(defaultLogLevel)
 }

pkg/services/apiserver/endpoints/responsewriter/responsewriter.go

@@ -0,0 +1,132 @@
+package responsewriter
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net/http"
+
+	"k8s.io/apiserver/pkg/endpoints/responsewriter"
+	"k8s.io/klog/v2"
+)
+
+var _ responsewriter.CloseNotifierFlusher = (*ResponseAdapter)(nil)
+var _ http.ResponseWriter = (*ResponseAdapter)(nil)
+var _ io.ReadCloser = (*ResponseAdapter)(nil)
+
+func WrapHandler(handler http.Handler) func(req *http.Request) (*http.Response, error) {
+	// ignore the lint error because the response is passed directly to the client,
+	// so the client will be responsible for closing the response body.
+	//nolint:bodyclose
+	return func(req *http.Request) (*http.Response, error) {
+		w := NewAdapter(req)
+		resp := w.Response()
+		go func() {
+			handler.ServeHTTP(w, req)
+			if err := w.CloseWriter(); err != nil {
+				klog.Errorf("error closing writer: %v", err)
+			}
+		}()
+		return resp, nil
+	}
+}
+
+// ResponseAdapter is an implementation of [http.ResponseWriter] that allows conversion to a [http.Response].
+type ResponseAdapter struct {
+	req      *http.Request
+	res      *http.Response
+	reader   io.ReadCloser
+	writer   io.WriteCloser
+	buffered *bufio.ReadWriter
+}
+
+// NewAdapter returns an initialized [ResponseAdapter].
+func NewAdapter(req *http.Request) *ResponseAdapter {
+	r, w := io.Pipe()
+	writer := bufio.NewWriter(w)
+	reader := bufio.NewReader(r)
+	buffered := bufio.NewReadWriter(reader, writer)
+	return &ResponseAdapter{
+		req: req,
+		res: &http.Response{
+			Proto:      req.Proto,
+			ProtoMajor: req.ProtoMajor,
+			ProtoMinor: req.ProtoMinor,
+			Header:     make(http.Header),
+		},
+		reader:   r,
+		writer:   w,
+		buffered: buffered,
+	}
+}
+
+// Header implements [http.ResponseWriter].
+// It returns the response headers to mutate within a handler.
+func (ra *ResponseAdapter) Header() http.Header {
+	return ra.res.Header
+}
+
+// Write implements [http.ResponseWriter].
+func (ra *ResponseAdapter) Write(buf []byte) (int, error) {
+	return ra.buffered.Write(buf)
+}
+
+// Read implements [io.Reader].
+func (ra *ResponseAdapter) Read(buf []byte) (int, error) {
+	return ra.buffered.Read(buf)
+}
+
+// WriteHeader implements [http.ResponseWriter].
+func (ra *ResponseAdapter) WriteHeader(code int) {
+	ra.res.StatusCode = code
+	ra.res.Status = fmt.Sprintf("%03d %s", code, http.StatusText(code))
+}
+
+// Flush implements [http.Flusher].
+func (ra *ResponseAdapter) Flush() {
+	if ra.buffered.Writer.Buffered() == 0 {
+		return
+	}
+
+	if err := ra.buffered.Writer.Flush(); err != nil {
+		klog.Error("Error flushing response buffer: ", "error", err)
+	}
+}
+
+// Response returns the [http.Response] generated by the [http.Handler].
+func (ra *ResponseAdapter) Response() *http.Response {
+	// make sure to set the status code to 200 if the request is a watch
+	// this is to ensure that client-go uses a streamwatcher:
+	// https://github.com/kubernetes/client-go/blob/76174b8af8cfd938018b04198595d65b48a69334/rest/request.go#L737
+	if ra.res.StatusCode == 0 && ra.req.URL.Query().Get("watch") == "true" {
+		ra.WriteHeader(http.StatusOK)
+	}
+	ra.res.Body = ra
+	return ra.res
+}
+
+// Decorate implements [responsewriter.UserProvidedDecorator].
+func (ra *ResponseAdapter) Unwrap() http.ResponseWriter {
+	return ra
+}
+
+// CloseNotify implements [http.CloseNotifier].
+func (ra *ResponseAdapter) CloseNotify() <-chan bool {
+	ch := make(chan bool)
+	go func() {
+		<-ra.req.Context().Done()
+		ch <- true
+	}()
+	return ch
+}
+
+// Close implements [io.Closer].
+func (ra *ResponseAdapter) Close() error {
+	return ra.reader.Close()
+}
+
+// CloseWriter should be called after the http.Handler has returned.
+func (ra *ResponseAdapter) CloseWriter() error {
+	ra.Flush()
+	return ra.writer.Close()
+}

pkg/services/apiserver/endpoints/responsewriter/responsewriter_test.go

@@ -0,0 +1,136 @@
+package responsewriter_test
+
+import (
+	"io"
+	"math/rand"
+	"net/http"
+	"testing"
+	"time"
+
+	grafanaresponsewriter "github.com/grafana/grafana/pkg/services/apiserver/endpoints/responsewriter"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResponseAdapter(t *testing.T) {
+	t.Run("should handle synchronous write", func(t *testing.T) {
+		client := &http.Client{
+			Transport: &roundTripperFunc{
+				ready: make(chan struct{}),
+				// ignore the lint error because the response is passed directly to the client,
+				// so the client will be responsible for closing the response body.
+				//nolint:bodyclose
+				fn: grafanaresponsewriter.WrapHandler(http.HandlerFunc(syncHandler)),
+			},
+		}
+		close(client.Transport.(*roundTripperFunc).ready)
+		req, err := http.NewRequest("GET", "http://localhost/test", nil)
+		require.NoError(t, err)
+
+		resp, err := client.Do(req)
+		require.NoError(t, err)
+
+		defer func() {
+			err := resp.Body.Close()
+			require.NoError(t, err)
+		}()
+
+		bodyBytes, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		require.Equal(t, "OK", string(bodyBytes))
+	})
+
+	t.Run("should handle synchronous write", func(t *testing.T) {
+		generateRandomStrings(10)
+		client := &http.Client{
+			Transport: &roundTripperFunc{
+				ready: make(chan struct{}),
+				// ignore the lint error because the response is passed directly to the client,
+				// so the client will be responsible for closing the response body.
+				//nolint:bodyclose
+				fn: grafanaresponsewriter.WrapHandler(http.HandlerFunc(asyncHandler)),
+			},
+		}
+		close(client.Transport.(*roundTripperFunc).ready)
+		req, err := http.NewRequest("GET", "http://localhost/test?watch=true", nil)
+		require.NoError(t, err)
+
+		resp, err := client.Do(req)
+		require.NoError(t, err)
+
+		defer func() {
+			err := resp.Body.Close()
+			require.NoError(t, err)
+		}()
+
+		// ensure that watch request is a 200
+		require.Equal(t, http.StatusOK, resp.StatusCode)
+
+		// limit to 100 bytes to test the reader buffer
+		buf := make([]byte, 100)
+		// holds the read bytes between iterations
+		cache := []byte{}
+
+		for i := 0; i < 10; {
+			n, err := resp.Body.Read(buf)
+			require.NoError(t, err)
+			if n == 0 {
+				continue
+			}
+			cache = append(cache, buf[:n]...)
+
+			if len(cache) >= len(randomStrings[i]) {
+				str := cache[:len(randomStrings[i])]
+				require.Equal(t, randomStrings[i], string(str))
+				cache = cache[len(randomStrings[i]):]
+				i++
+			}
+		}
+	})
+}
+
+func syncHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	_, _ = w.Write([]byte("OK"))
+}
+
+func asyncHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	for _, s := range randomStrings {
+		time.Sleep(100 * time.Millisecond)
+		// write the current iteration
+		_, _ = w.Write([]byte(s))
+		w.(http.Flusher).Flush()
+	}
+}
+
+var randomStrings = []string{}
+
+func generateRandomStrings(n int) {
+	for i := 0; i < n; i++ {
+		randomString := generateRandomString(1000 * (i + 1))
+		randomStrings = append(randomStrings, randomString)
+	}
+}
+
+func generateRandomString(n int) string {
+	gen := rand.New(rand.NewSource(time.Now().UnixNano()))
+	var chars = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = chars[gen.Intn(len(chars))]
+	}
+	return string(b)
+}
+
+type roundTripperFunc struct {
+	ready chan struct{}
+	fn    func(req *http.Request) (*http.Response, error)
+}
+
+func (f *roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	if f.fn == nil {
+		<-f.ready
+	}
+	res, err := f.fn(req)
+	return res, err
+}

pkg/services/apiserver/options/aggregator.go

@@ -97,6 +97,9 @@ func (o *AggregatorServerOptions) ApplyTo(aggregatorConfig *aggregatorapiserver.
 	}
 	genericConfig.MergedResourceConfig = mergedResourceConfig
 
+	aggregatorConfig.ExtraConfig.ProxyClientCertFile = o.ProxyClientCertFile
+	aggregatorConfig.ExtraConfig.ProxyClientKeyFile = o.ProxyClientKeyFile
+
 	namer := openapinamer.NewDefinitionNamer(aggregatorscheme.Scheme)
 	genericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(o.getMergedOpenAPIDefinitions, namer)
 	genericConfig.OpenAPIV3Config.Info.Title = "Kubernetes"

pkg/services/apiserver/options/options.go

@@ -75,6 +75,9 @@ func (o *Options) Validate() []error {
 func (o *Options) ApplyTo(serverConfig *genericapiserver.RecommendedConfig) error {
 	serverConfig.AggregatedDiscoveryGroupManager = aggregated.NewResourceManager("apis")
 
+	// avoid picking up an in-cluster service account token
+	o.RecommendedOptions.Authentication.SkipInClusterLookup = true
+
 	if err := o.ExtraOptions.ApplyTo(serverConfig); err != nil {
 		return err
 	}
@@ -87,12 +90,8 @@ func (o *Options) ApplyTo(serverConfig *genericapiserver.RecommendedConfig) erro
 		return err
 	}
 
-	if o.ExtraOptions.DevMode {
-		// NOTE: Only consider authn for dev mode - resolves the failure due to missing extension apiserver auth-config
-		// in parent k8s
-		if err := o.RecommendedOptions.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, serverConfig.OpenAPIConfig); err != nil {
-			return err
-		}
+	if err := o.RecommendedOptions.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, serverConfig.OpenAPIConfig); err != nil {
+		return err
 	}
 
 	if !o.ExtraOptions.DevMode {

pkg/services/apiserver/service.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"net/http/httptest"
 	"path"
 
 	"github.com/grafana/dskit/services"
@@ -26,8 +25,10 @@ import (
 	"github.com/grafana/grafana/pkg/middleware"
 	"github.com/grafana/grafana/pkg/modules"
 	"github.com/grafana/grafana/pkg/registry"
+	"github.com/grafana/grafana/pkg/services/apiserver/aggregator"
 	"github.com/grafana/grafana/pkg/services/apiserver/auth/authorizer"
 	"github.com/grafana/grafana/pkg/services/apiserver/builder"
+	grafanaresponsewriter "github.com/grafana/grafana/pkg/services/apiserver/endpoints/responsewriter"
 	grafanaapiserveroptions "github.com/grafana/grafana/pkg/services/apiserver/options"
 	entitystorage "github.com/grafana/grafana/pkg/services/apiserver/storage/entity"
 	filestorage "github.com/grafana/grafana/pkg/services/apiserver/storage/file"
@@ -189,12 +190,17 @@ func (s *service) start(ctx context.Context) error {
 
 	groupVersions := make([]schema.GroupVersion, 0, len(builders))
 	// Install schemas
-	for _, b := range builders {
+	for i, b := range builders {
 		groupVersions = append(groupVersions, b.GetGroupVersion())
 		if err := b.InstallSchema(Scheme); err != nil {
 			return err
 		}
 
+		if s.features.IsEnabledGlobally(featuremgmt.FlagKubernetesAggregator) {
+			// set the priority for the group+version
+			aggregator.APIVersionPriorities[b.GetGroupVersion()] = aggregator.Priority{Group: 15000, Version: int32(i + 1)}
+		}
+
 		auth := b.GetAuthorizer()
 		if auth != nil {
 			s.authorizer.Register(b.GetGroupVersion(), auth)
@@ -216,7 +222,7 @@ func (s *service) start(ctx context.Context) error {
 	serverConfig.Authorization.Authorizer = s.authorizer
 	serverConfig.TracerProvider = s.tracing.GetTracerProvider()
 
-	// setup loopback transport
+	// setup loopback transport for the aggregator server
 	transport := &roundTripperFunc{ready: make(chan struct{})}
 	serverConfig.LoopbackClientConfig.Transport = transport
 	serverConfig.LoopbackClientConfig.TLSClientConfig = clientrest.TLSClientConfig{}
@@ -283,41 +289,100 @@ func (s *service) start(ctx context.Context) error {
 		return err
 	}
 
+	// dual writing is only enabled when the storage type is not legacy.
+	// this is needed to support setting a default RESTOptionsGetter for new APIs that don't
+	// support the legacy storage type.
 	dualWriteEnabled := o.StorageOptions.StorageType != grafanaapiserveroptions.StorageTypeLegacy
 
-	// Install the API Group+version
+	// Install the API group+version
 	err = builder.InstallAPIs(Scheme, Codecs, server, serverConfig.RESTOptionsGetter, builders, dualWriteEnabled)
 	if err != nil {
 		return err
 	}
 
-	// set the transport function and signal that it's ready
-	transport.fn = func(req *http.Request) (*http.Response, error) {
-		w := newWrappedResponseWriter()
-		resp := responsewriter.WrapForHTTP1Or2(w)
-		server.Handler.ServeHTTP(resp, req)
-		return w.Result(), nil
-	}
-	close(transport.ready)
+	// stash the options for later use
+	s.options = o
 
-	// only write kubeconfig in dev mode
-	if o.ExtraOptions.DevMode {
-		if err := ensureKubeConfig(server.LoopbackClientConfig, o.StorageOptions.DataPath); err != nil {
+	var runningServer *genericapiserver.GenericAPIServer
+	if s.features.IsEnabledGlobally(featuremgmt.FlagKubernetesAggregator) {
+		runningServer, err = s.startAggregator(transport, serverConfig, server)
+		if err != nil {
+			return err
+		}
+	} else {
+		runningServer, err = s.startCoreServer(transport, serverConfig, server)
+		if err != nil {
 			return err
 		}
 	}
 
-	// Used by the proxy wrapper registered in ProvideService
-	s.handler = server.Handler
-	s.restConfig = server.LoopbackClientConfig
-	s.options = o
+	// only write kubeconfig in dev mode
+	if o.ExtraOptions.DevMode {
+		if err := ensureKubeConfig(runningServer.LoopbackClientConfig, o.StorageOptions.DataPath); err != nil {
+			return err
+		}
+	}
+
+	// used by the proxy wrapper registered in ProvideService
+	s.handler = runningServer.Handler
+	// used by local clients to make requests to the server
+	s.restConfig = runningServer.LoopbackClientConfig
+
+	return nil
+}
+
+func (s *service) startCoreServer(
+	transport *roundTripperFunc,
+	serverConfig *genericapiserver.RecommendedConfig,
+	server *genericapiserver.GenericAPIServer,
+) (*genericapiserver.GenericAPIServer, error) {
+	// setup the loopback transport and signal that it's ready.
+	// ignore the lint error because the response is passed directly to the client,
+	// so the client will be responsible for closing the response body.
+	// nolint:bodyclose
+	transport.fn = grafanaresponsewriter.WrapHandler(server.Handler)
+	close(transport.ready)
 
 	prepared := server.PrepareRun()
+	go func() {
+		s.stoppedCh <- prepared.Run(s.stopCh)
+	}()
+
+	return server, nil
+}
+
+func (s *service) startAggregator(
+	transport *roundTripperFunc,
+	serverConfig *genericapiserver.RecommendedConfig,
+	server *genericapiserver.GenericAPIServer,
+) (*genericapiserver.GenericAPIServer, error) {
+	aggregatorConfig, aggregatorInformers, err := aggregator.CreateAggregatorConfig(s.options, *serverConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	aggregatorServer, err := aggregator.CreateAggregatorServer(aggregatorConfig, aggregatorInformers, server)
+	if err != nil {
+		return nil, err
+	}
+
+	// setup the loopback transport for the aggregator server and signal that it's ready
+	// ignore the lint error because the response is passed directly to the client,
+	// so the client will be responsible for closing the response body.
+	// nolint:bodyclose
+	transport.fn = grafanaresponsewriter.WrapHandler(aggregatorServer.GenericAPIServer.Handler)
+	close(transport.ready)
+
+	prepared, err := aggregatorServer.PrepareRun()
+	if err != nil {
+		return nil, err
+	}
 
 	go func() {
 		s.stoppedCh <- prepared.Run(s.stopCh)
 	}()
-	return nil
+
+	return aggregatorServer.GenericAPIServer, nil
 }
 
 func (s *service) GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Config {
@@ -325,9 +390,8 @@ func (s *service) GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Co
 		Transport: &roundTripperFunc{
 			fn: func(req *http.Request) (*http.Response, error) {
 				ctx := appcontext.WithUser(req.Context(), c.SignedInUser)
-				w := httptest.NewRecorder()
-				s.handler.ServeHTTP(w, req.WithContext(ctx))
-				return w.Result(), nil
+				wrapped := grafanaresponsewriter.WrapHandler(s.handler)
+				return wrapped(req.WithContext(ctx))
 			},
 		},
 	}
@@ -367,24 +431,3 @@ func (f *roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error)
 	}
 	return f.fn(req)
 }
-
-var _ http.ResponseWriter = (*wrappedResponseWriter)(nil)
-var _ responsewriter.UserProvidedDecorator = (*wrappedResponseWriter)(nil)
-
-type wrappedResponseWriter struct {
-	*httptest.ResponseRecorder
-}
-
-func newWrappedResponseWriter() *wrappedResponseWriter {
-	w := httptest.NewRecorder()
-	return &wrappedResponseWriter{w}
-}
-
-func (w *wrappedResponseWriter) Unwrap() http.ResponseWriter {
-	return w.ResponseRecorder
-}
-
-func (w *wrappedResponseWriter) CloseNotify() <-chan bool {
-	// TODO: this is probably not the right thing to do here
-	return make(<-chan bool)
-}

pkg/services/apiserver/storage/file/file.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/bwmarrin/snowflake"
@@ -57,8 +58,16 @@ var ErrFileNotExists = fmt.Errorf("file doesn't exist")
 // ErrNamespaceNotExists means the directory for the namespace doesn't actually exist.
 var ErrNamespaceNotExists = errors.New("namespace does not exist")
 
+var (
+	node *snowflake.Node
+	once sync.Once
+)
+
 func getResourceVersion() (*uint64, error) {
-	node, err := snowflake.NewNode(1)
+	var err error
+	once.Do(func() {
+		node, err = snowflake.NewNode(1)
+	})
 	if err != nil {
 		return nil, err
 	}

pkg/services/featuremgmt/registry.go

@@ -1176,6 +1176,13 @@ var (
 			Stage:       FeatureStageExperimental,
 			Owner:       grafanaSharingSquad,
 		},
+		{
+			Name:            "kubernetesAggregator",
+			Description:     "Enable grafana aggregator",
+			Stage:           FeatureStageExperimental,
+			Owner:           grafanaAppPlatformSquad,
+			RequiresRestart: true,
+		},
 	}
 )
 

pkg/services/featuremgmt/toggles_gen.csv

@@ -157,3 +157,4 @@ promQLScope,experimental,@grafana/observability-metrics,false,false,false
 nodeGraphDotLayout,experimental,@grafana/observability-traces-and-profiling,false,false,true
 groupToNestedTableTransformation,preview,@grafana/dataviz-squad,false,false,true
 newPDFRendering,experimental,@grafana/sharing-squad,false,false,false
+kubernetesAggregator,experimental,@grafana/grafana-app-platform-squad,false,true,false

pkg/services/featuremgmt/toggles_gen.go

@@ -638,4 +638,8 @@ const (
 	// FlagNewPDFRendering
 	// New implementation for the dashboard to PDF rendering
 	FlagNewPDFRendering = "newPDFRendering"
+
+	// FlagKubernetesAggregator
+	// Enable grafana aggregator
+	FlagKubernetesAggregator = "kubernetesAggregator"
 )