Grafana App Platform: provide an example-apiserver to easily deploy aggregated APIservers (#77826)

.github/CODEOWNERS

@@ -69,6 +69,7 @@
 /pkg/apis/ @grafana/grafana-app-platform-squad
 /pkg/bus/ @grafana/backend-platform
 /pkg/cmd/ @grafana/backend-platform
+/pkg/cmd/grafana-example-apiserver @grafana/grafana-app-platform-squad
 /pkg/components/apikeygen/ @grafana/identity-access-team
 /pkg/components/satokengen/ @grafana/identity-access-team
 /pkg/components/dashdiffs/ @grafana/backend-platform

.gitignore

@@ -78,6 +78,9 @@ public/css/*.min.css
 /data/*
 /bin/*
 
+# any certificates generated by grafana-example-apiserver
+apiserver.local.config/
+
 # devenv
 /devenv/docker-compose.yaml
 /devenv/docker-compose.override.yaml

Makefile

@@ -132,6 +132,10 @@ build-server: ## Build Grafana server.
 	@echo "build server"
 	$(GO) run build.go $(GO_BUILD_FLAGS) build-server
 
+build-example-apiserver: ## Build Grafana example-apiserver application.
+	@echo "build grafana-cli"
+	$(GO) run build.go $(GO_BUILD_FLAGS) build-example-apiserver
+
 build-cli: ## Build Grafana CLI application.
 	@echo "build grafana-cli"
 	$(GO) run build.go $(GO_BUILD_FLAGS) build-cli

pkg/build/cmd.go

@@ -17,12 +17,13 @@ const (
 	GoOSWindows = "windows"
 	GoOSLinux   = "linux"
 
-	BackendBinary = "grafana"
-	ServerBinary  = "grafana-server"
-	CLIBinary     = "grafana-cli"
+	BackendBinary          = "grafana"
+	ExampleAPIServerBinary = "grafana-example-apiserver"
+	ServerBinary           = "grafana-server"
+	CLIBinary              = "grafana-cli"
 )
 
-var binaries = []string{BackendBinary, ServerBinary, CLIBinary}
+var binaries = []string{BackendBinary, ExampleAPIServerBinary, ServerBinary, CLIBinary}
 
 func logError(message string, err error) int {
 	log.Println(message, err)
@@ -85,6 +86,13 @@ func RunCmd() int {
 				return 1
 			}
 
+		case "build-example-apiserver":
+			clean(opts)
+			if err := doBuild("grafana-example-apiserver", "./pkg/cmd/grafana-example-apiserver", opts); err != nil {
+				log.Println(err)
+				return 1
+			}
+
 		case "build-cli":
 			clean(opts)
 			if err := doBuild("grafana-cli", "./pkg/cmd/grafana-cli", opts); err != nil {

pkg/build/grafana/build.go

@@ -12,7 +12,7 @@ import (
 	"github.com/grafana/grafana/pkg/build/golangutils"
 )
 
-var binaries = []string{"grafana", "grafana-server", "grafana-cli"}
+var binaries = []string{"grafana", "grafana-example-apiserver", "grafana-server", "grafana-cli"}
 
 const (
 	SuffixEnterprise2 = "-enterprise2"

pkg/cmd/grafana-example-apiserver/README.md

@@ -0,0 +1,77 @@
+# grafana-example-apiserver
+
+The example-apiserver closely resembles the 
+[sample-apiserver](https://github.com/kubernetes/sample-apiserver/tree/master) project in code and thus
+allows the same
+[CLI flags](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/) as kube-apiserver
+It is currently used for testing our deployment pipelines for aggregated servers.
+
+## Prerequisites:
+1. etcd
+  ```shell
+  brew install etcd
+  ```
+2. kind: you will need kind (or another local K8s setup) if you want to test aggregation.
+  ```
+  go install sigs.k8s.io/kind@v0.20.0 && kind create cluster
+  ```
+
+## Usage
+
+Have `etcd` running in your environment:
+
+```shell
+etcd &
+```
+
+With etcd running, you can now start the example-apiserver. The Authn / Authz flags are set up so that the kind cluster
+can be used as a root server for this example-apiserver (in aggregated mode). Here, it's assumed that you have a local
+kind cluster and that you can provide its kubeconfig in the parameters to the example-apiserver.
+
+```shell
+go run ./pkg/cmd/grafana-example-apiserver \
+  --etcd-servers=http://127.0.0.1:2379 \
+  --authentication-kubeconfig ~/.kube/config \
+  --authorization-kubeconfig ~/.kube/config \
+  --kubeconfig ~/.kube/config \
+  --secure-port 8443
+```
+
+Once, the `example-apiserver` is running, you can configure aggregation against your kind cluster
+by applying a `APIService` and it's corresponding `Service` object. Sample kustomizations are provided
+for local development on [Linux](./deploy/linux/kustomization.yaml) and [macOS](./deploy/darwin/kustomization.yaml).
+
+```shell
+kubectl deploy -k ./deploy/darwin # or /linux
+```
+
+
+## Verify that all works
+
+With kubectl configured against `kind-kind` context, you can run the following:
+
+```shell
+kubectl get --raw /apis/example.grafana.app/v0alpha1 | jq -r
+{
+  "kind": "APIResourceList",
+  "apiVersion": "v1",
+  "groupVersion": "example.grafana.app/v0alpha1",
+  "resources": [
+    {
+      "name": "runtime",
+      "singularName": "runtime",
+      "namespaced": false,
+      "kind": "RuntimeInfo",
+      "verbs": [
+        "list"
+      ]
+    }
+  ]
+}
+```
+
+```shell
+kubectl get apiservice v0alpha1.example.grafana.app
+NAME                           SERVICE                     AVAILABLE   AGE
+v0alpha1.example.grafana.app   grafana/example-apiserver   True        4h1m
+```

pkg/cmd/grafana-example-apiserver/deploy/base/apiservice.yaml

@@ -0,0 +1,14 @@
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v0alpha1.example.grafana.app
+spec:
+  version: v0alpha1
+  insecureSkipTLSVerify: true
+  group: example.grafana.app
+  groupPriorityMinimum: 1000
+  versionPriority: 15
+  service:
+    name: example-apiserver
+    namespace: grafana
+    port: 8443

pkg/cmd/grafana-example-apiserver/deploy/base/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+  - namespace.yaml
+  - apiservice.yaml

pkg/cmd/grafana-example-apiserver/deploy/base/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: grafana

pkg/cmd/grafana-example-apiserver/deploy/darwin/kustomization.yaml

@@ -0,0 +1,5 @@
+namespace: grafana
+
+resources:
+  - ../base
+  - service.yaml

pkg/cmd/grafana-example-apiserver/deploy/darwin/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: example-apiserver
+spec:
+  type: ExternalName
+  externalName: host.docker.internal
+  ports:
+    - port: 8443
+      name: https

pkg/cmd/grafana-example-apiserver/deploy/linux/kustomization.yaml

@@ -0,0 +1,5 @@
+namespace: grafana
+
+resources:
+  - ../base
+  - service.yaml

pkg/cmd/grafana-example-apiserver/deploy/linux/service.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: example-apiserver
+subsets:
+  - addresses:
+      - ip: 172.17.0.1 # this is the gateway IP in the "bridge" docker network
+    ports:
+      - appProtocol: https
+        port: 8443
+        protocol: TCP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: example-apiserver
+spec:
+  ports:
+    - protocol: TCP
+      appProtocol: https
+      port: 8443
+      targetPort: 8443

pkg/cmd/grafana-example-apiserver/main.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/component-base/cli"
+)
+
+func NewCommandStartExampleAPIServer(defaults *ExampleServerOptions, stopCh <-chan struct{}) *cobra.Command {
+	o := *defaults
+	cmd := &cobra.Command{
+		Short: "Launch the example API server",
+		Long:  "Launch the example API server",
+		RunE: func(c *cobra.Command, args []string) error {
+			if err := o.Complete(); err != nil {
+				return err
+			}
+
+			if err := o.Validate(args); err != nil {
+				return err
+			}
+
+			config, err := o.Config()
+			if err != nil {
+				return err
+			}
+
+			if err := o.RunExampleServer(config, stopCh); err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+
+	flags := cmd.Flags()
+	o.RecommendedOptions.AddFlags(flags)
+
+	return cmd
+}
+
+func main() {
+	stopCh := genericapiserver.SetupSignalHandler()
+	options, err := NewExampleServerOptions(os.Stdout, os.Stderr)
+	if err != nil {
+		panic(err)
+	}
+
+	cmd := NewCommandStartExampleAPIServer(options, stopCh)
+
+	code := cli.Run(cmd)
+	os.Exit(code)
+}

pkg/cmd/grafana-example-apiserver/server.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net"
+
+	exampleAPI "github.com/grafana/grafana/pkg/registry/apis/example"
+	grafanaAPIServer "github.com/grafana/grafana/pkg/services/grafana-apiserver"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/options"
+	netutils "k8s.io/utils/net"
+)
+
+const defaultEtcdPathPrefix = "/registry/example.grafana.app"
+
+var (
+	Scheme = runtime.NewScheme()
+	Codecs = serializer.NewCodecFactory(Scheme)
+
+	unversionedVersion = schema.GroupVersion{Group: "", Version: "v1"}
+	unversionedTypes   = []runtime.Object{
+		&metav1.Status{},
+		&metav1.WatchEvent{},
+		&metav1.APIVersions{},
+		&metav1.APIGroupList{},
+		&metav1.APIGroup{},
+		&metav1.APIResourceList{},
+	}
+)
+
+func init() {
+	// we need to add the options to empty v1
+	metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Group: "", Version: "v1"})
+	Scheme.AddUnversionedTypes(unversionedVersion, unversionedTypes...)
+}
+
+// ExampleServerOptions contains the state for the apiserver
+type ExampleServerOptions struct {
+	RecommendedOptions *options.RecommendedOptions
+	Builders           []grafanaAPIServer.APIGroupBuilder
+	AlternateDNS       []string
+
+	StdOut io.Writer
+	StdErr io.Writer
+}
+
+func NewExampleServerOptions(out, errOut io.Writer) (*ExampleServerOptions, error) {
+	builder := &exampleAPI.TestingAPIBuilder{}
+
+	// Install schema
+	if err := builder.InstallSchema(Scheme); err != nil {
+		return nil, err
+	}
+
+	return &ExampleServerOptions{
+		Builders: []grafanaAPIServer.APIGroupBuilder{builder},
+		RecommendedOptions: options.NewRecommendedOptions(
+			defaultEtcdPathPrefix,
+			Codecs.LegacyCodec(builder.GetGroupVersion()),
+		),
+		StdOut: out,
+		StdErr: errOut,
+	}, nil
+}
+
+func (o ExampleServerOptions) Config() (*genericapiserver.RecommendedConfig, error) {
+	if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", o.AlternateDNS, []net.IP{netutils.ParseIPSloppy("127.0.0.1")}); err != nil {
+		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
+	}
+
+	o.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true
+	o.RecommendedOptions.Authorization.RemoteKubeConfigFileOptional = true
+
+	o.RecommendedOptions.Admission = nil
+	o.RecommendedOptions.CoreAPI = nil
+
+	serverConfig := genericapiserver.NewRecommendedConfig(Codecs)
+
+	if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
+		return nil, err
+	}
+
+	return serverConfig, nil
+}
+
+// Validate validates ExampleServerOptions
+func (o ExampleServerOptions) Validate(args []string) error {
+	errors := []error{}
+	errors = append(errors, o.RecommendedOptions.Validate()...)
+	return utilerrors.NewAggregate(errors)
+}
+
+// Complete fills in fields required to have valid data
+func (o ExampleServerOptions) Complete() error {
+	return nil
+}
+
+func (o ExampleServerOptions) RunExampleServer(config *genericapiserver.RecommendedConfig, stopCh <-chan struct{}) error {
+	delegationTarget := genericapiserver.NewEmptyDelegate()
+	completedConfig := config.Complete()
+	server, err := completedConfig.New("example-apiserver", delegationTarget)
+	if err != nil {
+		return err
+	}
+
+	// Install the API Group+version
+	for _, b := range o.Builders {
+		g, err := b.GetAPIGroupInfo(Scheme, Codecs, completedConfig.RESTOptionsGetter)
+		if err != nil {
+			return err
+		}
+		if g == nil || len(g.PrioritizedVersions) < 1 {
+			continue
+		}
+		err = server.InstallAPIGroup(g)
+		if err != nil {
+			return err
+		}
+	}
+
+	return server.PrepareRun().Run(stopCh)
+}

pkg/registry/apis/example/register.go

@@ -192,6 +192,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 	scheme.AddKnownTypes(SchemeGroupVersion,
 		&example.RuntimeInfo{},
 	)
+
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
-	return nil
+	return scheme.SetVersionPriority(SchemeGroupVersion)
 }