Chore: Add grafana-apiserver (#70721)

* add grafana-apiserver
* remove watchset & move provisioning and http server to background
services
* remove scheme
* otel fixes (#70874)
* remove module ProvideRegistry test
* use certgenerator from apiserver package
* Control collector/pdata from going to v1.0.0-rc8 (as Tempo 1.5.1 would have it)

pkg/services/grafana-apiserver/log.go

@@ -0,0 +1,50 @@
+package grafanaapiserver
+
+import (
+	"cuelang.org/go/pkg/strings"
+	"github.com/go-logr/logr"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+)
+
+var _ logr.LogSink = (*logAdapter)(nil)
+
+type logAdapter struct {
+	log log.Logger
+}
+
+func newLogAdapter() *logAdapter {
+	return &logAdapter{log: log.New("k8s.apiserver")}
+}
+
+func (l *logAdapter) WithName(name string) logr.LogSink {
+	l.log = l.log.New("name", name)
+	return l
+}
+
+func (l *logAdapter) WithValues(keysAndValues ...interface{}) logr.LogSink {
+	l.log = l.log.New(keysAndValues...)
+	return l
+}
+
+func (l *logAdapter) Init(_ logr.RuntimeInfo) {
+	// TODO: shrug emoji
+}
+
+func (l *logAdapter) Enabled(level int) bool {
+	return level <= 5
+}
+
+func (l *logAdapter) Info(level int, msg string, keysAndValues ...interface{}) {
+	msg = strings.TrimSpace(msg)
+	if level < 1 {
+		l.log.Info(msg, keysAndValues...)
+		return
+	}
+	l.log.Debug(msg, keysAndValues...)
+}
+
+func (l *logAdapter) Error(err error, msg string, keysAndValues ...interface{}) {
+	msg = strings.TrimSpace(msg)
+	l.log.Error(msg, keysAndValues...)
+}

pkg/services/grafana-apiserver/service.go

@@ -0,0 +1,248 @@
+package grafanaapiserver
+
+import (
+	"context"
+	"crypto/x509"
+	"net"
+	"os"
+	"path"
+	"strconv"
+
+	"cuelang.org/go/pkg/strings"
+	"github.com/go-logr/logr"
+	"github.com/grafana/dskit/services"
+	grafanaapiserveroptions "github.com/grafana/grafana-apiserver/pkg/cmd/server/options"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/endpoints/responsewriter"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/options"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/klog/v2"
+
+	"github.com/grafana/grafana-apiserver/pkg/certgenerator"
+	"github.com/grafana/grafana/pkg/api/routing"
+	"github.com/grafana/grafana/pkg/infra/appcontext"
+	"github.com/grafana/grafana/pkg/middleware"
+	"github.com/grafana/grafana/pkg/modules"
+	contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/web"
+)
+
+const (
+	DefaultAPIServerHost = "https://" + certgenerator.DefaultAPIServerIp + ":6443"
+)
+
+var (
+	_ Service            = (*service)(nil)
+	_ RestConfigProvider = (*service)(nil)
+)
+
+type Service interface {
+	services.NamedService
+}
+
+type RestConfigProvider interface {
+	GetRestConfig() *rest.Config
+}
+
+type service struct {
+	*services.BasicService
+
+	restConfig *rest.Config
+	rr         routing.RouteRegister
+
+	handler   web.Handler
+	dataPath  string
+	stopCh    chan struct{}
+	stoppedCh chan error
+}
+
+func ProvideService(cfg *setting.Cfg, rr routing.RouteRegister) (*service, error) {
+	s := &service{
+		rr:       rr,
+		dataPath: path.Join(cfg.DataPath, "k8s"),
+		stopCh:   make(chan struct{}),
+	}
+
+	s.BasicService = services.NewBasicService(s.start, s.running, nil).WithName(modules.GrafanaAPIServer)
+
+	s.rr.Group("/k8s", func(k8sRoute routing.RouteRegister) {
+		handler := func(c *contextmodel.ReqContext) {
+			if s.handler == nil {
+				c.Resp.WriteHeader(404)
+				_, _ = c.Resp.Write([]byte("Not found"))
+				return
+			}
+
+			if handle, ok := s.handler.(func(c *contextmodel.ReqContext)); ok {
+				handle(c)
+				return
+			}
+		}
+		k8sRoute.Any("/", middleware.ReqSignedIn, handler)
+		k8sRoute.Any("/*", middleware.ReqSignedIn, handler)
+	})
+
+	return s, nil
+}
+
+func (s *service) GetRestConfig() *rest.Config {
+	return s.restConfig
+}
+
+func (s *service) start(ctx context.Context) error {
+	logger := logr.New(newLogAdapter())
+	logger.V(9)
+	klog.SetLoggerWithOptions(logger, klog.ContextualLogger(true))
+
+	o := grafanaapiserveroptions.NewGrafanaAPIServerOptions(os.Stdout, os.Stderr)
+	o.RecommendedOptions.SecureServing.BindPort = 6443
+	o.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true
+	o.RecommendedOptions.Authorization.RemoteKubeConfigFileOptional = true
+	o.RecommendedOptions.Authorization.AlwaysAllowPaths = []string{"*"}
+	o.RecommendedOptions.Authorization.AlwaysAllowGroups = []string{user.SystemPrivilegedGroup, "grafana"}
+	o.RecommendedOptions.Etcd = nil
+	// TODO: setting CoreAPI to nil currently segfaults in grafana-apiserver
+	o.RecommendedOptions.CoreAPI = nil
+
+	// Get the util to get the paths to pre-generated certs
+	certUtil := certgenerator.CertUtil{
+		K8sDataPath: s.dataPath,
+	}
+
+	o.RecommendedOptions.SecureServing.BindAddress = net.ParseIP(certgenerator.DefaultAPIServerIp)
+	o.RecommendedOptions.SecureServing.ServerCert.CertKey = options.CertKey{
+		CertFile: certUtil.APIServerCertFile(),
+		KeyFile:  certUtil.APIServerKeyFile(),
+	}
+
+	if err := o.Complete(); err != nil {
+		return err
+	}
+
+	if err := o.Validate(); err != nil {
+		return err
+	}
+
+	serverConfig, err := o.Config()
+	if err != nil {
+		return err
+	}
+
+	rootCert, err := certUtil.GetK8sCACert()
+	if err != nil {
+		return err
+	}
+
+	authenticator, err := newAuthenticator(rootCert)
+	if err != nil {
+		return err
+	}
+
+	serverConfig.GenericConfig.Authentication.Authenticator = authenticator
+
+	server, err := serverConfig.Complete().New(genericapiserver.NewEmptyDelegate())
+	if err != nil {
+		return err
+	}
+
+	s.restConfig = server.GenericAPIServer.LoopbackClientConfig
+	err = s.writeKubeConfiguration(s.restConfig)
+	if err != nil {
+		return err
+	}
+
+	prepared := server.GenericAPIServer.PrepareRun()
+
+	s.handler = func(c *contextmodel.ReqContext) {
+		req := c.Req
+		req.URL.Path = strings.TrimPrefix(req.URL.Path, "/k8s")
+		if req.URL.Path == "" {
+			req.URL.Path = "/"
+		}
+		ctx := req.Context()
+		signedInUser := appcontext.MustUser(ctx)
+
+		req.Header.Set("X-Remote-User", strconv.FormatInt(signedInUser.UserID, 10))
+		req.Header.Set("X-Remote-Group", "grafana")
+		req.Header.Set("X-Remote-Extra-token-name", signedInUser.Name)
+		req.Header.Set("X-Remote-Extra-org-role", string(signedInUser.OrgRole))
+		req.Header.Set("X-Remote-Extra-org-id", strconv.FormatInt(signedInUser.OrgID, 10))
+		req.Header.Set("X-Remote-Extra-user-id", strconv.FormatInt(signedInUser.UserID, 10))
+
+		resp := responsewriter.WrapForHTTP1Or2(c.Resp)
+		prepared.GenericAPIServer.Handler.ServeHTTP(resp, req)
+	}
+
+	go func() {
+		s.stoppedCh <- prepared.Run(s.stopCh)
+	}()
+
+	return nil
+}
+
+func (s *service) running(ctx context.Context) error {
+	select {
+	case err := <-s.stoppedCh:
+		if err != nil {
+			return err
+		}
+	case <-ctx.Done():
+		close(s.stopCh)
+	}
+	return nil
+}
+
+func (s *service) writeKubeConfiguration(restConfig *rest.Config) error {
+	clusters := make(map[string]*clientcmdapi.Cluster)
+	clusters["default-cluster"] = &clientcmdapi.Cluster{
+		Server:                restConfig.Host,
+		InsecureSkipTLSVerify: true,
+	}
+
+	contexts := make(map[string]*clientcmdapi.Context)
+	contexts["default-context"] = &clientcmdapi.Context{
+		Cluster:   "default-cluster",
+		Namespace: "default",
+		AuthInfo:  "default",
+	}
+
+	authinfos := make(map[string]*clientcmdapi.AuthInfo)
+	authinfos["default"] = &clientcmdapi.AuthInfo{
+		Token: restConfig.BearerToken,
+	}
+
+	clientConfig := clientcmdapi.Config{
+		Kind:           "Config",
+		APIVersion:     "v1",
+		Clusters:       clusters,
+		Contexts:       contexts,
+		CurrentContext: "default-context",
+		AuthInfos:      authinfos,
+	}
+	return clientcmd.WriteToFile(clientConfig, path.Join(s.dataPath, "grafana.kubeconfig"))
+}
+
+func newAuthenticator(cert *x509.Certificate) (authenticator.Request, error) {
+	reqHeaderOptions := options.RequestHeaderAuthenticationOptions{
+		UsernameHeaders:     []string{"X-Remote-User"},
+		GroupHeaders:        []string{"X-Remote-Group"},
+		ExtraHeaderPrefixes: []string{"X-Remote-Extra-"},
+	}
+
+	requestHeaderAuthenticator, err := headerrequest.New(
+		reqHeaderOptions.UsernameHeaders,
+		reqHeaderOptions.GroupHeaders,
+		reqHeaderOptions.ExtraHeaderPrefixes,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return requestHeaderAuthenticator, nil
+}

pkg/services/grafana-apiserver/wire.go

@@ -0,0 +1,11 @@
+package grafanaapiserver
+
+import (
+	"github.com/google/wire"
+)
+
+var WireSet = wire.NewSet(
+	ProvideService,
+	wire.Bind(new(Service), new(*service)),
+	wire.Bind(new(RestConfigProvider), new(*service)),
+)