K8s: Remove duplicate listener in production (#76583)

go.mod

@@ -275,11 +275,11 @@ require (
 )
 
 require (
-	github.com/grafana/grafana-apiserver v0.0.0-20230713001719-88a9ed41992d // @grafana/grafana-app-platform-squad
 	go.opentelemetry.io/otel v1.19.0 // @grafana/backend-platform
 	k8s.io/apimachinery v0.27.1 // @grafana/grafana-app-platform-squad
 	k8s.io/apiserver v0.27.1 // @grafana/grafana-app-platform-squad
 	k8s.io/client-go v0.27.1 // @grafana/grafana-app-platform-squad
+	k8s.io/component-base v0.27.1 // @grafana/grafana-app-platform-squad
 	k8s.io/klog/v2 v2.90.1 // @grafana/grafana-app-platform-squad
 	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // @grafana/grafana-app-platform-squad
 )
@@ -411,7 +411,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	k8s.io/api v0.27.1 // indirect
-	k8s.io/component-base v0.27.1 // indirect
 	k8s.io/kms v0.27.1 // indirect
 	lukechampine.com/uint128 v1.2.0 // indirect
 	modernc.org/cc/v3 v3.40.0 // indirect

go.sum

@@ -1812,8 +1812,6 @@ github.com/grafana/dskit v0.0.0-20230706162620-5081d8ed53e6 h1:/19OPOCKP95g9hKLn
 github.com/grafana/dskit v0.0.0-20230706162620-5081d8ed53e6/go.mod h1:M03k2fzuQ2n9TVE1xfVKTESibxsXdw0wYfWT3+9Owp4=
 github.com/grafana/gofpdf v0.0.0-20231002120153-857cc45be447 h1:jxJJ5z0GxqhWFbQUsys3BHG8jnmniJ2Q74tXAG1NaDo=
 github.com/grafana/gofpdf v0.0.0-20231002120153-857cc45be447/go.mod h1:IxsY6mns6Q5sAnWcrptrgUrSglTZJXH/kXr9nbpb/9I=
-github.com/grafana/grafana-apiserver v0.0.0-20230713001719-88a9ed41992d h1:fjc6vlNnKCDewr/5GiObiiGLeqr6YnXUbZ44YZtlu5Q=
-github.com/grafana/grafana-apiserver v0.0.0-20230713001719-88a9ed41992d/go.mod h1:2g9qGdCeU6x/69QAs82WM52bwBUT5/CBaBD0I94+txU=
 github.com/grafana/grafana-aws-sdk v0.19.1 h1:5GBiOv2AgdyjwlgAX+dtgPtXU4FgMTD9rfQUPQseEpQ=
 github.com/grafana/grafana-aws-sdk v0.19.1/go.mod h1:ntq2NDH12Y2Fkbc6fozpF8kYsJM9k6KNr+Xfo5w3/iM=
 github.com/grafana/grafana-azure-sdk-go v1.9.0 h1:4JRwlqgUtPRAQSoiV4DFZDQ3lbNsauHqj9kC6SMR9Ak=

pkg/services/grafana-apiserver/README.md

@@ -23,7 +23,7 @@ Start `etcd`:
 make devenv sources=etcd
 ```
 
-Enable dual write to `etcd`:
+Add etcd server to `custom.ini`:
 
 ```ini
 [grafana-apiserver]
@@ -32,10 +32,9 @@ etcd_servers = 127.0.0.1:2379
 
 ### `kubectl` access
 
-From the root of the repository:
-
+From the root of the Grafanaa repository, run the following:
 ```bash
-export KUBECONFIG=$PWD/data/k8s/grafana.kubeconfig
+export KUBECONFIG=$PWD/data/grafana-apiserver/grafana.kubeconfig
 kubectl api-resources
 ```
 

pkg/services/grafana-apiserver/config.go

@@ -2,7 +2,9 @@ package grafanaapiserver
 
 import (
 	"fmt"
+	"net"
 	"path/filepath"
+	"strconv"
 
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/setting"
@@ -12,8 +14,11 @@ type config struct {
 	enabled bool
 	devMode bool
 
-	host        string
-	appURL      string
+	ip     net.IP
+	port   int
+	host   string
+	apiURL string
+
 	etcdServers []string
 	dataPath    string
 
@@ -22,17 +27,31 @@ type config struct {
 
 func newConfig(cfg *setting.Cfg) *config {
 	defaultLogLevel := 0
+	ip := net.ParseIP(cfg.HTTPAddr)
+	apiURL := cfg.AppURL
+	port, err := strconv.Atoi(cfg.HTTPPort)
+	if err != nil {
+		port = 3000
+	}
+
 	if cfg.Env == setting.Dev {
 		defaultLogLevel = 10
+		port = 6443
+		ip = net.ParseIP("127.0.0.1")
+		apiURL = fmt.Sprintf("https://%s:%d", ip, port)
 	}
 
+	host := fmt.Sprintf("%s:%d", ip, port)
+
 	return &config{
 		enabled:     cfg.IsFeatureToggleEnabled(featuremgmt.FlagGrafanaAPIServer),
 		devMode:     cfg.Env == setting.Dev,
 		dataPath:    filepath.Join(cfg.DataPath, "grafana-apiserver"),
-		host:        fmt.Sprintf("%s:%s", cfg.HTTPAddr, cfg.HTTPPort),
+		ip:          ip,
+		port:        port,
+		host:        host,
 		logLevel:    cfg.SectionWithEnvOverrides("grafana-apiserver").Key("log_level").MustInt(defaultLogLevel),
 		etcdServers: cfg.SectionWithEnvOverrides("grafana-apiserver").Key("etcd_servers").Strings(","),
-		appURL:      cfg.AppURL,
+		apiURL:      apiURL,
 	}
 }

pkg/services/grafana-apiserver/config_test.go

@@ -1,6 +1,7 @@
 package grafanaapiserver
 
 import (
+	"net"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -10,9 +11,9 @@ import (
 
 func TestNewConfig(t *testing.T) {
 	cfg := setting.NewCfg()
-	cfg.Env = setting.Dev
+	cfg.Env = setting.Prod
 	cfg.DataPath = "/tmp/grafana"
-	cfg.HTTPAddr = "test"
+	cfg.HTTPAddr = "10.0.0.1"
 	cfg.HTTPPort = "4000"
 	cfg.IsFeatureToggleEnabled = func(_ string) bool { return true }
 	cfg.AppURL = "http://test:4000"
@@ -25,10 +26,12 @@ func TestNewConfig(t *testing.T) {
 
 	expected := &config{
 		enabled:     true,
-		devMode:     true,
+		devMode:     false,
 		etcdServers: []string{"http://localhost:2379"},
-		appURL:      "http://test:4000",
-		host:        "test:4000",
+		apiURL:      "http://test:4000",
+		ip:          net.ParseIP("10.0.0.1"),
+		port:        4000,
+		host:        "10.0.0.1:4000",
 		dataPath:    "/tmp/grafana/grafana-apiserver",
 		logLevel:    5,
 	}

pkg/services/grafana-apiserver/log.go

@@ -1,7 +1,8 @@
 package grafanaapiserver
 
 import (
-	"cuelang.org/go/pkg/strings"
+	"strings"
+
 	"github.com/go-logr/logr"
 
 	"github.com/grafana/grafana/pkg/infra/log"

pkg/services/grafana-apiserver/service.go

@@ -2,23 +2,18 @@ package grafanaapiserver
 
 import (
 	"context"
-	"crypto/x509"
 	"fmt"
-	"net"
 	"net/http"
 	"path"
 	"strconv"
 
 	"github.com/go-logr/logr"
 	"github.com/grafana/dskit/services"
-	"github.com/grafana/grafana-apiserver/pkg/certgenerator"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
-	"k8s.io/apiserver/pkg/authentication/authenticator"
-	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
 	"k8s.io/apiserver/pkg/endpoints/responsewriter"
@@ -29,6 +24,7 @@ import (
 	clientrest "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/component-base/logs"
 	"k8s.io/klog/v2"
 
 	"github.com/grafana/grafana/pkg/api/routing"
@@ -41,18 +37,12 @@ import (
 	"github.com/grafana/grafana/pkg/web"
 )
 
-const (
-	DefaultAPIServerHost = "https://" + certgenerator.DefaultAPIServerIp + ":6443"
-)
-
 var (
 	_ Service                    = (*service)(nil)
 	_ RestConfigProvider         = (*service)(nil)
 	_ registry.BackgroundService = (*service)(nil)
 	_ registry.CanBeDisabled     = (*service)(nil)
-)
 
-var (
 	Scheme = runtime.NewScheme()
 	Codecs = serializer.NewCodecFactory(Scheme)
 
@@ -169,9 +159,13 @@ func (s *service) RegisterAPI(builder APIGroupBuilder) {
 func (s *service) start(ctx context.Context) error {
 	logger := logr.New(newLogAdapter(s.config.logLevel))
 	klog.SetLoggerWithOptions(logger, klog.ContextualLogger(true))
+	if _, err := logs.GlogSetter(strconv.Itoa(s.config.logLevel)); err != nil {
+		logger.Error(err, "failed to set log level")
+	}
 
 	o := options.NewRecommendedOptions("", unstructured.UnstructuredJSONScheme)
-	o.SecureServing.BindPort = 6443
+	o.SecureServing.BindAddress = s.config.ip
+	o.SecureServing.BindPort = s.config.port
 	o.Authentication.RemoteKubeConfigFileOptional = true
 	o.Authorization.RemoteKubeConfigFileOptional = true
 	o.Etcd.StorageConfig.Transport.ServerList = s.config.etcdServers
@@ -182,55 +176,52 @@ func (s *service) start(ctx context.Context) error {
 		o.Etcd = nil
 	}
 
-	// Get the util to get the paths to pre-generated certs
-	certUtil := certgenerator.CertUtil{
-		K8sDataPath: s.config.dataPath,
-	}
-
-	if err := certUtil.InitializeCACertPKI(); err != nil {
-		return err
-	}
-
-	if err := certUtil.EnsureApiServerPKI(certgenerator.DefaultAPIServerIp); err != nil {
-		return err
-	}
-
-	o.SecureServing.BindAddress = net.ParseIP(certgenerator.DefaultAPIServerIp)
-	o.SecureServing.ServerCert.CertKey = options.CertKey{
-		CertFile: certUtil.APIServerCertFile(),
-		KeyFile:  certUtil.APIServerKeyFile(),
-	}
-
 	if err := o.Validate(); len(err) > 0 {
 		return err[0]
 	}
 
 	serverConfig := genericapiserver.NewRecommendedConfig(Codecs)
-	err := o.ApplyTo(serverConfig)
-	if err != nil {
-		return err
+	serverConfig.ExternalAddress = s.config.host
+
+	if s.config.devMode {
+		// SecureServingOptions is used when the apiserver needs it's own listener.
+		// this is not needed in production, but it's useful for development kubectl access.
+		if err := o.SecureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil {
+			return err
+		}
+		// AuthenticationOptions is needed to authenticate requests from kubectl in dev mode.
+		if err := o.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, serverConfig.OpenAPIConfig); err != nil {
+			return err
+		}
 	}
 
-	rootCert, err := certUtil.GetK8sCACert()
-	if err != nil {
-		return err
+	// override ExternalAddress and LoopbackClientConfig in prod mode.
+	// in dev mode we want to use the loopback client config
+	// and address provided by SecureServingOptions.
+	if !s.config.devMode {
+		serverConfig.ExternalAddress = s.config.host
+		serverConfig.LoopbackClientConfig = &clientrest.Config{
+			Host: s.config.apiURL,
+			TLSClientConfig: clientrest.TLSClientConfig{
+				Insecure: true,
+			},
+		}
 	}
 
-	authenticator, err := newAuthenticator(rootCert)
-	if err != nil {
-		return err
+	if o.Etcd != nil {
+		if err := o.Etcd.ApplyTo(&serverConfig.Config); err != nil {
+			return err
+		}
 	}
 
 	serverConfig.Authorization.Authorizer = s.authorizer
-	serverConfig.Authentication.Authenticator = authenticator
 
 	// Get the list of groups the server will support
 	builders := s.builders
 
 	// Install schemas
 	for _, b := range builders {
-		err = b.InstallSchema(Scheme) // previously was in init
-		if err != nil {
+		if err := b.InstallSchema(Scheme); err != nil {
 			return err
 		}
 	}
@@ -282,12 +273,13 @@ func (s *service) start(ctx context.Context) error {
 	}
 
 	s.restConfig = server.LoopbackClientConfig
-	err = s.writeKubeConfiguration(s.restConfig)
-	if err != nil {
-		return err
-	}
 
-	prepared := server.PrepareRun()
+	// only write kubeconfig in dev mode
+	if s.config.devMode {
+		if err := s.ensureKubeConfig(); err != nil {
+			return err
+		}
+	}
 
 	// TODO: this is a hack. see note in ProvideService
 	s.handler = func(c *contextmodel.ReqContext) {
@@ -308,17 +300,28 @@ func (s *service) start(ctx context.Context) error {
 		req.Header.Set("X-Remote-Group", "grafana")
 
 		resp := responsewriter.WrapForHTTP1Or2(c.Resp)
-		prepared.GenericAPIServer.Handler.ServeHTTP(resp, req)
+		server.Handler.ServeHTTP(resp, req)
 	}
 
+	// skip starting the server in prod mode
+	if !s.config.devMode {
+		return nil
+	}
+
+	prepared := server.PrepareRun()
 	go func() {
 		s.stoppedCh <- prepared.Run(s.stopCh)
 	}()
-
 	return nil
 }
 
 func (s *service) running(ctx context.Context) error {
+	// skip waiting for the server in prod mode
+	if !s.config.devMode {
+		<-ctx.Done()
+		return nil
+	}
+
 	select {
 	case err := <-s.stoppedCh:
 		if err != nil {
@@ -330,10 +333,10 @@ func (s *service) running(ctx context.Context) error {
 	return nil
 }
 
-func (s *service) writeKubeConfiguration(restConfig *clientrest.Config) error {
+func (s *service) ensureKubeConfig() error {
 	clusters := make(map[string]*clientcmdapi.Cluster)
 	clusters["default-cluster"] = &clientcmdapi.Cluster{
-		Server:                restConfig.Host,
+		Server:                s.restConfig.Host,
 		InsecureSkipTLSVerify: true,
 	}
 
@@ -346,7 +349,7 @@ func (s *service) writeKubeConfiguration(restConfig *clientrest.Config) error {
 
 	authinfos := make(map[string]*clientcmdapi.AuthInfo)
 	authinfos["default"] = &clientcmdapi.AuthInfo{
-		Token: restConfig.BearerToken,
+		Token: s.restConfig.BearerToken,
 	}
 
 	clientConfig := clientcmdapi.Config{
@@ -357,24 +360,6 @@ func (s *service) writeKubeConfiguration(restConfig *clientrest.Config) error {
 		CurrentContext: "default-context",
 		AuthInfos:      authinfos,
 	}
+
 	return clientcmd.WriteToFile(clientConfig, path.Join(s.config.dataPath, "grafana.kubeconfig"))
 }
-
-func newAuthenticator(cert *x509.Certificate) (authenticator.Request, error) {
-	reqHeaderOptions := options.RequestHeaderAuthenticationOptions{
-		UsernameHeaders:     []string{"X-Remote-User"},
-		GroupHeaders:        []string{"X-Remote-Group"},
-		ExtraHeaderPrefixes: []string{"X-Remote-Extra-"},
-	}
-
-	requestHeaderAuthenticator, err := headerrequest.New(
-		reqHeaderOptions.UsernameHeaders,
-		reqHeaderOptions.GroupHeaders,
-		reqHeaderOptions.ExtraHeaderPrefixes,
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	return requestHeaderAuthenticator, nil
-}