Ensure that datasource apiservers receive and forwards headers (#92304)

* Ensure that datasource apiservers receive and forwards headers for datasources:
- adds log line for prometheus to see when from alert header is received
- add logging to the datasource apiserver
- Updates the Connect func in sub query to forward expected headers to datasources and log unexpected ones.

pkg/promlib/querydata/request.go

@@ -86,6 +86,8 @@ func New(
 
 func (s *QueryData) Execute(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
 	fromAlert := req.Headers["FromAlert"] == "true"
+	logger := s.log.FromContext(ctx)
+	logger.Debug("Begin query execution", "fromAlert", fromAlert)
 	result := backend.QueryDataResponse{
 		Responses: backend.Responses{},
 	}
@@ -104,7 +106,6 @@ func (s *QueryData) Execute(ctx context.Context, req *backend.QueryDataRequest)
 
 		concurrentQueryCount, err := req.PluginContext.GrafanaConfig.ConcurrentQueryCount()
 		if err != nil {
-			logger := s.log.FromContext(ctx)
 			logger.Debug(fmt.Sprintf("Concurrent Query Count read/parse error: %v", err), "prometheusRunQueriesInParallel")
 			concurrentQueryCount = 10
 		}

pkg/registry/apis/datasource/register.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/prometheus/client_golang/prometheus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -45,6 +46,7 @@ type DataSourceAPIBuilder struct {
 	contextProvider PluginContextWrapper
 	accessControl   accesscontrol.AccessControl
 	queryTypes      *query.QueryTypeDefinitionList
+	log             log.Logger
 }
 
 func RegisterAPIService(
@@ -121,6 +123,7 @@ func NewDataSourceAPIBuilder(
 		datasources:            datasources,
 		contextProvider:        contextProvider,
 		accessControl:          accessControl,
+		log:                    log.New("grafana-apiserver.datasource"),
 	}
 	if loadQueryTypes {
 		// In the future, this will somehow come from the plugin

pkg/registry/apis/datasource/sub_query.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1"
@@ -52,7 +53,6 @@ func (r *subQueryREST) Connect(ctx context.Context, name string, opts runtime.Ob
 	if err != nil {
 		return nil, err
 	}
-
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		dqr := data.QueryDataRequest{}
 		err := web.Bind(req, &dqr)
@@ -73,9 +73,30 @@ func (r *subQueryREST) Connect(ctx context.Context, name string, opts runtime.Ob
 
 		ctx = backend.WithGrafanaConfig(ctx, pluginCtx.GrafanaConfig)
 		ctx = contextualMiddlewares(ctx)
+
+		// only forward expected headers, log unexpected ones
+		headers := make(map[string]string)
+		// headers are case insensitive, however some datasources still check for camel casing so we have to send them camel cased
+		expectedHeaders := map[string]string{
+			"fromalert":      "FromAlert",
+			"content-type":   "Content-Type",
+			"content-length": "Content-Length",
+			"user-agent":     "User-Agent",
+			"accept":         "Accept",
+		}
+		for k, v := range req.Header {
+			headerToSend, ok := expectedHeaders[strings.ToLower(k)]
+			if ok {
+				headers[headerToSend] = v[0]
+			} else {
+				r.builder.log.Warn("datasource received an unexpected header, ignoring it", "header", k)
+			}
+		}
+
 		rsp, err := r.builder.client.QueryData(ctx, &backend.QueryDataRequest{
 			Queries:       queries,
 			PluginContext: pluginCtx,
+			Headers:       headers,
 		})
 		if err != nil {
 			responder.Error(err)

pkg/registry/apis/datasource/sub_query_test.go

@@ -0,0 +1,98 @@
+package datasource
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana/pkg/apis/datasource/v0alpha1"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+func TestSubQueryConnect(t *testing.T) {
+	sqr := subQueryREST{
+		builder: &DataSourceAPIBuilder{
+			client: mockClient{
+				lastCalledWithHeaders: &map[string]string{},
+			},
+			datasources:     mockDatasources{},
+			contextProvider: mockContextProvider{},
+			log:             log.NewNopLogger(),
+		},
+	}
+
+	mr := mockResponder{}
+	handler, err := sqr.Connect(context.Background(), "dsname", nil, mr)
+	require.NoError(t, err)
+
+	rr := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "/some-path", nil)
+	req.Header.Set("fromAlert", "true")
+	req.Header.Set("Content-Type", "application/json")
+	handler.ServeHTTP(rr, req)
+
+	// test that headers are forwarded and cased appropriately
+	require.Equal(t, map[string]string{
+		"FromAlert":    "true",
+		"Content-Type": "application/json",
+	}, *sqr.builder.client.(mockClient).lastCalledWithHeaders)
+}
+
+type mockClient struct {
+	lastCalledWithHeaders *map[string]string
+}
+
+func (m mockClient) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
+	*m.lastCalledWithHeaders = req.Headers
+	return nil, fmt.Errorf("mock error")
+}
+
+func (m mockClient) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
+	return nil
+}
+
+func (m mockClient) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
+	return nil, nil
+}
+
+type mockResponder struct {
+}
+
+// Object writes the provided object to the response. Invoking this method multiple times is undefined.
+func (m mockResponder) Object(statusCode int, obj runtime.Object) {
+}
+
+// Error writes the provided error to the response. This method may only be invoked once.
+func (m mockResponder) Error(err error) {
+}
+
+type mockDatasources struct {
+}
+
+// Get gets a specific datasource (that the user in context can see)
+func (m mockDatasources) Get(ctx context.Context, uid string) (*v0alpha1.DataSourceConnection, error) {
+	return nil, nil
+}
+
+// List lists all data sources the user in context can see
+func (m mockDatasources) List(ctx context.Context) (*v0alpha1.DataSourceConnectionList, error) {
+	return nil, nil
+}
+
+// Return settings (decrypted!) for a specific plugin
+// This will require "query" permission for the user in context
+func (m mockDatasources) GetInstanceSettings(ctx context.Context, uid string) (*backend.DataSourceInstanceSettings, error) {
+	return nil, nil
+}
+
+type mockContextProvider struct {
+}
+
+func (m mockContextProvider) PluginContextForDataSource(ctx context.Context, datasourceSettings *backend.DataSourceInstanceSettings) (backend.PluginContext, error) {
+	return backend.PluginContext{}, nil
+}