IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Datasource: Shared HTTP client provider for core backend data sources and any data source using the data source proxy (#33439)

Browse Source 
Uses new httpclient package from grafana-plugin-sdk-go introduced 
via grafana/grafana-plugin-sdk-go#328. 
Replaces the GetHTTPClient, GetTransport, GetTLSConfig methods defined 
on DataSource model.
Longer-term the goal is to migrate core HTTP backend data sources to use the 
SDK contracts and using httpclient.Provider for creating HTTP clients and such.

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
This commit is contained in:
Marcus Efraimsson 2021-05-19 23:53:41 +02:00 committed by GitHub
parent 7a83d1f9ff
commit 348e76fc8e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 1082 additions and 467 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
conf/defaults.ini
View File

@ -158,6 +158,9 @@ expect_continue_timeout_seconds = 1
# The maximum number of idle connections that Grafana will keep alive.
max_idle_connections = 100
# The maximum number of idle connections per host that Grafana will keep alive.
max_idle_connections_per_host = 2
# How many seconds the data proxy keeps an idle connection open before timing out.
idle_conn_timeout_seconds = 90

3
conf/sample.ini
View File

@ -164,6 +164,9 @@
# The maximum number of idle connections that Grafana will keep alive.
;max_idle_connections = 100
# The maximum number of idle connections per host that Grafana will keep alive.
;max_idle_connections_per_host = 2
# How many seconds the data proxy keeps an idle connection open before timing out.
;idle_conn_timeout_seconds = 90

2
devenv/docker/blocks/slow_proxy/.env Normal file
View File

@ -0,0 +1,2 @@
ORIGIN_SERVER=http://localhost:9090/
SLEEP_DURATION=60s

3
devenv/docker/blocks/slow_proxy/docker-compose.yaml
View File

@ -4,4 +4,5 @@
ports:
- "3011:3011"
environment:
ORIGIN_SERVER: "http://localhost:9090/"
ORIGIN_SERVER: ${ORIGIN_SERVER}
SLEEP_DURATION: ${SLEEP_DURATION}

13
devenv/docker/blocks/slow_proxy/main.go
View File

@ -1,7 +1,6 @@
package main
import (
"fmt"
"log"
"net/http"
"net/http/httputil"
@ -16,13 +15,21 @@ func main() {
origin = "http://localhost:9090/"
}
sleep := time.Minute
sleepDurationStr := os.Getenv("SLEEP_DURATION")
if sleepDurationStr == "" {
sleepDurationStr = "60s"
}
sleep, err := time.ParseDuration(sleepDurationStr)
if err != nil {
log.Fatalf("failed to parse SLEEP_DURATION: %v", err)
}
originURL, _ := url.Parse(origin)
proxy := httputil.NewSingleHostReverseProxy(originURL)
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
fmt.Printf("sleeping for %s then proxying request: %s", sleep.String(), r.RequestURI)
log.Printf("sleeping for %s then proxying request: url '%s', headers: '%v'", sleep.String(), r.RequestURI, r.Header)
<-time.After(sleep)
proxy.ServeHTTP(w, r)
})

4
docs/sources/administration/configuration.md
View File

@ -418,6 +418,10 @@ The length of time that Grafana will wait for a datasource’s first response he
The maximum number of idle connections that Grafana will maintain. Default is `100`. For more details check the [Transport.MaxIdleConns](https://golang.org/pkg/net/http/#Transport.MaxIdleConns) documentation.
### max_idle_connections_per_host
The maximum number of idle connections per host that Grafana will maintain. Default is `2`. For more details check the [Transport.MaxIdleConnsPerHost](https://golang.org/pkg/net/http/#Transport.MaxIdleConnsPerHost) documentation.
### idle_conn_timeout_seconds
The length of time that Grafana maintains idle connections before closing them. Default is `90` seconds. For more details check the [Transport.IdleConnTimeout](https://golang.org/pkg/net/http/#Transport.IdleConnTimeout) documentation.

33
pkg/api/pluginproxy/ds_proxy.go
View File

@ -14,6 +14,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/api/datasource"
"github.com/grafana/grafana/pkg/infra/httpclient"
glog "github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
@ -30,13 +31,14 @@ var (
)
type DataSourceProxy struct {
ds *models.DataSource
ctx *models.ReqContext
targetUrl *url.URL
proxyPath string
route *plugins.AppPluginRoute
plugin *plugins.DataSourcePlugin
cfg *setting.Cfg
ds *models.DataSource
ctx *models.ReqContext
targetUrl *url.URL
proxyPath string
route *plugins.AppPluginRoute
plugin *plugins.DataSourcePlugin
cfg *setting.Cfg
clientProvider httpclient.Provider
}
type handleResponseTransport struct {
@ -69,19 +71,20 @@ func (lw *logWrapper) Write(p []byte) (n int, err error) {
// NewDataSourceProxy creates a new Datasource proxy
func NewDataSourceProxy(ds *models.DataSource, plugin *plugins.DataSourcePlugin, ctx *models.ReqContext,
proxyPath string, cfg *setting.Cfg) (*DataSourceProxy, error) {
proxyPath string, cfg *setting.Cfg, clientProvider httpclient.Provider) (*DataSourceProxy, error) {
targetURL, err := datasource.ValidateURL(ds.Type, ds.Url)
if err != nil {
return nil, err
}
return &DataSourceProxy{
ds: ds,
plugin: plugin,
ctx: ctx,
proxyPath: proxyPath,
targetUrl: targetURL,
cfg: cfg,
ds: ds,
plugin: plugin,
ctx: ctx,
proxyPath: proxyPath,
targetUrl: targetURL,
cfg: cfg,
clientProvider: clientProvider,
}, nil
}
@ -101,7 +104,7 @@ func (proxy *DataSourceProxy) HandleRequest() {
proxyErrorLogger := logger.New("userId", proxy.ctx.UserId, "orgId", proxy.ctx.OrgId, "uname", proxy.ctx.Login,
"path", proxy.ctx.Req.URL.Path, "remote_addr", proxy.ctx.RemoteAddr(), "referer", proxy.ctx.Req.Referer())
transport, err := proxy.ds.GetHttpTransport()
transport, err := proxy.ds.GetHTTPTransport(proxy.clientProvider)
if err != nil {
proxy.ctx.JsonApiErr(400, "Unable to load TLS certificate", err)
return

80
pkg/api/pluginproxy/ds_proxy_test.go
View File

@ -12,23 +12,24 @@ import (
"time"
"github.com/grafana/grafana/pkg/api/datasource"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/login/social"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/oauth2"
macaron "gopkg.in/macaron.v1"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/login/social"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/oauth2"
macaron "gopkg.in/macaron.v1"
)
func TestDataSourceProxy_routeRule(t *testing.T) {
httpClientProvider := httpclient.NewProvider()
t.Run("Plugin with routes", func(t *testing.T) {
plugin := &plugins.DataSourcePlugin{
Routes: []*plugins.AppPluginRoute{
@ -113,7 +114,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("When matching route path", func(t *testing.T) {
ctx, req := setUp()
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method", cfg)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method", cfg, httpClientProvider)
require.NoError(t, err)
proxy.route = plugin.Routes[0]
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds, cfg)
@ -124,7 +125,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("When matching route path and has dynamic url", func(t *testing.T) {
ctx, req := setUp()
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/common/some/method", cfg)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/common/some/method", cfg, httpClientProvider)
require.NoError(t, err)
proxy.route = plugin.Routes[3]
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds, cfg)
@ -135,7 +136,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("When matching route path with no url", func(t *testing.T) {
ctx, req := setUp()
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", cfg)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", cfg, httpClientProvider)
require.NoError(t, err)
proxy.route = plugin.Routes[4]
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds, cfg)
@ -145,7 +146,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("When matching route path and has dynamic body", func(t *testing.T) {
ctx, req := setUp()
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/body", cfg)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/body", cfg, httpClientProvider)
require.NoError(t, err)
proxy.route = plugin.Routes[5]
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds, cfg)
@ -158,7 +159,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("Validating request", func(t *testing.T) {
t.Run("plugin route with valid role", func(t *testing.T) {
ctx, _ := setUp()
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method", cfg)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method", cfg, httpClientProvider)
require.NoError(t, err)
err = proxy.validateRequest()
require.NoError(t, err)
@ -166,7 +167,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("plugin route with admin role and user is editor", func(t *testing.T) {
ctx, _ := setUp()
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/admin", cfg)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/admin", cfg, httpClientProvider)
require.NoError(t, err)
err = proxy.validateRequest()
require.Error(t, err)
@ -175,7 +176,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("plugin route with admin role and user is admin", func(t *testing.T) {
ctx, _ := setUp()
ctx.SignedInUser.OrgRole = models.ROLE_ADMIN
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/admin", cfg)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/admin", cfg, httpClientProvider)
require.NoError(t, err)
err = proxy.validateRequest()
require.NoError(t, err)
@ -257,7 +258,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
cfg := &setting.Cfg{}
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1", cfg)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1", cfg, httpClientProvider)
require.NoError(t, err)
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, plugin.Routes[0], proxy.ds, cfg)
@ -272,7 +273,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
req, err := http.NewRequest("GET", "http://localhost/asd", nil)
require.NoError(t, err)
client = newFakeHTTPClient(t, json2)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken2", cfg)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken2", cfg, httpClientProvider)
require.NoError(t, err)
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, plugin.Routes[1], proxy.ds, cfg)
@ -288,7 +289,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
require.NoError(t, err)
client = newFakeHTTPClient(t, []byte{})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1", cfg)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1", cfg, httpClientProvider)
require.NoError(t, err)
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, plugin.Routes[0], proxy.ds, cfg)
@ -304,17 +305,11 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
})
t.Run("When proxying graphite", func(t *testing.T) {
origBuildVer := setting.BuildVersion
t.Cleanup(func() {
setting.BuildVersion = origBuildVer
})
setting.BuildVersion = "5.3.0"
plugin := &plugins.DataSourcePlugin{}
ds := &models.DataSource{Url: "htttp://graphite:8080", Type: models.DS_GRAPHITE}
ctx := &models.ReqContext{}
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider)
require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
require.NoError(t, err)
@ -324,7 +319,6 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("Can translate request URL and path", func(t *testing.T) {
assert.Equal(t, "graphite:8080", req.URL.Host)
assert.Equal(t, "/render", req.URL.Path)
assert.Equal(t, "Grafana/5.3.0", req.Header.Get("User-Agent"))
})
})
@ -340,7 +334,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
}
ctx := &models.ReqContext{}
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{}, httpClientProvider)
require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
@ -363,7 +357,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
}
ctx := &models.ReqContext{}
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{}, httpClientProvider)
require.NoError(t, err)
requestURL, err := url.Parse("http://grafana.com/sub")
@ -390,7 +384,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
}
ctx := &models.ReqContext{}
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{}, httpClientProvider)
require.NoError(t, err)
requestURL, err := url.Parse("http://grafana.com/sub")
@ -411,7 +405,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
Url: "http://host/root/",
}
ctx := &models.ReqContext{}
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/", &setting.Cfg{})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider)
require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
req.Header.Set("Origin", "grafana.com")
@ -472,7 +466,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
Req: macaron.Request{Request: req},
},
}
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/", &setting.Cfg{})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider)
require.NoError(t, err)
req, err = http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
require.NoError(t, err)
@ -545,6 +539,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
// test DataSourceProxy request handling.
func TestDataSourceProxy_requestHandling(t *testing.T) {
httpClientProvider := httpclient.NewProvider()
var writeErr error
plugin := &plugins.DataSourcePlugin{}
@ -605,7 +600,7 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
t.Run("When response header Set-Cookie is not set should remove proxied Set-Cookie header", func(t *testing.T) {
ctx, ds := setUp(t)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}, httpClientProvider)
require.NoError(t, err)
proxy.HandleRequest()
@ -620,7 +615,7 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
"Set-Cookie": "important_cookie=important_value",
},
})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}, httpClientProvider)
require.NoError(t, err)
proxy.HandleRequest()
@ -639,7 +634,7 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
t.Log("Wrote 401 response")
},
})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}, httpClientProvider)
require.NoError(t, err)
proxy.HandleRequest()
@ -661,7 +656,7 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
})
ctx.Req.Request = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{})
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider)
require.NoError(t, err)
proxy.HandleRequest()
@ -685,7 +680,7 @@ func TestNewDataSourceProxy_InvalidURL(t *testing.T) {
}
cfg := setting.Cfg{}
plugin := plugins.DataSourcePlugin{}
_, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg)
_, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg, httpclient.NewProvider())
require.Error(t, err)
assert.True(t, strings.HasPrefix(err.Error(), `validation of data source URL "://host/root" failed`))
}
@ -704,7 +699,7 @@ func TestNewDataSourceProxy_ProtocolLessURL(t *testing.T) {
cfg := setting.Cfg{}
plugin := plugins.DataSourcePlugin{}
_, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg)
_, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg, httpclient.NewProvider())
require.NoError(t, err)
}
@ -744,7 +739,7 @@ func TestNewDataSourceProxy_MSSQL(t *testing.T) {
Url: tc.url,
}
p, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg)
p, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg, httpclient.NewProvider())
if tc.err == nil {
require.NoError(t, err)
assert.Equal(t, &url.URL{
@ -782,7 +777,7 @@ func getDatasourceProxiedRequest(t *testing.T, ctx *models.ReqContext, cfg *sett
Url: "http://host/root/",
}
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", cfg)
proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", cfg, httpclient.NewProvider())
require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
require.NoError(t, err)
@ -840,6 +835,7 @@ func createAuthTest(t *testing.T, dsType string, authType string, authCheck stri
test := &testCase{
datasource: &models.DataSource{
Id: 1,
Type: dsType,
JsonData: simplejson.New(),
},
@ -892,7 +888,7 @@ func createAuthTest(t *testing.T, dsType string, authType string, authCheck stri
func runDatasourceAuthTest(t *testing.T, test *testCase) {
plugin := &plugins.DataSourcePlugin{}
ctx := &models.ReqContext{}
proxy, err := NewDataSourceProxy(test.datasource, plugin, ctx, "", &setting.Cfg{})
proxy, err := NewDataSourceProxy(test.datasource, plugin, ctx, "", &setting.Cfg{}, httpclient.NewProvider())
require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
@ -933,7 +929,7 @@ func Test_PathCheck(t *testing.T) {
return ctx, req
}
ctx, _ := setUp()
proxy, err := NewDataSourceProxy(&models.DataSource{}, plugin, ctx, "b", &setting.Cfg{})
proxy, err := NewDataSourceProxy(&models.DataSource{}, plugin, ctx, "b", &setting.Cfg{}, httpclient.NewProvider())
require.NoError(t, err)
require.Nil(t, proxy.validateRequest())

105
pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware.go Normal file
View File

@ -0,0 +1,105 @@
package httpclientprovider
import (
"net/http"
"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana/pkg/infra/metrics/metricutil"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var datasourceRequestCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "grafana",
Name: "datasource_request_total",
Help: "A counter for outgoing requests for a datasource",
},
[]string{"datasource", "code", "method"},
)
var datasourceRequestSummary = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "grafana",
Name: "datasource_request_duration_seconds",
Help: "summary of outgoing datasource requests sent from Grafana",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}, []string{"datasource", "code", "method"},
)
var datasourceResponseSummary = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "grafana",
Name: "datasource_response_size_bytes",
Help: "summary of datasource response sizes returned to Grafana",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}, []string{"datasource"},
)
var datasourceRequestsInFlight = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "grafana",
Name: "datasource_request_in_flight",
Help: "A gauge of outgoing datasource requests currently being sent by Grafana",
},
[]string{"datasource"},
)
func init() {
prometheus.MustRegister(datasourceRequestSummary,
datasourceRequestCounter,
datasourceRequestsInFlight,
datasourceResponseSummary)
}
const DataSourceMetricsMiddlewareName = "metrics"
var executeMiddlewareFunc = executeMiddleware
func DataSourceMetricsMiddleware() httpclient.Middleware {
return httpclient.NamedMiddlewareFunc(DataSourceMetricsMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
if opts.Labels == nil {
return next
}
datasourceName, exists := opts.Labels["datasource_name"]
if !exists {
return next
}
datasourceLabelName, err := metricutil.SanitizeLabelName(datasourceName)
// if the datasource named cannot be turned into a prometheus
// label we will skip instrumenting these metrics.
if err != nil {
return next
}
datasourceLabel := prometheus.Labels{"datasource": datasourceLabelName}
return executeMiddlewareFunc(next, datasourceLabel)
})
}
func executeMiddleware(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper {
return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
requestCounter := datasourceRequestCounter.MustCurryWith(datasourceLabel)
requestSummary := datasourceRequestSummary.MustCurryWith(datasourceLabel)
requestInFlight := datasourceRequestsInFlight.With(datasourceLabel)
responseSizeSummary := datasourceResponseSummary.With(datasourceLabel)
res, err := promhttp.InstrumentRoundTripperDuration(requestSummary,
promhttp.InstrumentRoundTripperCounter(requestCounter,
promhttp.InstrumentRoundTripperInFlight(requestInFlight, next))).
RoundTrip(r)
if err != nil {
return nil, err
}
// we avoid measuring contentlength less than zero because it indicates
// that the content size is unknown. https://godoc.org/github.com/badu/http#Response
if res != nil && res.ContentLength > 0 {
responseSizeSummary.Observe(float64(res.ContentLength))
}
return res, nil
})
}

130
pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware_test.go Normal file
View File

@ -0,0 +1,130 @@
package httpclientprovider
import (
"net/http"
"testing"
"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
)
func TestDataSourceMetricsMiddleware(t *testing.T) {
t.Run("Without label options set should return next http.RoundTripper", func(t *testing.T) {
origExecuteMiddlewareFunc := executeMiddlewareFunc
executeMiddlewareCalled := false
middlewareCalled := false
executeMiddlewareFunc = func(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper {
executeMiddlewareCalled = true
return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
middlewareCalled = true
return next.RoundTrip(r)
})
}
t.Cleanup(func() {
executeMiddlewareFunc = origExecuteMiddlewareFunc
})
ctx := &testContext{}
finalRoundTripper := ctx.createRoundTripper("finalrt")
mw := DataSourceMetricsMiddleware()
rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper)
require.NotNil(t, rt)
middlewareName, ok := mw.(httpclient.MiddlewareName)
require.True(t, ok)
require.Equal(t, DataSourceMetricsMiddlewareName, middlewareName.MiddlewareName())
req, err := http.NewRequest(http.MethodGet, "http://", nil)
require.NoError(t, err)
res, err := rt.RoundTrip(req)
require.NoError(t, err)
require.NotNil(t, res)
if res.Body != nil {
require.NoError(t, res.Body.Close())
}
require.Len(t, ctx.callChain, 1)
require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain)
require.False(t, executeMiddlewareCalled)
require.False(t, middlewareCalled)
})
t.Run("Without data source name label options set should return next http.RoundTripper", func(t *testing.T) {
origExecuteMiddlewareFunc := executeMiddlewareFunc
executeMiddlewareCalled := false
middlewareCalled := false
executeMiddlewareFunc = func(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper {
executeMiddlewareCalled = true
return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
middlewareCalled = true
return next.RoundTrip(r)
})
}
t.Cleanup(func() {
executeMiddlewareFunc = origExecuteMiddlewareFunc
})
ctx := &testContext{}
finalRoundTripper := ctx.createRoundTripper("finalrt")
mw := DataSourceMetricsMiddleware()
rt := mw.CreateMiddleware(httpclient.Options{Labels: map[string]string{"test": "test"}}, finalRoundTripper)
require.NotNil(t, rt)
middlewareName, ok := mw.(httpclient.MiddlewareName)
require.True(t, ok)
require.Equal(t, DataSourceMetricsMiddlewareName, middlewareName.MiddlewareName())
req, err := http.NewRequest(http.MethodGet, "http://", nil)
require.NoError(t, err)
res, err := rt.RoundTrip(req)
require.NoError(t, err)
require.NotNil(t, res)
if res.Body != nil {
require.NoError(t, res.Body.Close())
}
require.Len(t, ctx.callChain, 1)
require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain)
require.False(t, executeMiddlewareCalled)
require.False(t, middlewareCalled)
})
t.Run("With datasource name label options set should execute middleware", func(t *testing.T) {
origExecuteMiddlewareFunc := executeMiddlewareFunc
executeMiddlewareCalled := false
datasourceLabels := prometheus.Labels{}
middlewareCalled := false
executeMiddlewareFunc = func(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper {
executeMiddlewareCalled = true
datasourceLabels = datasourceLabel
return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
middlewareCalled = true
return next.RoundTrip(r)
})
}
t.Cleanup(func() {
executeMiddlewareFunc = origExecuteMiddlewareFunc
})
ctx := &testContext{}
finalRoundTripper := ctx.createRoundTripper("finalrt")
mw := DataSourceMetricsMiddleware()
rt := mw.CreateMiddleware(httpclient.Options{Labels: map[string]string{"datasource_name": "My Data Source 123"}}, finalRoundTripper)
require.NotNil(t, rt)
middlewareName, ok := mw.(httpclient.MiddlewareName)
require.True(t, ok)
require.Equal(t, DataSourceMetricsMiddlewareName, middlewareName.MiddlewareName())
req, err := http.NewRequest(http.MethodGet, "http://", nil)
require.NoError(t, err)
res, err := rt.RoundTrip(req)
require.NoError(t, err)
require.NotNil(t, res)
if res.Body != nil {
require.NoError(t, res.Body.Close())
}
require.Len(t, ctx.callChain, 1)
require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain)
require.True(t, executeMiddlewareCalled)
require.Len(t, datasourceLabels, 1)
require.Equal(t, "My_Data_Source_123", datasourceLabels["datasource"])
require.True(t, middlewareCalled)
})
}

30
pkg/infra/httpclient/httpclientprovider/http_client_provider.go Normal file
View File

@ -0,0 +1,30 @@
package httpclientprovider
import (
"fmt"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/setting"
)
var newProviderFunc = sdkhttpclient.NewProvider
// New creates a new HTTP client provider with pre-configured middlewares.
func New(cfg *setting.Cfg) httpclient.Provider {
userAgent := fmt.Sprintf("Grafana/%s", cfg.BuildVersion)
middlewares := []sdkhttpclient.Middleware{
DataSourceMetricsMiddleware(),
SetUserAgentMiddleware(userAgent),
sdkhttpclient.BasicAuthenticationMiddleware(),
sdkhttpclient.CustomHeadersMiddleware(),
}
if cfg.SigV4AuthEnabled {
middlewares = append(middlewares, SigV4Middleware())
}
return newProviderFunc(sdkhttpclient.ProviderOptions{
Middlewares: middlewares,
})
}

52
pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go Normal file
View File

@ -0,0 +1,52 @@
package httpclientprovider
import (
"testing"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/require"
)
func TestHTTPClientProvider(t *testing.T) {
t.Run("When creating new provider and SigV4 is disabled should apply expected middleware", func(t *testing.T) {
origNewProviderFunc := newProviderFunc
providerOpts := []sdkhttpclient.ProviderOptions{}
newProviderFunc = func(opts ...sdkhttpclient.ProviderOptions) *sdkhttpclient.Provider {
providerOpts = opts
return nil
}
t.Cleanup(func() {
newProviderFunc = origNewProviderFunc
})
_ = New(&setting.Cfg{SigV4AuthEnabled: false})
require.Len(t, providerOpts, 1)
o := providerOpts[0]
require.Len(t, o.Middlewares, 4)
require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName())
})
t.Run("When creating new provider and SigV4 is enabled should apply expected middleware", func(t *testing.T) {
origNewProviderFunc := newProviderFunc
providerOpts := []sdkhttpclient.ProviderOptions{}
newProviderFunc = func(opts ...sdkhttpclient.ProviderOptions) *sdkhttpclient.Provider {
providerOpts = opts
return nil
}
t.Cleanup(func() {
newProviderFunc = origNewProviderFunc
})
_ = New(&setting.Cfg{SigV4AuthEnabled: true})
require.Len(t, providerOpts, 1)
o := providerOpts[0]
require.Len(t, o.Middlewares, 5)
require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, SigV4MiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName())
})
}

36
pkg/infra/httpclient/httpclientprovider/sigv4_middleware.go Normal file
View File

@ -0,0 +1,36 @@
package httpclientprovider
import (
"net/http"
"github.com/grafana/grafana-aws-sdk/pkg/sigv4"
"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
)
// SigV4MiddlewareName the middleware name used by SigV4Middleware.
const SigV4MiddlewareName = "sigv4"
var newSigV4Func = sigv4.New
// SigV4Middleware applies AWS Signature Version 4 request signing for the outgoing request.
func SigV4Middleware() httpclient.Middleware {
return httpclient.NamedMiddlewareFunc(SigV4MiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
if opts.SigV4 == nil {
return next
}
return newSigV4Func(
&sigv4.Config{
Service: opts.SigV4.Service,
AccessKey: opts.SigV4.AccessKey,
SecretKey: opts.SigV4.SecretKey,
Region: opts.SigV4.Region,
AssumeRoleARN: opts.SigV4.AssumeRoleARN,
AuthType: opts.SigV4.AuthType,
ExternalID: opts.SigV4.ExternalID,
Profile: opts.SigV4.Profile,
},
next,
)
})
}

89
pkg/infra/httpclient/httpclientprovider/sigv4_middleware_test.go Normal file
View File

@ -0,0 +1,89 @@
package httpclientprovider
import (
"net/http"
"testing"
"github.com/grafana/grafana-aws-sdk/pkg/sigv4"
"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/stretchr/testify/require"
)
func TestSigV4Middleware(t *testing.T) {
t.Run("Without sigv4 options set should return next http.RoundTripper", func(t *testing.T) {
origSigV4Func := newSigV4Func
newSigV4Called := false
middlewareCalled := false
newSigV4Func = func(config *sigv4.Config, next http.RoundTripper) http.RoundTripper {
newSigV4Called = true
return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
middlewareCalled = true
return next.RoundTrip(r)
})
}
t.Cleanup(func() {
newSigV4Func = origSigV4Func
})
ctx := &testContext{}
finalRoundTripper := ctx.createRoundTripper("finalrt")
mw := SigV4Middleware()
rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper)
require.NotNil(t, rt)
middlewareName, ok := mw.(httpclient.MiddlewareName)
require.True(t, ok)
require.Equal(t, SigV4MiddlewareName, middlewareName.MiddlewareName())
req, err := http.NewRequest(http.MethodGet, "http://", nil)
require.NoError(t, err)
res, err := rt.RoundTrip(req)
require.NoError(t, err)
require.NotNil(t, res)
if res.Body != nil {
require.NoError(t, res.Body.Close())
}
require.Len(t, ctx.callChain, 1)
require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain)
require.False(t, newSigV4Called)
require.False(t, middlewareCalled)
})
t.Run("With sigv4 options set should call sigv4 http.RoundTripper", func(t *testing.T) {
origSigV4Func := newSigV4Func
newSigV4Called := false
middlewareCalled := false
newSigV4Func = func(config *sigv4.Config, next http.RoundTripper) http.RoundTripper {
newSigV4Called = true
return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
middlewareCalled = true
return next.RoundTrip(r)
})
}
t.Cleanup(func() {
newSigV4Func = origSigV4Func
})
ctx := &testContext{}
finalRoundTripper := ctx.createRoundTripper("final")
mw := SigV4Middleware()
rt := mw.CreateMiddleware(httpclient.Options{SigV4: &httpclient.SigV4Config{}}, finalRoundTripper)
require.NotNil(t, rt)
middlewareName, ok := mw.(httpclient.MiddlewareName)
require.True(t, ok)
require.Equal(t, SigV4MiddlewareName, middlewareName.MiddlewareName())
req, err := http.NewRequest(http.MethodGet, "http://", nil)
require.NoError(t, err)
res, err := rt.RoundTrip(req)
require.NoError(t, err)
require.NotNil(t, res)
if res.Body != nil {
require.NoError(t, res.Body.Close())
}
require.Len(t, ctx.callChain, 1)
require.ElementsMatch(t, []string{"final"}, ctx.callChain)
require.True(t, newSigV4Called)
require.True(t, middlewareCalled)
})
}

18
pkg/infra/httpclient/httpclientprovider/testing.go Normal file
View File

@ -0,0 +1,18 @@
package httpclientprovider
import (
"net/http"
"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
)
type testContext struct {
callChain []string
}
func (c *testContext) createRoundTripper(name string) http.RoundTripper {
return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
c.callChain = append(c.callChain, name)
return &http.Response{StatusCode: http.StatusOK}, nil
})
}

27
pkg/infra/httpclient/httpclientprovider/user_agent_middleware.go Normal file
View File

@ -0,0 +1,27 @@
package httpclientprovider
import (
"net/http"
"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
)
// SetUserAgentMiddlewareName is the middleware name used by SetUserAgentMiddleware.
const SetUserAgentMiddlewareName = "user-agent"
// SetUserAgentMiddleware is middleware that sets the HTTP header User-Agent on the outgoing request.
// If User-Agent already set, it will not be overridden by this middleware.
func SetUserAgentMiddleware(userAgent string) httpclient.Middleware {
return httpclient.NamedMiddlewareFunc(SetUserAgentMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
if userAgent == "" {
return next
}
return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
if req.Header.Get("User-Agent") == "" {
req.Header.Set("User-Agent", userAgent)
}
return next.RoundTrip(req)
})
})
}

82
pkg/infra/httpclient/httpclientprovider/user_agent_middleware_test.go Normal file
View File

@ -0,0 +1,82 @@
package httpclientprovider
import (
"net/http"
"testing"
"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/stretchr/testify/require"
)
func TestCustomHeadersMiddleware(t *testing.T) {
t.Run("Without user agent set should return next http.RoundTripper", func(t *testing.T) {
ctx := &testContext{}
finalRoundTripper := ctx.createRoundTripper("finalrt")
mw := SetUserAgentMiddleware("")
rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper)
require.NotNil(t, rt)
middlewareName, ok := mw.(httpclient.MiddlewareName)
require.True(t, ok)
require.Equal(t, SetUserAgentMiddlewareName, middlewareName.MiddlewareName())
req, err := http.NewRequest(http.MethodGet, "http://", nil)
require.NoError(t, err)
res, err := rt.RoundTrip(req)
require.NoError(t, err)
require.NotNil(t, res)
if res.Body != nil {
require.NoError(t, res.Body.Close())
}
require.Len(t, ctx.callChain, 1)
require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain)
})
t.Run("With user agent set should apply HTTP headers to the request", func(t *testing.T) {
ctx := &testContext{}
finalRoundTripper := ctx.createRoundTripper("final")
mw := SetUserAgentMiddleware("Grafana/8.0.0")
rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper)
require.NotNil(t, rt)
middlewareName, ok := mw.(httpclient.MiddlewareName)
require.True(t, ok)
require.Equal(t, SetUserAgentMiddlewareName, middlewareName.MiddlewareName())
req, err := http.NewRequest(http.MethodGet, "http://", nil)
require.NoError(t, err)
res, err := rt.RoundTrip(req)
require.NoError(t, err)
require.NotNil(t, res)
if res.Body != nil {
require.NoError(t, res.Body.Close())
}
require.Len(t, ctx.callChain, 1)
require.ElementsMatch(t, []string{"final"}, ctx.callChain)
require.Equal(t, "Grafana/8.0.0", req.Header.Get("User-Agent"))
})
t.Run("With user agent set, but request already has User-Agent header set should not apply HTTP headers to the request", func(t *testing.T) {
ctx := &testContext{}
finalRoundTripper := ctx.createRoundTripper("final")
mw := SetUserAgentMiddleware("Grafana/8.0.0")
rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper)
require.NotNil(t, rt)
middlewareName, ok := mw.(httpclient.MiddlewareName)
require.True(t, ok)
require.Equal(t, SetUserAgentMiddlewareName, middlewareName.MiddlewareName())
req, err := http.NewRequest(http.MethodGet, "http://", nil)
require.NoError(t, err)
req.Header.Set("User-Agent", "ua")
res, err := rt.RoundTrip(req)
require.NoError(t, err)
require.NotNil(t, res)
if res.Body != nil {
require.NoError(t, res.Body.Close())
}
require.Len(t, ctx.callChain, 1)
require.ElementsMatch(t, []string{"final"}, ctx.callChain)
require.Equal(t, "ua", req.Header.Get("User-Agent"))
})
}

32
pkg/infra/httpclient/provider.go Normal file
View File

@ -0,0 +1,32 @@
package httpclient
import (
"crypto/tls"
"net/http"
"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
)
// Provider provides abilities to create http.Client, http.RoundTripper and tls.Config.
type Provider interface {
// New creates a new http.Client given provided options.
New(opts ...httpclient.Options) (*http.Client, error)
// GetTransport creates a new http.RoundTripper given provided options.
GetTransport(opts ...httpclient.Options) (http.RoundTripper, error)
// GetTLSConfig creates a new tls.Config given provided options.
GetTLSConfig(opts ...httpclient.Options) (*tls.Config, error)
}
// NewProvider creates a new HTTP client provider.
// Optionally provide ProviderOptions options that will be used as default if
// not specified in Options argument to Provider.New, Provider.GetTransport and
// Provider.GetTLSConfig.
// If no middlewares are provided in opts the DefaultMiddlewares() will be used. If you
// provide middlewares you have to manually add the DefaultMiddlewares() for it to be
// enabled.
// Note: Middlewares will be executed in the same order as provided.
func NewProvider(opts ...httpclient.ProviderOptions) *httpclient.Provider {
return httpclient.NewProvider(opts...)
}

278
pkg/models/datasource_cache.go
View File

@ -2,128 +2,17 @@ package models
import (
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"net"
"net/http"
"sync"
"time"
"github.com/grafana/grafana-aws-sdk/pkg/sigv4"
"github.com/grafana/grafana/pkg/infra/metrics/metricutil"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/setting"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var datasourceRequestCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "grafana",
Name: "datasource_request_total",
Help: "A counter for outgoing requests for a datasource",
},
[]string{"datasource", "code", "method"},
)
var datasourceRequestSummary = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "grafana",
Name: "datasource_request_duration_seconds",
Help: "summary of outgoing datasource requests sent from Grafana",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}, []string{"datasource", "code", "method"},
)
var datasourceResponseSummary = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "grafana",
Name: "datasource_response_size_bytes",
Help: "summary of datasource response sizes returned to Grafana",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}, []string{"datasource"},
)
var datasourceRequestsInFlight = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "grafana",
Name: "datasource_request_in_flight",
Help: "A gauge of outgoing datasource requests currently being sent by Grafana",
},
[]string{"datasource"},
)
func init() {
prometheus.MustRegister(datasourceRequestSummary,
datasourceRequestCounter,
datasourceRequestsInFlight,
datasourceResponseSummary)
}
type proxyTransportCache struct {
cache map[int64]cachedTransport
sync.Mutex
}
// dataSourceTransport implements http.RoundTripper (https://golang.org/pkg/net/http/#RoundTripper)
type dataSourceTransport struct {
datasourceName string
headers map[string]string
transport *http.Transport
next http.RoundTripper
}
func instrumentRoundtrip(datasourceName string, next http.RoundTripper) promhttp.RoundTripperFunc {
return promhttp.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
datasourceLabelName, err := metricutil.SanitizeLabelName(datasourceName)
// if the datasource named cannot be turned into a prometheus
// label we will skip instrumenting these metrics.
if err != nil {
return next.RoundTrip(r)
}
datasourceLabel := prometheus.Labels{"datasource": datasourceLabelName}
requestCounter := datasourceRequestCounter.MustCurryWith(datasourceLabel)
requestSummary := datasourceRequestSummary.MustCurryWith(datasourceLabel)
requestInFlight := datasourceRequestsInFlight.With(datasourceLabel)
responseSizeSummary := datasourceResponseSummary.With(datasourceLabel)
res, err := promhttp.InstrumentRoundTripperDuration(requestSummary,
promhttp.InstrumentRoundTripperCounter(requestCounter,
promhttp.InstrumentRoundTripperInFlight(requestInFlight, next))).
RoundTrip(r)
// we avoid measuring contentlength less than zero because it indicates
// that the content size is unknown. https://godoc.org/github.com/badu/http#Response
if res != nil && res.ContentLength > 0 {
responseSizeSummary.Observe(float64(res.ContentLength))
}
return res, err
})
}
// RoundTrip executes a single HTTP transaction, returning a Response for the provided Request.
func (d *dataSourceTransport) RoundTrip(req *http.Request) (*http.Response, error) {
for key, value := range d.headers {
req.Header.Set(key, value)
}
return instrumentRoundtrip(d.datasourceName, d.next).RoundTrip(req)
}
type cachedTransport struct {
updated time.Time
*dataSourceTransport
}
var ptc = proxyTransportCache{
cache: make(map[int64]cachedTransport),
}
func (ds *DataSource) getTimeout() time.Duration {
timeout := 0
if ds.JsonData != nil {
@ -135,8 +24,22 @@ func (ds *DataSource) getTimeout() time.Duration {
return time.Duration(timeout) * time.Second
}
func (ds *DataSource) GetHttpClient() (*http.Client, error) {
transport, err := ds.GetHttpTransport()
type proxyTransportCache struct {
cache map[int64]cachedRoundTripper
sync.Mutex
}
type cachedRoundTripper struct {
updated time.Time
roundTripper http.RoundTripper
}
var ptc = proxyTransportCache{
cache: make(map[int64]cachedRoundTripper),
}
func (ds *DataSource) GetHTTPClient(provider httpclient.Provider) (*http.Client, error) {
transport, err := ds.GetHTTPTransport(provider)
if err != nil {
return nil, err
}
@ -147,79 +50,86 @@ func (ds *DataSource) GetHttpClient() (*http.Client, error) {
}, nil
}
// Creates a HTTP Transport middleware chain
func (ds *DataSource) GetHttpTransport() (*dataSourceTransport, error) {
func (ds *DataSource) GetHTTPTransport(provider httpclient.Provider) (http.RoundTripper, error) {
ptc.Lock()
defer ptc.Unlock()
if t, present := ptc.cache[ds.Id]; present && ds.Updated.Equal(t.updated) {
return t.dataSourceTransport, nil
return t.roundTripper, nil
}
tlsConfig, err := ds.GetTLSConfig()
rt, err := provider.GetTransport(ds.HTTPClientOptions())
if err != nil {
return nil, err
}
tlsConfig.Renegotiation = tls.RenegotiateFreelyAsClient
// Create transport which adds all
customHeaders := ds.getCustomHeaders()
transport := &http.Transport{
TLSClientConfig: tlsConfig,
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: ds.getTimeout(),
KeepAlive: time.Duration(setting.DataProxyKeepAlive) * time.Second,
}).Dial,
TLSHandshakeTimeout: time.Duration(setting.DataProxyTLSHandshakeTimeout) * time.Second,
ExpectContinueTimeout: time.Duration(setting.DataProxyExpectContinueTimeout) * time.Second,
MaxIdleConns: setting.DataProxyMaxIdleConns,
IdleConnTimeout: time.Duration(setting.DataProxyIdleConnTimeout) * time.Second,
ptc.cache[ds.Id] = cachedRoundTripper{
roundTripper: rt,
updated: ds.Updated,
}
// Set default next round tripper to the default transport
next := http.RoundTripper(transport)
// Add SigV4 middleware if enabled, which will then defer to the default transport
if ds.JsonData != nil && ds.JsonData.Get("sigV4Auth").MustBool() && setting.SigV4AuthEnabled {
next = ds.sigV4Middleware(transport)
}
dsTransport := &dataSourceTransport{
datasourceName: ds.Name,
headers: customHeaders,
transport: transport,
next: next,
}
ptc.cache[ds.Id] = cachedTransport{
dataSourceTransport: dsTransport,
updated: ds.Updated,
}
return dsTransport, nil
return rt, nil
}
func (ds *DataSource) sigV4Middleware(next http.RoundTripper) http.RoundTripper {
decrypted := ds.DecryptedValues()
func (ds *DataSource) HTTPClientOptions() sdkhttpclient.Options {
tlsOptions := ds.TLSOptions()
opts := sdkhttpclient.Options{
Timeouts: &sdkhttpclient.TimeoutOptions{
Timeout: ds.getTimeout(),
KeepAlive: time.Duration(setting.DataProxyKeepAlive) * time.Second,
TLSHandshakeTimeout: time.Duration(setting.DataProxyTLSHandshakeTimeout) * time.Second,
ExpectContinueTimeout: time.Duration(setting.DataProxyExpectContinueTimeout) * time.Second,
MaxIdleConns: setting.DataProxyMaxIdleConns,
MaxIdleConnsPerHost: setting.DataProxyMaxIdleConnsPerHost,
IdleConnTimeout: time.Duration(setting.DataProxyIdleConnTimeout) * time.Second,
},
Headers: getCustomHeaders(ds.JsonData, ds.DecryptedValues()),
Labels: map[string]string{
"datasource_name": ds.Name,
"datasource_uid": ds.Uid,
},
TLS: &tlsOptions,
}
return sigv4.New(
&sigv4.Config{
if ds.JsonData != nil {
opts.CustomOptions = ds.JsonData.MustMap()
}
if ds.BasicAuth {
opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{
User: ds.BasicAuthUser,
Password: ds.DecryptedBasicAuthPassword(),
}
} else if ds.User != "" {
opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{
User: ds.User,
Password: ds.DecryptedPassword(),
}
}
if ds.JsonData != nil && ds.JsonData.Get("sigV4Auth").MustBool(false) {
opts.SigV4 = &sdkhttpclient.SigV4Config{
Service: awsServiceNamespace(ds.Type),
AccessKey: decrypted["sigV4AccessKey"],
SecretKey: decrypted["sigV4SecretKey"],
Region: ds.JsonData.Get("sigV4Region").MustString(),
AssumeRoleARN: ds.JsonData.Get("sigV4AssumeRoleArn").MustString(),
AuthType: ds.JsonData.Get("sigV4AuthType").MustString(),
ExternalID: ds.JsonData.Get("sigV4ExternalId").MustString(),
Profile: ds.JsonData.Get("sigV4Profile").MustString(),
},
next,
)
}
if val, exists := ds.DecryptedValue("sigV4AccessKey"); exists {
opts.SigV4.AccessKey = val
}
if val, exists := ds.DecryptedValue("sigV4SecretKey"); exists {
opts.SigV4.SecretKey = val
}
}
return opts
}
func (ds *DataSource) GetTLSConfig() (*tls.Config, error) {
func (ds *DataSource) TLSOptions() sdkhttpclient.TLSOptions {
var tlsSkipVerify, tlsClientAuth, tlsAuthWithCACert bool
var serverName string
@ -230,55 +140,55 @@ func (ds *DataSource) GetTLSConfig() (*tls.Config, error) {
serverName = ds.JsonData.Get("serverName").MustString()
}
tlsConfig := &tls.Config{
opts := sdkhttpclient.TLSOptions{
InsecureSkipVerify: tlsSkipVerify,
ServerName: serverName,
}
if tlsClientAuth || tlsAuthWithCACert {
decrypted := ds.SecureJsonData.Decrypt()
if tlsAuthWithCACert && len(decrypted["tlsCACert"]) > 0 {
caPool := x509.NewCertPool()
ok := caPool.AppendCertsFromPEM([]byte(decrypted["tlsCACert"]))
if !ok {
return nil, errors.New("failed to parse TLS CA PEM certificate")
if tlsAuthWithCACert {
if val, exists := ds.DecryptedValue("tlsCACert"); exists && len(val) > 0 {
opts.CACertificate = val
}
tlsConfig.RootCAs = caPool
}
if tlsClientAuth {
cert, err := tls.X509KeyPair([]byte(decrypted["tlsClientCert"]), []byte(decrypted["tlsClientKey"]))
if err != nil {
return nil, err
if val, exists := ds.DecryptedValue("tlsClientCert"); exists && len(val) > 0 {
opts.ClientCertificate = val
}
if val, exists := ds.DecryptedValue("tlsClientKey"); exists && len(val) > 0 {
opts.ClientKey = val
}
tlsConfig.Certificates = []tls.Certificate{cert}
}
}
return tlsConfig, nil
return opts
}
func (ds *DataSource) GetTLSConfig(httpClientProvider httpclient.Provider) (*tls.Config, error) {
return httpClientProvider.GetTLSConfig(ds.HTTPClientOptions())
}
// getCustomHeaders returns a map with all the to be set headers
// The map key represents the HeaderName and the value represents this header's value
func (ds *DataSource) getCustomHeaders() map[string]string {
func getCustomHeaders(jsonData *simplejson.Json, decryptedValues map[string]string) map[string]string {
headers := make(map[string]string)
if ds.JsonData == nil {
if jsonData == nil {
return headers
}
decrypted := ds.SecureJsonData.Decrypt()
index := 1
for {
headerNameSuffix := fmt.Sprintf("httpHeaderName%d", index)
headerValueSuffix := fmt.Sprintf("httpHeaderValue%d", index)
key := ds.JsonData.Get(headerNameSuffix).MustString()
key := jsonData.Get(headerNameSuffix).MustString()
if key == "" {
// No (more) header values are available
break
}
if val, ok := decrypted[headerValueSuffix]; ok {
if val, ok := decryptedValues[headerValueSuffix]; ok {
headers[key] = val
}
index++

163
pkg/models/datasource_cache_test.go
View File

@ -8,17 +8,25 @@ import (
"testing"
"time"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
//nolint:goconst
func TestDataSource_GetHttpTransport(t *testing.T) {
t.Run("Should use cached proxy", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
clearDSProxyCache(t)
ds := DataSource{
Id: 1,
@ -26,20 +34,30 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
Type: "Kubernetes",
}
tr1, err := ds.GetHttpTransport()
rt1, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt1)
tr1 := configuredTransport
tr2, err := ds.GetHttpTransport()
rt2, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt2)
tr2 := configuredTransport
require.Same(t, tr1, tr2)
assert.False(t, tr1.transport.TLSClientConfig.InsecureSkipVerify)
assert.Empty(t, tr1.transport.TLSClientConfig.Certificates)
assert.Nil(t, tr1.transport.TLSClientConfig.RootCAs)
require.False(t, tr1.TLSClientConfig.InsecureSkipVerify)
require.Empty(t, tr1.TLSClientConfig.Certificates)
require.Nil(t, tr1.TLSClientConfig.RootCAs)
})
t.Run("Should not use cached proxy when datasource updated", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
clearDSProxyCache(t)
setting.SecretKey = "password"
@ -56,26 +74,36 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
Updated: time.Now().Add(-2 * time.Minute),
}
tr1, err := ds.GetHttpTransport()
rt1, err := ds.GetHTTPTransport(provider)
require.NotNil(t, rt1)
require.NoError(t, err)
assert.False(t, tr1.transport.TLSClientConfig.InsecureSkipVerify)
assert.Empty(t, tr1.transport.TLSClientConfig.Certificates)
assert.Nil(t, tr1.transport.TLSClientConfig.RootCAs)
tr1 := configuredTransport
require.False(t, tr1.TLSClientConfig.InsecureSkipVerify)
require.Empty(t, tr1.TLSClientConfig.Certificates)
require.Nil(t, tr1.TLSClientConfig.RootCAs)
ds.JsonData = nil
ds.SecureJsonData = map[string][]byte{}
ds.Updated = time.Now()
tr2, err := ds.GetHttpTransport()
rt2, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt2)
tr2 := configuredTransport
require.NotSame(t, tr1, tr2)
assert.Nil(t, tr2.transport.TLSClientConfig.RootCAs)
require.Nil(t, tr2.TLSClientConfig.RootCAs)
})
t.Run("Should set TLS client authentication enabled if configured in JsonData", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
clearDSProxyCache(t)
setting.SecretKey = "password"
@ -98,15 +126,24 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
},
}
tr, err := ds.GetHttpTransport()
rt, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt)
tr := configuredTransport
assert.False(t, tr.transport.TLSClientConfig.InsecureSkipVerify)
require.Len(t, tr.transport.TLSClientConfig.Certificates, 1)
require.False(t, tr.TLSClientConfig.InsecureSkipVerify)
require.Len(t, tr.TLSClientConfig.Certificates, 1)
})
t.Run("Should set user-supplied TLS CA if configured in JsonData", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
clearDSProxyCache(t)
ClearDSDecryptionCache()
setting.SecretKey = "password"
json := simplejson.New()
@ -126,15 +163,23 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
},
}
tr, err := ds.GetHttpTransport()
rt, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt)
tr := configuredTransport
assert.False(t, tr.transport.TLSClientConfig.InsecureSkipVerify)
require.Len(t, tr.transport.TLSClientConfig.RootCAs.Subjects(), 1)
assert.Equal(t, "server-name", tr.transport.TLSClientConfig.ServerName)
require.False(t, tr.TLSClientConfig.InsecureSkipVerify)
require.Len(t, tr.TLSClientConfig.RootCAs.Subjects(), 1)
require.Equal(t, "server-name", tr.TLSClientConfig.ServerName)
})
t.Run("Should set skip TLS verification if configured in JsonData", func(t *testing.T) {
var configuredTransport *http.Transport
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredTransport = transport
},
})
clearDSProxyCache(t)
json := simplejson.New()
@ -147,19 +192,24 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
JsonData: json,
}
tr1, err := ds.GetHttpTransport()
rt1, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt1)
tr1 := configuredTransport
tr2, err := ds.GetHttpTransport()
rt2, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt2)
tr2 := configuredTransport
require.Same(t, tr1, tr2)
assert.True(t, tr1.transport.TLSClientConfig.InsecureSkipVerify)
require.True(t, tr1.TLSClientConfig.InsecureSkipVerify)
})
t.Run("Should set custom headers if configured in JsonData", func(t *testing.T) {
provider := httpclient.NewProvider()
clearDSProxyCache(t)
ClearDSDecryptionCache()
json := simplejson.NewFromAny(map[string]interface{}{
"httpHeaderName1": "Authorization",
@ -177,8 +227,8 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
SecureJsonData: map[string][]byte{"httpHeaderValue1": encryptedData},
}
headers := ds.getCustomHeaders()
assert.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"])
headers := getCustomHeaders(json, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"})
require.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"])
// 1. Start HTTP test server which checks the request headers
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -197,12 +247,13 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
// 2. Get HTTP transport from datasource which uses the test server as backend
ds.Url = backend.URL
tr, err := ds.GetHttpTransport()
rt, err := ds.GetHTTPTransport(provider)
require.NoError(t, err)
require.NotNil(t, rt)
// 3. Send test request which should have the Authorization header set
req := httptest.NewRequest("GET", backend.URL+"/test-headers", nil)
res, err := tr.RoundTrip(req)
res, err := rt.RoundTrip(req)
require.NoError(t, err)
t.Cleanup(func() {
err := res.Body.Close()
@ -211,10 +262,11 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
body, err := ioutil.ReadAll(res.Body)
require.NoError(t, err)
bodyStr := string(body)
assert.Equal(t, "Ok", bodyStr)
require.Equal(t, "Ok", bodyStr)
})
t.Run("Should use request timeout if configured in JsonData", func(t *testing.T) {
provider := httpclient.NewProvider()
clearDSProxyCache(t)
json := simplejson.NewFromAny(map[string]interface{}{
@ -227,47 +279,34 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
JsonData: json,
}
client, err := ds.GetHttpClient()
client, err := ds.GetHTTPClient(provider)
require.NoError(t, err)
assert.Equal(t, 19*time.Second, client.Timeout)
require.NotNil(t, client)
require.Equal(t, 19*time.Second, client.Timeout)
})
t.Run("Should not include SigV4 middleware if not configured in JsonData", func(t *testing.T) {
t.Run("Should populate SigV4 options if configured in JsonData", func(t *testing.T) {
var configuredOpts sdkhttpclient.Options
provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
configuredOpts = opts
},
})
clearDSProxyCache(t)
origEnabled := setting.SigV4AuthEnabled
setting.SigV4AuthEnabled = true
t.Cleanup(func() { setting.SigV4AuthEnabled = origEnabled })
ds := DataSource{}
tr, err := ds.GetHttpTransport()
require.NoError(t, err)
_, ok := tr.next.(*http.Transport)
require.True(t, ok)
})
t.Run("Should not include SigV4 middleware if not configured in app config", func(t *testing.T) {
clearDSProxyCache(t)
origEnabled := setting.SigV4AuthEnabled
setting.SigV4AuthEnabled = false
t.Cleanup(func() { setting.SigV4AuthEnabled = origEnabled })
json, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`))
require.NoError(t, err)
ds := DataSource{
Type: DS_ES,
JsonData: json,
}
tr, err := ds.GetHttpTransport()
_, err = ds.GetHTTPTransport(provider)
require.NoError(t, err)
_, ok := tr.next.(*http.Transport)
require.True(t, ok)
require.NotNil(t, configuredOpts)
require.NotNil(t, configuredOpts.SigV4)
require.Equal(t, "es", configuredOpts.SigV4.Service)
})
}
@ -288,7 +327,7 @@ func TestDataSource_DecryptedValue(t *testing.T) {
// Populate cache
password, ok := ds.DecryptedValue("password")
require.True(t, ok)
assert.Equal(t, "password", password)
require.Equal(t, "password", password)
ds.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{
"password": "",
@ -296,7 +335,7 @@ func TestDataSource_DecryptedValue(t *testing.T) {
password, ok = ds.DecryptedValue("password")
require.True(t, ok)
assert.Equal(t, "password", password)
require.Equal(t, "password", password)
})
t.Run("When datasource is updated, encrypted JSON should not be fetched from cache", func(t *testing.T) {
@ -315,7 +354,7 @@ func TestDataSource_DecryptedValue(t *testing.T) {
// Populate cache
password, ok := ds.DecryptedValue("password")
require.True(t, ok)
assert.Equal(t, "password", password)
require.Equal(t, "password", password)
ds.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{
"password": "",
@ -324,7 +363,7 @@ func TestDataSource_DecryptedValue(t *testing.T) {
password, ok = ds.DecryptedValue("password")
require.True(t, ok)
assert.Empty(t, password)
require.Empty(t, password)
})
}
@ -334,7 +373,7 @@ func clearDSProxyCache(t *testing.T) {
ptc.Lock()
defer ptc.Unlock()
ptc.cache = make(map[int64]cachedTransport)
ptc.cache = make(map[int64]cachedRoundTripper)
}
const caCert string = `-----BEGIN CERTIFICATE-----

2
pkg/server/server.go
View File

@ -19,6 +19,7 @@ import (
"github.com/grafana/grafana/pkg/api/routing"
"github.com/grafana/grafana/pkg/bus"
_ "github.com/grafana/grafana/pkg/extensions"
"github.com/grafana/grafana/pkg/infra/httpclient/httpclientprovider"
"github.com/grafana/grafana/pkg/infra/localcache"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/metrics"
@ -281,6 +282,7 @@ func (s *Server) buildServiceGraph(services []*registry.Descriptor) error {
s.cfg,
routing.NewRouteRegister(middleware.ProvideRouteOperationName, middleware.RequestMetrics(s.cfg)),
localcache.New(5*time.Minute, 10*time.Minute),
httpclientprovider.New(s.cfg),
s,
}
return registry.BuildServiceGraph(objs, services)

4
pkg/services/datasourceproxy/datasourceproxy.go
View File

@ -8,6 +8,7 @@ import (
"github.com/grafana/grafana/pkg/api/datasource"
"github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
@ -25,6 +26,7 @@ type DatasourceProxyService struct {
PluginRequestValidator models.PluginRequestValidator `inject:""`
PluginManager plugins.Manager `inject:""`
Cfg *setting.Cfg `inject:""`
HTTPClientProvider httpclient.Provider `inject:""`
}
func (p *DatasourceProxyService) Init() error {
@ -62,7 +64,7 @@ func (p *DatasourceProxyService) ProxyDatasourceRequestWithID(c *models.ReqConte
}
proxyPath := getProxyPath(c)
proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin, c, proxyPath, p.Cfg)
proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin, c, proxyPath, p.Cfg, p.HTTPClientProvider)
if err != nil {
if errors.Is(err, datasource.URLValidationError{}) {
c.JsonApiErr(http.StatusBadRequest, fmt.Sprintf("Invalid data source URL: %q", ds.Url), err)

2
pkg/setting/setting.go
View File

@ -81,6 +81,7 @@ var (
DataProxyTLSHandshakeTimeout int
DataProxyExpectContinueTimeout int
DataProxyMaxIdleConns int
DataProxyMaxIdleConnsPerHost int
DataProxyKeepAlive int
DataProxyIdleConnTimeout int
StaticRootPath string
@ -827,6 +828,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
DataProxyTLSHandshakeTimeout = dataproxy.Key("tls_handshake_timeout_seconds").MustInt(10)
DataProxyExpectContinueTimeout = dataproxy.Key("expect_continue_timeout_seconds").MustInt(1)
DataProxyMaxIdleConns = dataproxy.Key("max_idle_connections").MustInt(100)
DataProxyMaxIdleConnsPerHost = dataproxy.Key("max_idle_connections_per_host").MustInt(2)
DataProxyIdleConnTimeout = dataproxy.Key("idle_conn_timeout_seconds").MustInt(90)
cfg.SendUserHeader = dataproxy.Key("send_user_header").MustBool(false)

2
pkg/tsdb/azuremonitor/applicationinsights-datasource.go
View File

@ -241,8 +241,6 @@ func (e *ApplicationInsightsDatasource) createRequest(ctx context.Context, dsInf
return nil, errutil.Wrap("Failed to create request", err)
}
req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
pluginproxy.ApplyRoute(ctx, req, proxyPass, appInsightsRoute, dsInfo, e.cfg)
return req, nil

1
pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go
View File

@ -221,7 +221,6 @@ func (e *AzureLogAnalyticsDatasource) createRequest(ctx context.Context, dsInfo
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
// find plugin
plugin := e.pluginManager.GetDataSource(dsInfo.Type)

1
pkg/tsdb/azuremonitor/azuremonitor-datasource.go
View File

@ -254,7 +254,6 @@ func (e *AzureMonitorDatasource) createRequest(ctx context.Context, dsInfo *mode
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
pluginproxy.ApplyRoute(ctx, req, proxyPass, azureMonitorRoute, dsInfo, e.cfg)

8
pkg/tsdb/azuremonitor/azuremonitor.go
View File

@ -6,6 +6,7 @@ import (
"net/http"
"regexp"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
@ -29,8 +30,9 @@ func init() {
}
type Service struct {
PluginManager plugins.Manager `inject:""`
Cfg *setting.Cfg `inject:""`
PluginManager plugins.Manager `inject:""`
HTTPClientProvider httpclient.Provider `inject:""`
Cfg *setting.Cfg `inject:""`
}
func (s *Service) Init() error {
@ -48,7 +50,7 @@ type AzureMonitorExecutor struct {
// NewAzureMonitorExecutor initializes a http client
//nolint: staticcheck // plugins.DataPlugin deprecated
func (s *Service) NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
httpClient, err := dsInfo.GetHttpClient()
httpClient, err := dsInfo.GetHTTPClient(s.HTTPClientProvider)
if err != nil {
return nil, err
}

2
pkg/tsdb/azuremonitor/insights-analytics-datasource.go
View File

@ -215,8 +215,6 @@ func (e *InsightsAnalyticsDatasource) createRequest(ctx context.Context, dsInfo
return nil, errutil.Wrap("Failed to create request", err)
}
req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
pluginproxy.ApplyRoute(ctx, req, proxyPass, appInsightsRoute, dsInfo, e.cfg)
return req, nil

9
pkg/tsdb/cloudmonitoring/cloudmonitoring.go
View File

@ -22,6 +22,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
@ -75,8 +76,9 @@ func init() {
}
type Service struct {
PluginManager plugins.Manager `inject:""`
Cfg *setting.Cfg `inject:""`
PluginManager plugins.Manager `inject:""`
HTTPClientProvider httpclient.Provider `inject:""`
Cfg *setting.Cfg `inject:""`
}
func (s *Service) Init() error {
@ -94,7 +96,7 @@ type Executor struct {
// NewExecutor returns an Executor.
//nolint: staticcheck // plugins.DataPlugin deprecated
func (s *Service) NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
httpClient, err := dsInfo.GetHttpClient()
httpClient, err := dsInfo.GetHTTPClient(s.HTTPClientProvider)
if err != nil {
return nil, err
}
@ -540,7 +542,6 @@ func (e *Executor) createRequest(ctx context.Context, dsInfo *models.DataSource,
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
// find plugin
plugin := e.pluginManager.GetDataSource(dsInfo.Type)

35
pkg/tsdb/elasticsearch/client/client.go
View File

@ -15,6 +15,7 @@ import (
"github.com/Masterminds/semver"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/tsdb/interval"
@ -29,8 +30,8 @@ var (
clientLog = log.New(loggerName)
)
var newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) {
return ds.GetHttpClient()
var newDatasourceHttpClient = func(httpClientProvider httpclient.Provider, ds *models.DataSource) (*http.Client, error) {
return ds.GetHTTPClient(httpClientProvider)
}
// Client represents a client which can interact with elasticsearch api
@ -72,7 +73,7 @@ func coerceVersion(v *simplejson.Json) (*semver.Version, error) {
}
// NewClient creates a new elasticsearch client
var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange plugins.DataTimeRange) (Client, error) {
var NewClient = func(ctx context.Context, httpClientProvider httpclient.Provider, ds *models.DataSource, timeRange plugins.DataTimeRange) (Client, error) {
version, err := coerceVersion(ds.JsonData.Get("esVersion"))
if err != nil {
@ -108,13 +109,14 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange plugi
}
type baseClientImpl struct {
ctx context.Context
ds *models.DataSource
version *semver.Version
timeField string
indices []string
timeRange plugins.DataTimeRange
debugEnabled bool
ctx context.Context
httpClientProvider httpclient.Provider
ds *models.DataSource
version *semver.Version
timeField string
indices []string
timeRange plugins.DataTimeRange
debugEnabled bool
}
func (c *baseClientImpl) GetVersion() *semver.Version {
@ -208,20 +210,9 @@ func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body [
}
}
req.Header.Set("User-Agent", "Grafana")
req.Header.Set("Content-Type", "application/x-ndjson")
if c.ds.BasicAuth {
clientLog.Debug("Request configured to use basic authentication")
req.SetBasicAuth(c.ds.BasicAuthUser, c.ds.DecryptedBasicAuthPassword())
}
if !c.ds.BasicAuth && c.ds.User != "" {
clientLog.Debug("Request configured to use basic authentication")
req.SetBasicAuth(c.ds.User, c.ds.DecryptedPassword())
}
httpClient, err := newDatasourceHttpClient(c.ds)
httpClient, err := newDatasourceHttpClient(c.httpClientProvider, c.ds)
if err != nil {
return nil, err
}

25
pkg/tsdb/elasticsearch/client/client_test.go
View File

@ -11,6 +11,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/interval"
@ -24,7 +25,7 @@ func TestNewClient(t *testing.T) {
JsonData: simplejson.NewFromAny(make(map[string]interface{})),
}
_, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
_, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
require.Error(t, err)
})
@ -35,7 +36,7 @@ func TestNewClient(t *testing.T) {
}),
}
_, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
_, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
require.Error(t, err)
})
@ -48,7 +49,7 @@ func TestNewClient(t *testing.T) {
}),
}
_, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
_, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
require.Error(t, err)
})
@ -60,7 +61,7 @@ func TestNewClient(t *testing.T) {
}),
}
c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
require.NoError(t, err)
assert.Equal(t, "2.0.0", c.GetVersion().String())
})
@ -73,7 +74,7 @@ func TestNewClient(t *testing.T) {
}),
}
c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
require.NoError(t, err)
assert.Equal(t, "5.0.0", c.GetVersion().String())
})
@ -86,7 +87,7 @@ func TestNewClient(t *testing.T) {
}),
}
c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
require.NoError(t, err)
assert.Equal(t, "5.6.0", c.GetVersion().String())
})
@ -99,7 +100,7 @@ func TestNewClient(t *testing.T) {
}),
}
c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
require.NoError(t, err)
assert.Equal(t, "6.0.0", c.GetVersion().String())
})
@ -112,7 +113,7 @@ func TestNewClient(t *testing.T) {
}),
}
c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
require.NoError(t, err)
assert.Equal(t, "7.0.0", c.GetVersion().String())
})
@ -127,7 +128,7 @@ func TestNewClient(t *testing.T) {
}),
}
c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
require.NoError(t, err)
assert.Equal(t, version, c.GetVersion().String())
})
@ -141,7 +142,7 @@ func TestNewClient(t *testing.T) {
}),
}
_, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
_, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
require.Error(t, err)
})
}
@ -408,14 +409,14 @@ func httpClientScenario(t *testing.T, desc string, ds *models.DataSource, fn sce
toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond))
timeRange := plugins.NewDataTimeRange(fromStr, toStr)
c, err := NewClient(context.Background(), ds, timeRange)
c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, timeRange)
require.NoError(t, err)
require.NotNil(t, c)
sc.client = c
currentNewDatasourceHTTPClient := newDatasourceHttpClient
newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) {
newDatasourceHttpClient = func(httpClientProvider httpclient.Provider, ds *models.DataSource) (*http.Client, error) {
return ts.Client(), nil
}

21
pkg/tsdb/elasticsearch/elasticsearch.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
es "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client"
@ -12,15 +13,20 @@ import (
// ElasticsearchExecutor represents a handler for handling elasticsearch datasource request
type Executor struct {
httpClientProvider httpclient.Provider
intervalCalculator interval.Calculator
}
// NewExecutor creates a new Executor.
//nolint: staticcheck // plugins.DataPlugin deprecated
func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) {
return &Executor{
intervalCalculator: interval.NewCalculator(),
}, nil
// New creates a new Executor func.
// nolint:staticcheck // plugins.DataPlugin deprecated
func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) {
// nolint:staticcheck // plugins.DataPlugin deprecated
return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
return &Executor{
httpClientProvider: httpClientProvider,
intervalCalculator: interval.NewCalculator(),
}, nil
}
}
// Query handles an elasticsearch datasource request
@ -31,8 +37,7 @@ func (e *Executor) DataQuery(ctx context.Context, dsInfo *models.DataSource,
return plugins.DataResponse{}, fmt.Errorf("query contains no queries")
}
client, err := es.NewClient(ctx, dsInfo, *tsdbQuery.TimeRange)
client, err := es.NewClient(ctx, e.httpClientProvider, dsInfo, *tsdbQuery.TimeRange)
if err != nil {
return plugins.DataResponse{}, err
}

16
pkg/tsdb/graphite/graphite.go
View File

@ -15,6 +15,7 @@ import (
"golang.org/x/net/context/ctxhttp"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
@ -23,12 +24,17 @@ import (
)
type GraphiteExecutor struct {
HttpClient *http.Client
httpClientProvider httpclient.Provider
}
//nolint: staticcheck // plugins.DataPlugin deprecated
func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) {
return &GraphiteExecutor{}, nil
// nolint:staticcheck // plugins.DataPlugin deprecated
func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) {
// nolint:staticcheck // plugins.DataPlugin deprecated
return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
return &GraphiteExecutor{
httpClientProvider: httpClientProvider,
}, nil
}
}
var glog = log.New("tsdb.graphite")
@ -91,7 +97,7 @@ func (e *GraphiteExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSou
return plugins.DataResponse{}, err
}
httpClient, err := dsInfo.GetHttpClient()
httpClient, err := dsInfo.GetHTTPClient(e.httpClientProvider)
if err != nil {
return plugins.DataResponse{}, err
}

3
pkg/tsdb/influxdb/flux/executor_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/experimental"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -226,7 +227,7 @@ func TestRealQuery(t *testing.T) {
}),
}
runner, err := runnerFromDataSource(dsInfo)
runner, err := runnerFromDataSource(httpclient.NewProvider(), dsInfo)
require.NoError(t, err)
dr := executeQuery(context.Background(), queryModel{

9
pkg/tsdb/influxdb/flux/flux.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
@ -22,13 +23,13 @@ func init() {
// Query builds flux queries, executes them, and returns the results.
//nolint: staticcheck // plugins.DataQuery deprecated
func Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery plugins.DataQuery) (
func Query(ctx context.Context, httpClientProvider httpclient.Provider, dsInfo *models.DataSource, tsdbQuery plugins.DataQuery) (
plugins.DataResponse, error) {
glog.Debug("Received a query", "query", tsdbQuery)
tRes := plugins.DataResponse{
Results: make(map[string]plugins.DataQueryResult),
}
r, err := runnerFromDataSource(dsInfo)
r, err := runnerFromDataSource(httpClientProvider, dsInfo)
if err != nil {
return plugins.DataResponse{}, err
}
@ -69,7 +70,7 @@ func (r *runner) runQuery(ctx context.Context, fluxQuery string) (*api.QueryTabl
}
// runnerFromDataSource creates a runner from the datasource model (the datasource instance's configuration).
func runnerFromDataSource(dsInfo *models.DataSource) (*runner, error) {
func runnerFromDataSource(httpClientProvider httpclient.Provider, dsInfo *models.DataSource) (*runner, error) {
org := dsInfo.JsonData.Get("organization").MustString("")
if org == "" {
return nil, fmt.Errorf("missing organization in datasource configuration")
@ -85,7 +86,7 @@ func runnerFromDataSource(dsInfo *models.DataSource) (*runner, error) {
}
opts := influxdb2.DefaultOptions()
hc, err := dsInfo.GetHttpClient()
hc, err := dsInfo.GetHTTPClient(httpClientProvider)
if err != nil {
return nil, err
}

27
pkg/tsdb/influxdb/influxdb.go
View File

@ -9,6 +9,7 @@ import (
"path"
"strings"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
@ -17,17 +18,21 @@ import (
)
type Executor struct {
// *models.DataSource
QueryParser *InfluxdbQueryParser
ResponseParser *ResponseParser
httpClientProvider httpclient.Provider
QueryParser *InfluxdbQueryParser
ResponseParser *ResponseParser
}
//nolint: staticcheck // plugins.DataPlugin deprecated
func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) {
return &Executor{
QueryParser: &InfluxdbQueryParser{},
ResponseParser: &ResponseParser{},
}, nil
// nolint:staticcheck // plugins.DataPlugin deprecated
func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) {
// nolint:staticcheck // plugins.DataPlugin deprecated
return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
return &Executor{
httpClientProvider: httpClientProvider,
QueryParser: &InfluxdbQueryParser{},
ResponseParser: &ResponseParser{},
}, nil
}
}
var (
@ -47,7 +52,7 @@ func (e *Executor) DataQuery(ctx context.Context, dsInfo *models.DataSource, tsd
version := dsInfo.JsonData.Get("version").MustString("")
if version == "Flux" {
return flux.Query(ctx, dsInfo, tsdbQuery)
return flux.Query(ctx, e.httpClientProvider, dsInfo, tsdbQuery)
}
glog.Debug("Making a non-Flux type query")
@ -74,7 +79,7 @@ func (e *Executor) DataQuery(ctx context.Context, dsInfo *models.DataSource, tsd
return plugins.DataResponse{}, err
}
httpClient, err := dsInfo.GetHttpClient()
httpClient, err := dsInfo.GetHTTPClient(e.httpClientProvider)
if err != nil {
return plugins.DataResponse{}, err
}

22
pkg/tsdb/loki/loki.go
View File

@ -9,6 +9,7 @@ import (
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
@ -23,16 +24,17 @@ import (
type LokiExecutor struct {
intervalCalculator interval.Calculator
httpClientProvider httpclient.Provider
}
//nolint: staticcheck // plugins.DataPlugin deprecated
func NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
return newExecutor(), nil
}
func newExecutor() *LokiExecutor {
return &LokiExecutor{
intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}),
// nolint:staticcheck // plugins.DataPlugin deprecated
func New(httpClientProvider httpclient.Provider) func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
// nolint:staticcheck // plugins.DataPlugin deprecated
return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
return &LokiExecutor{
intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}),
httpClientProvider: httpClientProvider,
}, nil
}
}
@ -49,12 +51,12 @@ func (e *LokiExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSource,
Results: map[string]plugins.DataQueryResult{},
}
tlsConfig, err := dsInfo.GetTLSConfig()
tlsConfig, err := dsInfo.GetTLSConfig(e.httpClientProvider)
if err != nil {
return plugins.DataResponse{}, err
}
transport, err := dsInfo.GetHttpTransport()
transport, err := dsInfo.GetHTTPTransport(e.httpClientProvider)
if err != nil {
return plugins.DataResponse{}, err
}

13
pkg/tsdb/loki/loki_test.go
View File

@ -5,6 +5,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/loki/pkg/loghttp"
@ -60,9 +61,10 @@ func TestLoki(t *testing.T) {
TimeRange: &timeRange,
}
exe := newExecutor()
exe, err := New(httpclient.NewProvider())(dsInfo)
require.NoError(t, err)
models, err := exe.parseQuery(dsInfo, queryContext)
lokiExecutor := exe.(*LokiExecutor)
models, err := lokiExecutor.parseQuery(dsInfo, queryContext)
require.NoError(t, err)
require.Equal(t, time.Second*30, models[0].Step)
})
@ -82,15 +84,16 @@ func TestLoki(t *testing.T) {
{Model: jsonModel},
},
}
exe := newExecutor()
exe, err := New(httpclient.NewProvider())(dsInfo)
require.NoError(t, err)
models, err := exe.parseQuery(dsInfo, queryContext)
lokiExecutor := exe.(*LokiExecutor)
models, err := lokiExecutor.parseQuery(dsInfo, queryContext)
require.NoError(t, err)
require.Equal(t, time.Minute*2, models[0].Step)
timeRange = plugins.NewDataTimeRange("1h", "now")
queryContext.TimeRange = &timeRange
models, err = exe.parseQuery(dsInfo, queryContext)
models, err = lokiExecutor.parseQuery(dsInfo, queryContext)
require.NoError(t, err)
require.Equal(t, time.Second*2, models[0].Step)
})

78
pkg/tsdb/mysql/mysql.go
View File

@ -12,6 +12,7 @@ import (
"github.com/VividCortex/mysqlerr"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/setting"
"github.com/go-sql-driver/mysql"
@ -32,52 +33,55 @@ func characterEscape(s string, escapeChar string) string {
}
//nolint: staticcheck // plugins.DataPlugin deprecated
func NewExecutor(datasource *models.DataSource) (plugins.DataPlugin, error) {
logger := log.New("tsdb.mysql")
func New(httpClientProvider httpclient.Provider) func(datasource *models.DataSource) (plugins.DataPlugin, error) {
//nolint: staticcheck // plugins.DataPlugin deprecated
return func(datasource *models.DataSource) (plugins.DataPlugin, error) {
logger := log.New("tsdb.mysql")
protocol := "tcp"
if strings.HasPrefix(datasource.Url, "/") {
protocol = "unix"
}
protocol := "tcp"
if strings.HasPrefix(datasource.Url, "/") {
protocol = "unix"
}
cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC&allowNativePasswords=true",
characterEscape(datasource.User, ":"),
datasource.DecryptedPassword(),
protocol,
characterEscape(datasource.Url, ")"),
characterEscape(datasource.Database, "?"),
)
cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC&allowNativePasswords=true",
characterEscape(datasource.User, ":"),
datasource.DecryptedPassword(),
protocol,
characterEscape(datasource.Url, ")"),
characterEscape(datasource.Database, "?"),
)
tlsConfig, err := datasource.GetTLSConfig()
if err != nil {
return nil, err
}
if tlsConfig.RootCAs != nil || len(tlsConfig.Certificates) > 0 {
tlsConfigString := fmt.Sprintf("ds%d", datasource.Id)
if err := mysql.RegisterTLSConfig(tlsConfigString, tlsConfig); err != nil {
tlsConfig, err := datasource.GetTLSConfig(httpClientProvider)
if err != nil {
return nil, err
}
cnnstr += "&tls=" + tlsConfigString
}
if setting.Env == setting.Dev {
logger.Debug("getEngine", "connection", cnnstr)
}
if tlsConfig.RootCAs != nil || len(tlsConfig.Certificates) > 0 {
tlsConfigString := fmt.Sprintf("ds%d", datasource.Id)
if err := mysql.RegisterTLSConfig(tlsConfigString, tlsConfig); err != nil {
return nil, err
}
cnnstr += "&tls=" + tlsConfigString
}
config := sqleng.DataPluginConfiguration{
DriverName: "mysql",
ConnectionString: cnnstr,
Datasource: datasource,
TimeColumnNames: []string{"time", "time_sec"},
MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
}
if setting.Env == setting.Dev {
logger.Debug("getEngine", "connection", cnnstr)
}
rowTransformer := mysqlQueryResultTransformer{
log: logger,
}
config := sqleng.DataPluginConfiguration{
DriverName: "mysql",
ConnectionString: cnnstr,
Datasource: datasource,
TimeColumnNames: []string{"time", "time_sec"},
MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
}
return sqleng.NewDataPlugin(config, &rowTransformer, newMysqlMacroEngine(logger), logger)
rowTransformer := mysqlQueryResultTransformer{
log: logger,
}
return sqleng.NewDataPlugin(config, &rowTransformer, newMysqlMacroEngine(logger), logger)
}
}
type mysqlQueryResultTransformer struct {

3
pkg/tsdb/mysql/mysql_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/sqlstore"
@ -56,7 +57,7 @@ func TestMySQL(t *testing.T) {
return sql, nil
}
exe, err := NewExecutor(&models.DataSource{
exe, err := New(httpclient.NewProvider())(&models.DataSource{
JsonData: simplejson.New(),
SecureJsonData: securejsondata.SecureJsonData{},
})

15
pkg/tsdb/opentsdb/opentsdb.go
View File

@ -15,6 +15,7 @@ import (
"net/url"
"github.com/grafana/grafana/pkg/components/null"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
@ -22,11 +23,17 @@ import (
)
type OpenTsdbExecutor struct {
httpClientProvider httpclient.Provider
}
// nolint:staticcheck // plugins.DataQueryResult deprecated
func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) {
return &OpenTsdbExecutor{}, nil
//nolint: staticcheck // plugins.DataPlugin deprecated
func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) {
//nolint: staticcheck // plugins.DataPlugin deprecated
return func(*models.DataSource) (plugins.DataPlugin, error) {
return &OpenTsdbExecutor{
httpClientProvider: httpClientProvider,
}, nil
}
}
var (
@ -56,7 +63,7 @@ func (e *OpenTsdbExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSou
return plugins.DataResponse{}, err
}
httpClient, err := dsInfo.GetHttpClient()
httpClient, err := dsInfo.GetHTTPClient(e.httpClientProvider)
if err != nil {
return plugins.DataResponse{}, err
}

22
pkg/tsdb/prometheus/prometheus.go
View File

@ -14,6 +14,7 @@ import (
"net/http"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
@ -65,13 +66,20 @@ func (transport *prometheusTransport) RoundTrip(req *http.Request) (*http.Respon
}
//nolint: staticcheck // plugins.DataPlugin deprecated
func NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
return &PrometheusExecutor{
intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}),
baseRoundTripperFactory: func(ds *models.DataSource) (http.RoundTripper, error) {
return ds.GetHttpTransport()
},
}, nil
func New(provider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) {
return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
transport, err := dsInfo.GetHTTPTransport(provider)
if err != nil {
return nil, err
}
return &PrometheusExecutor{
intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}),
baseRoundTripperFactory: func(ds *models.DataSource) (http.RoundTripper, error) {
return transport, nil
},
}, nil
}
}
var (

3
pkg/tsdb/prometheus/prometheus_test.go
View File

@ -8,6 +8,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
p "github.com/prometheus/common/model"
@ -21,7 +22,7 @@ func TestPrometheus(t *testing.T) {
dsInfo := &models.DataSource{
JsonData: json,
}
plug, err := NewExecutor(dsInfo)
plug, err := New(httpclient.NewProvider())(dsInfo)
require.NoError(t, err)
executor := plug.(*PrometheusExecutor)

18
pkg/tsdb/service.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/registry"
@ -47,6 +48,7 @@ type Service struct {
CloudMonitoringService *cloudmonitoring.Service `inject:""`
AzureMonitorService *azuremonitor.Service `inject:""`
PluginManager plugins.Manager `inject:""`
HTTPClientProvider httpclient.Provider `inject:""`
//nolint: staticcheck // plugins.DataPlugin deprecated
registry map[string]func(*models.DataSource) (plugins.DataPlugin, error)
@ -54,18 +56,18 @@ type Service struct {
// Init initialises the service.
func (s *Service) Init() error {
s.registry["graphite"] = graphite.NewExecutor
s.registry["opentsdb"] = opentsdb.NewExecutor
s.registry["prometheus"] = prometheus.NewExecutor
s.registry["influxdb"] = influxdb.NewExecutor
s.registry["graphite"] = graphite.New(s.HTTPClientProvider)
s.registry["opentsdb"] = opentsdb.New(s.HTTPClientProvider)
s.registry["prometheus"] = prometheus.New(s.HTTPClientProvider)
s.registry["influxdb"] = influxdb.New(s.HTTPClientProvider)
s.registry["mssql"] = mssql.NewExecutor
s.registry["postgres"] = s.PostgresService.NewExecutor
s.registry["mysql"] = mysql.NewExecutor
s.registry["elasticsearch"] = elasticsearch.NewExecutor
s.registry["mysql"] = mysql.New(s.HTTPClientProvider)
s.registry["elasticsearch"] = elasticsearch.New(s.HTTPClientProvider)
s.registry["stackdriver"] = s.CloudMonitoringService.NewExecutor
s.registry["grafana-azure-monitor-datasource"] = s.AzureMonitorService.NewExecutor
s.registry["loki"] = loki.NewExecutor
s.registry["tempo"] = tempo.NewExecutor
s.registry["loki"] = loki.New(s.HTTPClientProvider)
s.registry["tempo"] = tempo.New(s.HTTPClientProvider)
return nil
}

22
pkg/tsdb/tempo/tempo.go
View File

@ -7,6 +7,7 @@ import (
"net/http"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
@ -18,17 +19,20 @@ type tempoExecutor struct {
httpClient *http.Client
}
// NewExecutor returns a tempoExecutor.DataQueryResult
// NewExecutor returns a tempoExecutor.
//nolint: staticcheck // plugins.DataPlugin deprecated
func NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
httpClient, err := dsInfo.GetHttpClient()
if err != nil {
return nil, err
}
func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) {
//nolint: staticcheck // plugins.DataPlugin deprecated
return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
httpClient, err := dsInfo.GetHTTPClient(httpClientProvider)
if err != nil {
return nil, err
}
return &tempoExecutor{
httpClient: httpClient,
}, nil
return &tempoExecutor{
httpClient: httpClient,
}, nil
}
}
var (

3
pkg/tsdb/tempo/tempo_test.go
View File

@ -4,13 +4,14 @@ import (
"context"
"testing"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestTempo(t *testing.T) {
plug, err := NewExecutor(&models.DataSource{})
plug, err := New(httpclient.NewProvider())(&models.DataSource{})
executor := plug.(*tempoExecutor)
require.NoError(t, err)