Fix: Add additional settings for dataproxy to help with network proxy timeouts (#27841)

* adding additional settings for datasource cache transport

* added documentation for the new changes

* fixing small typo in defaults.ini comment

* fixing small typo in configuration.md comment

* Update conf/defaults.ini keepalive comment per review

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

* Update conf/defaults.ini idle conn comment per review

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

* Update conf/defaults.ini anon user comment per review

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

* Update docs/sources/administration/configuration.md idle conn comment per review

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

* adding suggestions from papagian

* fixing configuration.md

* fixing configuration.md typo

* Apply suggestions from code review aknuds1

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

* updating sample.ini

* Apply suggestions for docs from code review papagian

Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com>

* Update docs/sources/administration/configuration.md fix typo

Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com>

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com>

conf/defaults.ini

@@ -130,7 +130,25 @@ logging = false
 # This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set.
 timeout = 30
 
-# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false.
+# How many seconds the data proxy waits before sending a keepalive request.
+keep_alive_seconds = 30
+
+# How many seconds the data proxy waits for a successful TLS Handshake before timing out.
+tls_handshake_timeout_seconds = 10
+
+# How many seconds the data proxy will wait for a server's first response headers after
+# fully writing the request headers if the request has an "Expect: 100-continue"
+# header. A value of 0 will result in the body being sent immediately, without
+# waiting for the server to approve.
+expect_continue_timeout_seconds = 1
+
+# The maximum number of idle connections that Grafana will keep alive.
+max_idle_connections = 100
+
+# How many seconds the data proxy keeps an idle connection open before timing out.
+idle_conn_timeout_seconds = 90
+
+# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request.
 send_user_header = false
 
 #################################### Analytics ###########################

conf/sample.ini

@@ -131,6 +131,24 @@
 # This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set.
 ;timeout = 30
 
+# How many seconds the data proxy waits before sending a keepalive probe request.
+;keep_alive_seconds = 30
+
+# How many seconds the data proxy waits for a successful TLS Handshake before timing out.
+;tls_handshake_timeout_seconds = 10
+
+# How many seconds the data proxy will wait for a server's first response headers after
+# fully writing the request headers if the request has an "Expect: 100-continue"
+# header. A value of 0 will result in the body being sent immediately, without
+# waiting for the server to approve.
+;expect_continue_timeout_seconds = 1
+
+# The maximum number of idle connections that Grafana will keep alive.
+;max_idle_connections = 100
+
+# How many seconds the data proxy keeps an idle connection open before timing out.
+;idle_conn_timeout_seconds = 90
+
 # If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false.
 ;send_user_header = false
 

docs/sources/administration/configuration.md

@@ -389,6 +389,26 @@ How long the data proxy should wait before timing out. Default is 30 seconds.
 
 This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set.
 
+### keep_alive_seconds
+
+Interval between keep-alive probes. Default is `30` seconds. For more details check the [Dialer.KeepAlive](https://golang.org/pkg/net/#Dialer.KeepAlive) documentation.
+
+### tls_handshake_timeout_seconds
+
+The length of time that Grafana will wait for a successful TLS handshake with the datasource. Default is `10` seconds. For more details check the [Transport.TLSHandshakeTimeout](https://golang.org/pkg/net/http/#Transport.TLSHandshakeTimeout) documentation.
+
+### expect_continue_timeout_seconds
+
+The length of time that Grafana will wait for a datasource’s first response headers after fully writing the request headers, if the request has an “Expect: 100-continue” header.  A value of `0` will result in the body being sent immediately. Default is `1` second. For more details check the [Transport.ExpectContinueTimeout](https://golang.org/pkg/net/http/#Transport.ExpectContinueTimeout) documentation.
+
+### max_idle_connections
+
+The maximum number of idle connections that Grafana will maintain. Default is `100`. For more details check the [Transport.MaxIdleConns](https://golang.org/pkg/net/http/#Transport.MaxIdleConns) documentation.
+
+### idle_conn_timeout_seconds
+
+The length of time that Grafana maintains idle connections before closing them. Default is `90` seconds. For more details check the [Transport.IdleConnTimeout](https://golang.org/pkg/net/http/#Transport.IdleConnTimeout) documentation.
+
 ### send_user_header
 
 If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request. Default is `false`.

pkg/models/datasource_cache.go

@@ -157,12 +157,12 @@ func (ds *DataSource) GetHttpTransport() (*dataSourceTransport, error) {
 		Proxy:           http.ProxyFromEnvironment,
 		Dial: (&net.Dialer{
 			Timeout:   time.Duration(setting.DataProxyTimeout) * time.Second,
-			KeepAlive: 30 * time.Second,
+			KeepAlive: time.Duration(setting.DataProxyKeepAlive) * time.Second,
 		}).Dial,
-		TLSHandshakeTimeout:   10 * time.Second,
+		TLSHandshakeTimeout:   time.Duration(setting.DataProxyTLSHandshakeTimeout) * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
+		ExpectContinueTimeout: time.Duration(setting.DataProxyExpectContinueTimeout) * time.Second,
-		MaxIdleConns:          100,
+		MaxIdleConns:          setting.DataProxyMaxIdleConns,
-		IdleConnTimeout:       90 * time.Second,
+		IdleConnTimeout:       time.Duration(setting.DataProxyIdleConnTimeout) * time.Second,
 	}
 
 	// Set default next round tripper to the default transport

pkg/setting/setting.go

@@ -78,18 +78,23 @@ var (
 	LogConfigs []util.DynMap
 
 	// Http server options
-	Protocol           Scheme
+	Protocol                       Scheme
-	Domain             string
+	Domain                         string
-	HttpAddr, HttpPort string
+	HttpAddr, HttpPort             string
-	SshPort            int
+	SshPort                        int
-	CertFile, KeyFile  string
+	CertFile, KeyFile              string
-	SocketPath         string
+	SocketPath                     string
-	RouterLogging      bool
+	RouterLogging                  bool
-	DataProxyLogging   bool
+	DataProxyLogging               bool
-	DataProxyTimeout   int
+	DataProxyTimeout               int
-	StaticRootPath     string
+	DataProxyTLSHandshakeTimeout   int
-	EnableGzip         bool
+	DataProxyExpectContinueTimeout int
-	EnforceDomain      bool
+	DataProxyMaxIdleConns          int
+	DataProxyKeepAlive             int
+	DataProxyIdleConnTimeout       int
+	StaticRootPath                 string
+	EnableGzip                     bool
+	EnforceDomain                  bool
 
 	// Security settings.
 	SecretKey                         string
@@ -684,6 +689,11 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
 	dataproxy := iniFile.Section("dataproxy")
 	DataProxyLogging = dataproxy.Key("logging").MustBool(false)
 	DataProxyTimeout = dataproxy.Key("timeout").MustInt(30)
+	DataProxyKeepAlive = dataproxy.Key("keep_alive_seconds").MustInt(30)
+	DataProxyTLSHandshakeTimeout = dataproxy.Key("tls_handshake_timeout_seconds").MustInt(10)
+	DataProxyExpectContinueTimeout = dataproxy.Key("expect_continue_timeout_seconds").MustInt(1)
+	DataProxyMaxIdleConns = dataproxy.Key("max_idle_connections").MustInt(100)
+	DataProxyIdleConnTimeout = dataproxy.Key("idle_conn_timeout_seconds").MustInt(90)
 	cfg.SendUserHeader = dataproxy.Key("send_user_header").MustBool(false)
 
 	if err := readSecuritySettings(iniFile, cfg); err != nil {