MM-10417 Improve HTTPService for use in image proxy (#9966)

* Replaced httpservice with proper http.Client

* Added HTTPService.MakeTransport

* Expose timeouts used by HTTPServiceImpl

* Add additional documentation to HTTPService

* Remove MockedHTTPService

* Fix missing license

services/httpservice/client.go

@@ -15,8 +15,8 @@ import (
 )
 
 const (
-	connectTimeout = 3 * time.Second
-	requestTimeout = 30 * time.Second
+	ConnectTimeout = 3 * time.Second
+	RequestTimeout = 30 * time.Second
 )
 
 var reservedIPRanges []*net.IPNet
@@ -102,25 +102,9 @@ func dialContextFilter(dial DialContextFunction, allowHost func(host string) boo
 	}
 }
 
-type Client struct {
-	*http.Client
-}
-
-func (c *Client) Do(req *http.Request) (*http.Response, error) {
-	req.Header.Set("User-Agent", defaultUserAgent)
-	return c.Client.Do(req)
-}
-
-// NewHTTPClient returns a variation the default implementation of Client.
-// It uses a Transport with the same settings as the default Transport
-// but with the following modifications:
-// - shorter timeout for dial and TLS handshake (defined as constant
-//   "connectTimeout")
-// - timeout for the end-to-end request (defined as constant
-//   "requestTimeout")
-func NewHTTPClient(enableInsecureConnections bool, allowHost func(host string) bool, allowIP func(ip net.IP) bool) *Client {
+func NewTransport(enableInsecureConnections bool, allowHost func(host string) bool, allowIP func(ip net.IP) bool) http.RoundTripper {
 	dialContext := (&net.Dialer{
-		Timeout:   connectTimeout,
+		Timeout:   ConnectTimeout,
 		KeepAlive: 30 * time.Second,
 	}).DialContext
 
@@ -128,20 +112,24 @@ func NewHTTPClient(enableInsecureConnections bool, allowHost func(host string) b
 		dialContext = dialContextFilter(dialContext, allowHost, allowIP)
 	}
 
-	client := &http.Client{
-		Transport: &http.Transport{
+	return &MattermostTransport{
+		&http.Transport{
 			Proxy:                 http.ProxyFromEnvironment,
 			DialContext:           dialContext,
 			MaxIdleConns:          100,
 			IdleConnTimeout:       90 * time.Second,
-			TLSHandshakeTimeout:   connectTimeout,
+			TLSHandshakeTimeout:   ConnectTimeout,
 			ExpectContinueTimeout: 1 * time.Second,
 			TLSClientConfig: &tls.Config{
 				InsecureSkipVerify: enableInsecureConnections,
 			},
 		},
-		Timeout: requestTimeout,
 	}
-
-	return &Client{Client: client}
+}
+
+func NewHTTPClient(transport http.RoundTripper) *http.Client {
+	return &http.Client{
+		Transport: transport,
+		Timeout:   RequestTimeout,
+	}
 }

services/httpservice/client_test.go

@@ -16,7 +16,7 @@ import (
 
 func TestHTTPClient(t *testing.T) {
 	for _, allowInternal := range []bool{true, false} {
-		c := NewHTTPClient(false, func(_ string) bool { return false }, func(ip net.IP) bool { return allowInternal || !IsReservedIP(ip) })
+		c := NewHTTPClient(NewTransport(false, func(_ string) bool { return false }, func(ip net.IP) bool { return allowInternal || !IsReservedIP(ip) }))
 		for _, tc := range []struct {
 			URL        string
 			IsInternal bool
@@ -56,9 +56,9 @@ func TestHTTPClientWithProxy(t *testing.T) {
 	proxy := createProxyServer()
 	defer proxy.Close()
 
-	c := NewHTTPClient(true, nil, nil)
+	c := NewHTTPClient(NewTransport(true, nil, nil))
 	purl, _ := url.Parse(proxy.URL)
-	c.Transport.(*http.Transport).Proxy = http.ProxyURL(purl)
+	c.Transport.(*MattermostTransport).Transport.(*http.Transport).Proxy = http.ProxyURL(purl)
 
 	resp, err := c.Get("http://acme.com")
 	if err != nil {
@@ -132,7 +132,7 @@ func TestUserAgentIsSet(t *testing.T) {
 		}
 	}))
 	defer ts.Close()
-	client := NewHTTPClient(true, nil, nil)
+	client := NewHTTPClient(NewTransport(true, nil, nil))
 	req, err := http.NewRequest("GET", ts.URL, nil)
 	if err != nil {
 		t.Fatal("NewRequest failed", err)

services/httpservice/httpservice.go

@@ -5,15 +5,25 @@ package httpservice
 
 import (
 	"net"
+	"net/http"
 	"strings"
 
 	"github.com/mattermost/mattermost-server/services/configservice"
 )
 
-// Wraps the functionality for creating a new http.Client to encapsulate that and allow it to be mocked when testing
+// HTTPService wraps the functionality for making http requests to provide some improvements to the default client
+// behaviour.
 type HTTPService interface {
-	MakeClient(trustURLs bool) *Client
-	Close()
+	// MakeClient returns an http client constructed with a RoundTripper as returned by MakeTransport.
+	MakeClient(trustURLs bool) *http.Client
+
+	// MakeTransport returns a RoundTripper that is suitable for making requests to external resources. The default
+	// implementation provides:
+	// - A shorter timeout for dial and TLS handshake (defined as constant "ConnectTimeout")
+	// - A timeout for end-to-end requests (defined as constant "RequestTimeout")
+	// - A Mattermost-specific user agent header
+	// - Additional security for untrusted and insecure connections
+	MakeTransport(trustURLs bool) http.RoundTripper
 }
 
 type HTTPServiceImpl struct {
@@ -24,11 +34,15 @@ func MakeHTTPService(configService configservice.ConfigService) HTTPService {
 	return &HTTPServiceImpl{configService}
 }
 
-func (h *HTTPServiceImpl) MakeClient(trustURLs bool) *Client {
+func (h *HTTPServiceImpl) MakeClient(trustURLs bool) *http.Client {
+	return NewHTTPClient(h.MakeTransport(trustURLs))
+}
+
+func (h *HTTPServiceImpl) MakeTransport(trustURLs bool) http.RoundTripper {
 	insecure := h.configService.Config().ServiceSettings.EnableInsecureOutgoingConnections != nil && *h.configService.Config().ServiceSettings.EnableInsecureOutgoingConnections
 
 	if trustURLs {
-		return NewHTTPClient(insecure, nil, nil)
+		return NewTransport(insecure, nil, nil)
 	}
 
 	allowHost := func(host string) bool {
@@ -58,9 +72,5 @@ func (h *HTTPServiceImpl) MakeClient(trustURLs bool) *Client {
 		return false
 	}
 
-	return NewHTTPClient(insecure, allowHost, allowIP)
-}
-
-func (h *HTTPServiceImpl) Close() {
-	// Does nothing, but allows this to be overridden when mocking the service
+	return NewTransport(insecure, allowHost, allowIP)
 }

services/httpservice/transport.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package httpservice
+
+import (
+	"net/http"
+)
+
+// MattermostTransport is an implementation of http.RoundTripper that ensures each request contains a custom user agent
+// string to indicate that the request is coming from a Mattermost instance.
+type MattermostTransport struct {
+	// Transport is the underlying http.RoundTripper that is actually used to make the request
+	Transport http.RoundTripper
+}
+
+func (t *MattermostTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.Header.Set("User-Agent", defaultUserAgent)
+
+	return t.Transport.RoundTrip(req)
+}