CLI: Allow installing custom binary plugins (#17551)

Make sure all data is sent to API to be able to select correct archive version.

pkg/cmd/grafana-cli/services/api_client.go

@@ -0,0 +1,160 @@
+package services
+
+import (
+	"crypto/md5"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"runtime"
+
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
+	"github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
+	"github.com/grafana/grafana/pkg/util/errutil"
+	"golang.org/x/xerrors"
+)
+
+type GrafanaComClient struct {
+	retryCount int
+}
+
+func (client *GrafanaComClient) GetPlugin(pluginId, repoUrl string) (models.Plugin, error) {
+	logger.Debugf("getting plugin metadata from: %v pluginId: %v \n", repoUrl, pluginId)
+	body, err := sendRequest(HttpClient, repoUrl, "repo", pluginId)
+
+	if err != nil {
+		if err == ErrNotFoundError {
+			return models.Plugin{}, errutil.Wrap("Failed to find requested plugin, check if the plugin_id is correct", err)
+		}
+		return models.Plugin{}, errutil.Wrap("Failed to send request", err)
+	}
+
+	var data models.Plugin
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		logger.Info("Failed to unmarshal plugin repo response error:", err)
+		return models.Plugin{}, err
+	}
+
+	return data, nil
+}
+
+func (client *GrafanaComClient) DownloadFile(pluginName, filePath, url string, checksum string) (content []byte, err error) {
+	// Try handling url like local file path first
+	if _, err := os.Stat(url); err == nil {
+		bytes, err := ioutil.ReadFile(url)
+		if err != nil {
+			return nil, errutil.Wrap("Failed to read file", err)
+		}
+		return bytes, nil
+	}
+
+	client.retryCount = 0
+
+	defer func() {
+		if r := recover(); r != nil {
+			client.retryCount++
+			if client.retryCount < 3 {
+				logger.Info("Failed downloading. Will retry once.")
+				content, err = client.DownloadFile(pluginName, filePath, url, checksum)
+			} else {
+				client.retryCount = 0
+				failure := fmt.Sprintf("%v", r)
+				if failure == "runtime error: makeslice: len out of range" {
+					err = xerrors.New("Corrupt http response from source. Please try again")
+				} else {
+					panic(r)
+				}
+			}
+		}
+	}()
+
+	// TODO: this would be better if it was streamed file by file instead of buffered.
+	// Using no timeout here as some plugins can be bigger and smaller timeout would prevent to download a plugin on
+	// slow network. As this is CLI operation hanging is not a big of an issue as user can just abort.
+	body, err := sendRequest(HttpClientNoTimeout, url)
+
+	if err != nil {
+		return nil, errutil.Wrap("Failed to send request", err)
+	}
+
+	if len(checksum) > 0 && checksum != fmt.Sprintf("%x", md5.Sum(body)) {
+		return nil, xerrors.New("Expected MD5 checksum does not match the downloaded archive. Please contact security@grafana.com.")
+	}
+	return body, nil
+}
+
+func (client *GrafanaComClient) ListAllPlugins(repoUrl string) (models.PluginRepo, error) {
+	body, err := sendRequest(HttpClient, repoUrl, "repo")
+
+	if err != nil {
+		logger.Info("Failed to send request", "error", err)
+		return models.PluginRepo{}, errutil.Wrap("Failed to send request", err)
+	}
+
+	var data models.PluginRepo
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		logger.Info("Failed to unmarshal plugin repo response error:", err)
+		return models.PluginRepo{}, err
+	}
+
+	return data, nil
+}
+
+func sendRequest(client http.Client, repoUrl string, subPaths ...string) ([]byte, error) {
+	u, _ := url.Parse(repoUrl)
+	for _, v := range subPaths {
+		u.Path = path.Join(u.Path, v)
+	}
+
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+
+	req.Header.Set("grafana-version", grafanaVersion)
+	req.Header.Set("grafana-os", runtime.GOOS)
+	req.Header.Set("grafana-arch", runtime.GOARCH)
+	req.Header.Set("User-Agent", "grafana "+grafanaVersion)
+
+	if err != nil {
+		return []byte{}, err
+	}
+
+	res, err := client.Do(req)
+	if err != nil {
+		return []byte{}, err
+	}
+	return handleResponse(res)
+}
+
+func handleResponse(res *http.Response) ([]byte, error) {
+	if res.StatusCode == 404 {
+		return []byte{}, ErrNotFoundError
+	}
+
+	if res.StatusCode/100 != 2 && res.StatusCode/100 != 4 {
+		return []byte{}, fmt.Errorf("Api returned invalid status: %s", res.Status)
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	defer res.Body.Close()
+
+	if res.StatusCode/100 == 4 {
+		if len(body) == 0 {
+			return []byte{}, &BadRequestError{Status: res.Status}
+		}
+		var message string
+		var jsonBody map[string]string
+		err = json.Unmarshal(body, &jsonBody)
+		if err != nil || len(jsonBody["message"]) == 0 {
+			message = string(body)
+		} else {
+			message = jsonBody["message"]
+		}
+		return []byte{}, &BadRequestError{Status: res.Status, Message: message}
+	}
+
+	return body, err
+}

pkg/cmd/grafana-cli/services/api_client_test.go

@@ -0,0 +1,67 @@
+package services
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHandleResponse(t *testing.T) {
+	t.Run("Returns body if status == 200", func(t *testing.T) {
+		body, err := handleResponse(makeResponse(200, "test"))
+		assert.Nil(t, err)
+		assert.Equal(t, "test", string(body))
+	})
+
+	t.Run("Returns ErrorNotFound if status == 404", func(t *testing.T) {
+		_, err := handleResponse(makeResponse(404, ""))
+		assert.Equal(t, ErrNotFoundError, err)
+	})
+
+	t.Run("Returns message from body if status == 400", func(t *testing.T) {
+		_, err := handleResponse(makeResponse(400, "{ \"message\": \"error_message\" }"))
+		assert.NotNil(t, err)
+		assert.Equal(t, "error_message", asBadRequestError(t, err).Message)
+	})
+
+	t.Run("Returns body if status == 400 and no message key", func(t *testing.T) {
+		_, err := handleResponse(makeResponse(400, "{ \"test\": \"test_message\"}"))
+		assert.NotNil(t, err)
+		assert.Equal(t, "{ \"test\": \"test_message\"}", asBadRequestError(t, err).Message)
+	})
+
+	t.Run("Returns Bad request error if status == 400 and no body", func(t *testing.T) {
+		_, err := handleResponse(makeResponse(400, ""))
+		assert.NotNil(t, err)
+		_ = asBadRequestError(t, err)
+	})
+
+	t.Run("Returns error with invalid status if status == 500", func(t *testing.T) {
+		_, err := handleResponse(makeResponse(500, ""))
+		assert.NotNil(t, err)
+		assert.Contains(t, err.Error(), "invalid status")
+	})
+}
+
+func makeResponse(status int, body string) *http.Response {
+	return &http.Response{
+		StatusCode: status,
+		Body:       makeBody(body),
+	}
+}
+
+func makeBody(body string) io.ReadCloser {
+	return ioutil.NopCloser(bytes.NewReader([]byte(body)))
+}
+
+func asBadRequestError(t *testing.T, err error) *BadRequestError {
+	if badRequestError, ok := err.(*BadRequestError); ok {
+		return badRequestError
+	}
+	assert.FailNow(t, "Error was not of type BadRequestError")
+	return nil
+}

pkg/cmd/grafana-cli/services/services.go

@@ -5,12 +5,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"net/http"
-	"net/url"
 	"path"
-	"runtime"
 	"time"
 
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
@@ -18,15 +15,33 @@ import (
 )
 
 var (
-	IoHelper         m.IoUtil = IoUtilImp{}
-	HttpClient       http.Client
-	grafanaVersion   string
-	ErrNotFoundError = errors.New("404 not found error")
+	IoHelper            m.IoUtil = IoUtilImp{}
+	HttpClient          http.Client
+	HttpClientNoTimeout http.Client
+	grafanaVersion      string
+	ErrNotFoundError    = errors.New("404 not found error")
 )
 
+type BadRequestError struct {
+	Message string
+	Status  string
+}
+
+func (e *BadRequestError) Error() string {
+	if len(e.Message) > 0 {
+		return fmt.Sprintf("%s: %s", e.Status, e.Message)
+	}
+	return e.Status
+}
+
 func Init(version string, skipTLSVerify bool) {
 	grafanaVersion = version
 
+	HttpClient = makeHttpClient(skipTLSVerify, 10*time.Second)
+	HttpClientNoTimeout = makeHttpClient(skipTLSVerify, 0)
+}
+
+func makeHttpClient(skipTLSVerify bool, timeout time.Duration) http.Client {
 	tr := &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
 		DialContext: (&net.Dialer{
@@ -42,30 +57,12 @@ func Init(version string, skipTLSVerify bool) {
 		},
 	}
 
-	HttpClient = http.Client{
-		Timeout:   10 * time.Second,
+	return http.Client{
+		Timeout:   timeout,
 		Transport: tr,
 	}
 }
 
-func ListAllPlugins(repoUrl string) (m.PluginRepo, error) {
-	body, err := sendRequest(repoUrl, "repo")
-
-	if err != nil {
-		logger.Info("Failed to send request", "error", err)
-		return m.PluginRepo{}, fmt.Errorf("Failed to send request. error: %v", err)
-	}
-
-	var data m.PluginRepo
-	err = json.Unmarshal(body, &data)
-	if err != nil {
-		logger.Info("Failed to unmarshal plugin repo response error:", err)
-		return m.PluginRepo{}, err
-	}
-
-	return data, nil
-}
-
 func ReadPlugin(pluginDir, pluginName string) (m.InstalledPlugin, error) {
 	distPluginDataPath := path.Join(pluginDir, pluginName, "dist", "plugin.json")
 
@@ -120,60 +117,3 @@ func RemoveInstalledPlugin(pluginPath, pluginName string) error {
 
 	return IoHelper.RemoveAll(pluginDir)
 }
-
-func GetPlugin(pluginId, repoUrl string) (m.Plugin, error) {
-	logger.Debugf("getting plugin metadata from: %v pluginId: %v \n", repoUrl, pluginId)
-	body, err := sendRequest(repoUrl, "repo", pluginId)
-
-	if err != nil {
-		logger.Info("Failed to send request: ", err)
-		if err == ErrNotFoundError {
-			return m.Plugin{}, fmt.Errorf("Failed to find requested plugin, check if the plugin_id is correct. error: %v", err)
-		}
-		return m.Plugin{}, fmt.Errorf("Failed to send request. error: %v", err)
-	}
-
-	var data m.Plugin
-	err = json.Unmarshal(body, &data)
-	if err != nil {
-		logger.Info("Failed to unmarshal plugin repo response error:", err)
-		return m.Plugin{}, err
-	}
-
-	return data, nil
-}
-
-func sendRequest(repoUrl string, subPaths ...string) ([]byte, error) {
-	u, _ := url.Parse(repoUrl)
-	for _, v := range subPaths {
-		u.Path = path.Join(u.Path, v)
-	}
-
-	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
-
-	req.Header.Set("grafana-version", grafanaVersion)
-	req.Header.Set("grafana-os", runtime.GOOS)
-	req.Header.Set("grafana-arch", runtime.GOARCH)
-	req.Header.Set("User-Agent", "grafana "+grafanaVersion)
-
-	if err != nil {
-		return []byte{}, err
-	}
-
-	res, err := HttpClient.Do(req)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	if res.StatusCode == 404 {
-		return []byte{}, ErrNotFoundError
-	}
-	if res.StatusCode/100 != 2 {
-		return []byte{}, fmt.Errorf("Api returned invalid status: %s", res.Status)
-	}
-
-	body, err := ioutil.ReadAll(res.Body)
-	defer res.Body.Close()
-
-	return body, err
-}