diff --git a/go.mod b/go.mod index b7fe5194552..46d37191fb5 100644 --- a/go.mod +++ b/go.mod @@ -144,6 +144,7 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/alecthomas/units v0.0.0-20210912230133-d1bdfacee922 // indirect + github.com/andybalholm/brotli v1.0.3 github.com/apache/arrow/go/arrow v0.0.0-20210223225224-5bea62493d91 // indirect github.com/armon/go-metrics v0.3.8 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect @@ -189,7 +190,7 @@ require ( github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f // indirect - github.com/grafana/grafana-google-sdk-go v0.0.0-20211019132340-3ff525a010d5 + github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.0 // indirect diff --git a/go.sum b/go.sum index 11f569743cf..0bc56a152ac 100644 --- a/go.sum +++ b/go.sum @@ -265,6 +265,8 @@ github.com/alicebob/miniredis/v2 v2.14.3/go.mod h1:gquAfGbzn92jvtrSC69+6zZnwSODV github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM= +github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antonmedv/expr v1.8.9/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= @@ -1219,8 +1221,8 @@ github.com/grafana/go-mssqldb v0.0.0-20210326084033-d0ce3c521036 h1:GplhUk6Xes5J github.com/grafana/go-mssqldb v0.0.0-20210326084033-d0ce3c521036/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/grafana/grafana-aws-sdk v0.7.0 h1:D+Lhxi3P/7vpyDHUK/fdX9bL2mRz8hLG04ucNf1E02o= github.com/grafana/grafana-aws-sdk v0.7.0/go.mod h1:+pPo5U+pX0zWimR7YBc7ASeSQfbRkcTyQYqMiAj7G5U= -github.com/grafana/grafana-google-sdk-go v0.0.0-20211019132340-3ff525a010d5 h1:o7w/t0nLNfkERMdj09U0h3Fl63z8ws1CxwiImeUKLIk= -github.com/grafana/grafana-google-sdk-go v0.0.0-20211019132340-3ff525a010d5/go.mod h1:Vo2TKWfDVmNTELBUM+3lkrZvFtBws0qSZdXhQxRdJrE= +github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58 h1:2ud7NNM7LrGPO4x0NFR8qLq68CqI4SmB7I2yRN2w9oE= +github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58/go.mod h1:Vo2TKWfDVmNTELBUM+3lkrZvFtBws0qSZdXhQxRdJrE= github.com/grafana/grafana-plugin-sdk-go v0.79.0/go.mod h1:NvxLzGkVhnoBKwzkst6CFfpMFKwAdIUZ1q8ssuLeF60= github.com/grafana/grafana-plugin-sdk-go v0.114.0 h1:9I55IXw7mOT71tZ/pdqCaWGz8vxfz31CXjaDtBV9ZBo= github.com/grafana/grafana-plugin-sdk-go v0.114.0/go.mod h1:D7x3ah+1d4phNXpbnOaxa/osSaZlwh9/ZUnGGzegRbk= diff --git a/pkg/tsdb/cloudmonitoring/cloudmonitoring.go b/pkg/tsdb/cloudmonitoring/cloudmonitoring.go index 9e09755535f..bd4f0d12571 100644 --- a/pkg/tsdb/cloudmonitoring/cloudmonitoring.go +++ b/pkg/tsdb/cloudmonitoring/cloudmonitoring.go @@ -3,7 +3,6 @@ package cloudmonitoring import ( "context" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -16,8 +15,7 @@ import ( "strings" "time" - "golang.org/x/oauth2/google" - + "github.com/grafana/grafana-google-sdk-go/pkg/utils" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/datasource" "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" @@ -87,6 +85,7 @@ func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider, re factory := coreplugin.New(backend.ServeOpts{ QueryDataHandler: s, CallResourceHandler: httpadapter.New(mux), + CheckHealthHandler: s, }) if err := registrar.LoadAndRegister(pluginID, factory); err != nil { @@ -95,6 +94,45 @@ func ProvideService(cfg *setting.Cfg, httpClientProvider httpclient.Provider, re return s } +func (s *Service) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { + dsInfo, err := s.getDSInfo(req.PluginContext) + if err != nil { + return nil, err + } + + defaultProject, err := s.getDefaultProject(ctx, *dsInfo) + if err != nil { + return nil, err + } + + url := fmt.Sprintf("%v/v3/projects/%v/metricDescriptors", dsInfo.services[cloudMonitor].url, defaultProject) + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + res, err := dsInfo.services[cloudMonitor].client.Do(request) + if err != nil { + return nil, err + } + defer func() { + if err := res.Body.Close(); err != nil { + slog.Warn("Failed to close response body", "err", err) + } + }() + + status := backend.HealthStatusOk + message := "Successfully queried the Google Cloud Monitoring API." + if res.StatusCode != 200 { + status = backend.HealthStatusError + message = res.Status + } + return &backend.CheckHealthResult{ + Status: status, + Message: message, + }, nil +} + type Service struct { httpClientProvider httpclient.Provider cfg *setting.Cfg @@ -206,8 +244,6 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) switch model.Type { case "annotationQuery": resp, err = s.executeAnnotationQuery(ctx, req, *dsInfo) - case "getGCEDefaultProject": - resp, err = s.getGCEDefaultProject(ctx, req, *dsInfo) case "timeSeriesQuery": fallthrough default: @@ -217,26 +253,6 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) return resp, err } -func (s *Service) getGCEDefaultProject(ctx context.Context, req *backend.QueryDataRequest, dsInfo datasourceInfo) (*backend.QueryDataResponse, error) { - gceDefaultProject, err := s.getDefaultProject(ctx, dsInfo) - if err != nil { - return backend.NewQueryDataResponse(), fmt.Errorf( - "failed to retrieve default project from GCE metadata server, error: %w", err) - } - - return &backend.QueryDataResponse{ - Responses: backend.Responses{ - req.Queries[0].RefID: { - Frames: data.Frames{data.NewFrame("").SetMeta(&data.FrameMeta{ - Custom: map[string]interface{}{ - "defaultProject": gceDefaultProject, - }, - })}, - }, - }, - }, nil -} - func (s *Service) executeTimeSeriesQuery(ctx context.Context, req *backend.QueryDataRequest, dsInfo datasourceInfo) ( *backend.QueryDataResponse, error) { resp := backend.NewQueryDataResponse() @@ -606,19 +622,7 @@ func (s *Service) createRequest(ctx context.Context, dsInfo *datasourceInfo, pro func (s *Service) getDefaultProject(ctx context.Context, dsInfo datasourceInfo) (string, error) { if dsInfo.authenticationType == gceAuthentication { - defaultCredentials, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/monitoring.read") - if err != nil { - return "", fmt.Errorf("failed to retrieve default project from GCE metadata server: %w", err) - } - token, err := defaultCredentials.TokenSource.Token() - if err != nil { - return "", fmt.Errorf("failed to retrieve GCP credential token: %w", err) - } - if !token.Valid() { - return "", errors.New("failed to validate GCP credentials") - } - - return defaultCredentials.ProjectID, nil + return utils.GCEDefaultProject(ctx) } return dsInfo.defaultProject, nil } diff --git a/pkg/tsdb/cloudmonitoring/resource_handler.go b/pkg/tsdb/cloudmonitoring/resource_handler.go index 5fb302613dc..fd4ab79b0fe 100644 --- a/pkg/tsdb/cloudmonitoring/resource_handler.go +++ b/pkg/tsdb/cloudmonitoring/resource_handler.go @@ -1,31 +1,244 @@ package cloudmonitoring import ( + "bytes" + "compress/flate" + "compress/gzip" + "encoding/json" "fmt" + "io" "io/ioutil" "net/http" "net/url" + "regexp" "strings" + "github.com/andybalholm/brotli" + "github.com/grafana/grafana-google-sdk-go/pkg/utils" "github.com/grafana/grafana-plugin-sdk-go/backend/resource/httpadapter" ) +// nameExp matches the part after the last '/' symbol +var nameExp = regexp.MustCompile(`([^\/]*)\/*$`) + +const resourceManagerPath = "/v1/projects" + +type processResponse func(body []byte) ([]byte, error) + func (s *Service) registerRoutes(mux *http.ServeMux) { - mux.HandleFunc("/cloudmonitoring/", s.resourceHandler(cloudMonitor)) - mux.HandleFunc("/cloudresourcemanager/", s.resourceHandler(resourceManager)) + mux.HandleFunc("/gceDefaultProject", getGCEDefaultProject) + + mux.HandleFunc("/metricDescriptors/", s.resourceHandler(cloudMonitor, processMetricDescriptors)) + mux.HandleFunc("/services/", s.resourceHandler(cloudMonitor, processServices)) + mux.HandleFunc("/slo-services/", s.resourceHandler(cloudMonitor, processSLOs)) + mux.HandleFunc("/projects", s.resourceHandler(resourceManager, processProjects)) } -func (s *Service) resourceHandler(subDataSource string) func(rw http.ResponseWriter, req *http.Request) { +func getGCEDefaultProject(rw http.ResponseWriter, req *http.Request) { + project, err := utils.GCEDefaultProject(req.Context()) + if err != nil { + writeResponse(rw, http.StatusBadRequest, fmt.Sprintf("unexpected error %v", err)) + return + } + writeResponse(rw, http.StatusOK, project) +} + +func (s *Service) resourceHandler(subDataSource string, responseFn processResponse) func(rw http.ResponseWriter, req *http.Request) { return func(rw http.ResponseWriter, req *http.Request) { client, code, err := s.setRequestVariables(req, subDataSource) if err != nil { writeResponse(rw, code, fmt.Sprintf("unexpected error %v", err)) return } - doRequest(rw, req, client) + s.doRequest(rw, req, client, responseFn) } } +func (s *Service) doRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client, responseFn processResponse) http.ResponseWriter { + res, err := cli.Do(req) + if err != nil { + writeResponse(rw, http.StatusBadRequest, fmt.Sprintf("unexpected error %v", err)) + return rw + } + defer func() { + if err := res.Body.Close(); err != nil { + slog.Warn("Failed to close response body", "err", err) + } + }() + + if responseFn == nil { + writeResponse(rw, http.StatusInternalServerError, "responseFn should not be nil") + return rw + } + + body, code, err := processData(res, responseFn) + if err != nil { + writeResponse(rw, code, fmt.Sprintf("unexpected error %v", err)) + return rw + } + writeResponseBytes(rw, res.StatusCode, body) + + for k, v := range res.Header { + rw.Header().Set(k, v[0]) + for _, v := range v[1:] { + rw.Header().Add(k, v) + } + } + return rw +} + +func processMetricDescriptors(body []byte) ([]byte, error) { + resp := metricDescriptorResponse{} + err := json.Unmarshal(body, &resp) + if err != nil { + return nil, err + } + + for i := range resp.Descriptors { + resp.Descriptors[i].Service = strings.SplitN(resp.Descriptors[i].Type, "/", 2)[0] + resp.Descriptors[i].ServiceShortName = strings.SplitN(resp.Descriptors[i].Service, ".", 2)[0] + if resp.Descriptors[i].DisplayName == "" { + resp.Descriptors[i].DisplayName = resp.Descriptors[i].Type + } + } + return json.Marshal(resp.Descriptors) +} + +func processServices(body []byte) ([]byte, error) { + resp := serviceResponse{} + err := json.Unmarshal(body, &resp) + if err != nil { + return nil, err + } + + values := []selectableValue{} + for _, service := range resp.Services { + name := nameExp.FindString(service.Name) + if name == "" { + return nil, fmt.Errorf("unexpected service name: %v", service.Name) + } + label := service.DisplayName + if label == "" { + label = name + } + values = append(values, selectableValue{ + Value: name, + Label: label, + }) + } + return json.Marshal(values) +} + +func processSLOs(body []byte) ([]byte, error) { + resp := sloResponse{} + err := json.Unmarshal(body, &resp) + if err != nil { + return nil, err + } + + values := []selectableValue{} + for _, slo := range resp.SLOs { + name := nameExp.FindString(slo.Name) + if name == "" { + return nil, fmt.Errorf("unexpected service name: %v", slo.Name) + } + values = append(values, selectableValue{ + Value: name, + Label: slo.DisplayName, + Goal: slo.Goal, + }) + } + return json.Marshal(values) +} + +func processProjects(body []byte) ([]byte, error) { + resp := projectResponse{} + err := json.Unmarshal(body, &resp) + if err != nil { + return nil, err + } + + values := []selectableValue{} + for _, project := range resp.Projects { + values = append(values, selectableValue{ + Value: project.ProjectID, + Label: project.Name, + }) + } + return json.Marshal(values) +} + +func processData(res *http.Response, responseFn processResponse) ([]byte, int, error) { + encoding := res.Header.Get("Content-Encoding") + + var reader io.Reader + var err error + switch encoding { + case "gzip": + reader, err = gzip.NewReader(res.Body) + if err != nil { + return nil, http.StatusBadRequest, fmt.Errorf("unexpected error %v", err) + } + defer func() { + if err := reader.(io.ReadCloser).Close(); err != nil { + slog.Warn("Failed to close reader body", "err", err) + } + }() + case "deflate": + reader = flate.NewReader(res.Body) + defer func() { + if err := reader.(io.ReadCloser).Close(); err != nil { + slog.Warn("Failed to close reader body", "err", err) + } + }() + case "br": + reader = brotli.NewReader(res.Body) + case "": + reader = res.Body + default: + return nil, http.StatusInternalServerError, fmt.Errorf("unexpected encoding type %v", err) + } + + body, err := ioutil.ReadAll(reader) + if err != nil { + return nil, http.StatusBadRequest, fmt.Errorf("unexpected error %v", err) + } + + body, err = responseFn(body) + if err != nil { + return nil, http.StatusInternalServerError, fmt.Errorf("data processing error %v", err) + } + + buf := new(bytes.Buffer) + var writer io.Writer = buf + switch encoding { + case "gzip": + writer = gzip.NewWriter(writer) + case "deflate": + writer, err = flate.NewWriter(writer, -1) + if err != nil { + return nil, http.StatusInternalServerError, fmt.Errorf("unexpected error %v", err) + } + case "br": + writer = brotli.NewWriter(writer) + case "": + default: + return nil, http.StatusInternalServerError, fmt.Errorf("unexpected encoding type %v", encoding) + } + + _, err = writer.Write(body) + if writeCloser, ok := writer.(io.WriteCloser); ok { + if err := writeCloser.Close(); err != nil { + slog.Warn("Failed to close writer body", "err", err) + } + } + if err != nil { + return nil, http.StatusInternalServerError, fmt.Errorf("unable to encode response %v", err) + } + + return buf.Bytes(), 0, nil +} + func (s *Service) setRequestVariables(req *http.Request, subDataSource string) (*http.Client, int, error) { slog.Debug("Received resource call", "url", req.URL.String(), "method", req.Method) @@ -50,48 +263,10 @@ func (s *Service) setRequestVariables(req *http.Request, subDataSource string) ( return dsInfo.services[subDataSource].client, 0, nil } -func doRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client) http.ResponseWriter { - res, err := cli.Do(req) - if err != nil { - rw.WriteHeader(http.StatusBadRequest) - _, err = rw.Write([]byte(fmt.Sprintf("unexpected error %v", err))) - if err != nil { - slog.Error("Unable to write HTTP response", "error", err) - } - return nil - } - defer func() { - if err := res.Body.Close(); err != nil { - slog.Warn("Failed to close response body", "err", err) - } - }() - - body, err := ioutil.ReadAll(res.Body) - if err != nil { - rw.WriteHeader(http.StatusInternalServerError) - _, err = rw.Write([]byte(fmt.Sprintf("unexpected error %v", err))) - if err != nil { - slog.Error("Unable to write HTTP response", "error", err) - } - return nil - } - rw.WriteHeader(res.StatusCode) - _, err = rw.Write(body) - if err != nil { - slog.Error("Unable to write HTTP response", "error", err) - } - - for k, v := range res.Header { - rw.Header().Set(k, v[0]) - for _, v := range v[1:] { - rw.Header().Add(k, v) - } - } - // Returning the response write for testing purposes - return rw -} - func getTarget(original string) (target string, err error) { + if original == "/projects" { + return resourceManagerPath, nil + } splittedPath := strings.SplitN(original, "/", 3) if len(splittedPath) < 3 { err = fmt.Errorf("the request should contain the service on its path") @@ -101,14 +276,18 @@ func getTarget(original string) (target string, err error) { return } -func writeResponse(rw http.ResponseWriter, code int, msg string) { +func writeResponseBytes(rw http.ResponseWriter, code int, msg []byte) { rw.WriteHeader(code) - _, err := rw.Write([]byte(msg)) + _, err := rw.Write(msg) if err != nil { slog.Error("Unable to write HTTP response", "error", err) } } +func writeResponse(rw http.ResponseWriter, code int, msg string) { + writeResponseBytes(rw, code, []byte(msg)) +} + func (s *Service) getDataSourceFromHTTPReq(req *http.Request) (*datasourceInfo, error) { ctx := req.Context() pluginContext := httpadapter.PluginConfigFromContext(ctx) diff --git a/pkg/tsdb/cloudmonitoring/resource_handler_test.go b/pkg/tsdb/cloudmonitoring/resource_handler_test.go index 755f7e1a18c..1a54cc123e7 100644 --- a/pkg/tsdb/cloudmonitoring/resource_handler_test.go +++ b/pkg/tsdb/cloudmonitoring/resource_handler_test.go @@ -1,6 +1,7 @@ package cloudmonitoring import ( + "encoding/json" "io/ioutil" "net/http" "net/http/httptest" @@ -42,6 +43,10 @@ func Test_parseResourcePath(t *testing.T) { } } +func fakeResponseFn(input []byte) ([]byte, error) { + return input, nil +} + func Test_doRequest(t *testing.T) { // test that it forwards the header and body srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -55,8 +60,10 @@ func Test_doRequest(t *testing.T) { if err != nil { t.Error(err) } + s := Service{} + rw := httptest.NewRecorder() - res := doRequest(rw, req, srv.Client()) + res := s.doRequest(rw, req, srv.Client(), fakeResponseFn) if res.Header().Get("foo") != "bar" { t.Errorf("Unexpected headers: %v", res.Header()) } @@ -112,3 +119,149 @@ func Test_setRequestVariables(t *testing.T) { t.Errorf("Unexpected result URL. Got %s, expecting %s", req.URL.String(), expectedURL) } } + +func Test_processData_functions(t *testing.T) { + // metricDescriptors + metricDescriptorResp := metricDescriptorResponse{ + Descriptors: []metricDescriptor{ + { + ValueType: "INT64", + MetricKind: "DELTA", + Type: "actions.googleapis.com/smarthome_action/local_event_count", + Unit: "1", + Service: "foo", + ServiceShortName: "bar", + DisplayName: "Local event count", + Description: "baz", + }, + }, + } + marshaledMDResponse, _ := json.Marshal(metricDescriptorResp) + metricDescriptorResult := []metricDescriptor{ + { + ValueType: "INT64", + MetricKind: "DELTA", + Type: "actions.googleapis.com/smarthome_action/local_event_count", + Unit: "1", + Service: "actions.googleapis.com", + ServiceShortName: "actions", + DisplayName: "Local event count", + Description: "baz", + }, + } + marshaledMDResult, _ := json.Marshal(metricDescriptorResult) + + // services + serviceResp := serviceResponse{ + Services: []serviceDescription{ + { + Name: "blah/foo", + DisplayName: "bar", + }, + { + Name: "abc", + DisplayName: "", + }, + }, + } + marshaledServiceResponse, _ := json.Marshal(serviceResp) + serviceResult := []selectableValue{ + { + Value: "foo", + Label: "bar", + }, + { + Value: "abc", + Label: "abc", + }, + } + marshaledServiceResult, _ := json.Marshal(serviceResult) + + // slos + sloResp := sloResponse{ + SLOs: []sloDescription{ + { + Name: "blah/foo", + DisplayName: "bar", + Goal: 0.1, + }, + { + Name: "abc", + DisplayName: "xyz", + Goal: 0.2, + }, + }, + } + marshaledSLOResponse, _ := json.Marshal(sloResp) + sloResult := []selectableValue{ + { + Value: "foo", + Label: "bar", + Goal: 0.1, + }, + { + Value: "abc", + Label: "xyz", + Goal: 0.2, + }, + } + marshaledSLOResult, _ := json.Marshal(sloResult) + + // cloudresourcemanager + cloudResourceResp := projectResponse{ + Projects: []projectDescription{ + { + ProjectID: "foo", + Name: "bar", + }, + { + ProjectID: "abc", + Name: "abc", + }, + }, + } + marshaledCRResponse, _ := json.Marshal(cloudResourceResp) + + tests := []struct { + name string + responseFn processResponse + input []byte + result []byte + }{ + { + "metricDescriptor", + processMetricDescriptors, + marshaledMDResponse, + marshaledMDResult, + }, + { + "services", + processServices, + marshaledServiceResponse, + marshaledServiceResult, + }, + { + "slos", + processSLOs, + marshaledSLOResponse, + marshaledSLOResult, + }, + { + "cloudresourcemanager", + processProjects, + marshaledCRResponse, + marshaledServiceResult, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + res, err := test.responseFn(test.input) + if err != nil { + t.Errorf("Unexpected error %v", err) + } + if string(test.result) != string(res) { + t.Errorf("Unexpected result. Got %s, expecting %s", res, test.result) + } + }) + } +} diff --git a/pkg/tsdb/cloudmonitoring/types.go b/pkg/tsdb/cloudmonitoring/types.go index ec6365eef65..7b5fb99a369 100644 --- a/pkg/tsdb/cloudmonitoring/types.go +++ b/pkg/tsdb/cloudmonitoring/types.go @@ -189,3 +189,50 @@ type timeSeries struct { } `json:"value"` } `json:"points"` } + +type metricDescriptorResponse struct { + Descriptors []metricDescriptor `json:"metricDescriptors"` +} +type metricDescriptor struct { + ValueType string `json:"valueType"` + MetricKind string `json:"metricKind"` + Type string `json:"type"` + Unit string `json:"unit"` + Service string `json:"service"` + ServiceShortName string `json:"serviceShortName"` + DisplayName string `json:"displayName"` + Description string `json:"description"` +} + +type projectResponse struct { + Projects []projectDescription `json:"projects"` +} + +type projectDescription struct { + ProjectID string `json:"projectId"` + Name string `json:"name"` +} + +type serviceResponse struct { + Services []serviceDescription `json:"services"` +} +type serviceDescription struct { + Name string `json:"name"` + DisplayName string `json:"displayName"` +} + +type sloResponse struct { + SLOs []sloDescription `json:"serviceLevelObjectives"` +} + +type sloDescription struct { + Name string `json:"name"` + DisplayName string `json:"displayName"` + Goal float64 `json:"goal"` +} + +type selectableValue struct { + Value string `json:"value"` + Label string `json:"label"` + Goal float64 `json:"goal,omitempty"` +} diff --git a/public/app/plugins/datasource/cloud-monitoring/api.test.ts b/public/app/plugins/datasource/cloud-monitoring/api.test.ts deleted file mode 100644 index 992413d09c0..00000000000 --- a/public/app/plugins/datasource/cloud-monitoring/api.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { of } from 'rxjs'; - -import Api from './api'; -import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__ -import { createFetchResponse } from 'test/helpers/createFetchResponse'; - -jest.mock('@grafana/runtime', () => ({ - ...((jest.requireActual('@grafana/runtime') as unknown) as object), - getBackendSrv: () => backendSrv, -})); - -const response = [ - { label: 'test1', value: 'test1' }, - { label: 'test2', value: 'test2' }, -]; - -type Args = { path?: string; options?: any; response?: any; cache?: any }; - -async function getTestContext({ path = 'some-resource', options = {}, response = {}, cache }: Args = {}) { - jest.clearAllMocks(); - - const fetchMock = jest.spyOn(backendSrv, 'fetch'); - - fetchMock.mockImplementation((options: any) => { - const data = { [options.url.match(/([^\/]*)\/*$/)![1].split('?')[0]]: response }; - return of(createFetchResponse(data)); - }); - - const api = new Api('/cloudmonitoring/'); - - if (cache) { - api.cache[path] = cache; - } - - const res = await api.get(path, options); - - return { res, api, fetchMock }; -} - -describe('api', () => { - describe('when resource was cached', () => { - test.each(['some-resource', 'some-resource?some=param', 'test/some-resource?param'])( - 'should return cached value and not load from source', - async (path) => { - const { res, api, fetchMock } = await getTestContext({ path, cache: response }); - - expect(res).toEqual(response); - expect(api.cache[path]).toEqual(response); - expect(fetchMock).not.toHaveBeenCalled(); - } - ); - }); - - describe('when resource was not cached', () => { - test.each(['some-resource', 'some-resource?some=param', 'test/some-resource?param'])( - 'should return from source and not from cache', - async (path) => { - const { res, api, fetchMock } = await getTestContext({ path, response }); - - expect(res).toEqual(response); - expect(api.cache[path]).toEqual(response); - expect(fetchMock).toHaveBeenCalled(); - } - ); - }); - - describe('when cache should be bypassed', () => { - test.each(['some-resource', 'some-resource?some=param', 'test/some-resource?param'])( - 'should return from source and not from cache', - async (path) => { - const options = { useCache: false }; - const { res, fetchMock } = await getTestContext({ path, response, cache: response, options }); - - expect(res).toEqual(response); - expect(fetchMock).toHaveBeenCalled(); - } - ); - }); -}); diff --git a/public/app/plugins/datasource/cloud-monitoring/api.ts b/public/app/plugins/datasource/cloud-monitoring/api.ts deleted file mode 100644 index 4bb08980805..00000000000 --- a/public/app/plugins/datasource/cloud-monitoring/api.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { lastValueFrom, Observable, of } from 'rxjs'; -import { catchError, map } from 'rxjs/operators'; -import { SelectableValue } from '@grafana/data'; -import { FetchResponse, getBackendSrv } from '@grafana/runtime'; - -import appEvents from 'app/core/app_events'; -import { CoreEvents } from 'app/types'; -import { formatCloudMonitoringError } from './functions'; -import { MetricDescriptor } from './types'; - -export interface PostResponse { - results: Record; -} - -interface Options { - responseMap?: (res: any) => SelectableValue | MetricDescriptor; - baseUrl?: string; - useCache?: boolean; -} - -export default class Api { - cache: { [key: string]: Array> }; - defaultOptions: Options; - - constructor(private baseUrl: string) { - this.cache = {}; - this.defaultOptions = { - useCache: true, - responseMap: (res: any) => res, - baseUrl: this.baseUrl, - }; - } - - get(path: string, options?: Options): Promise> | MetricDescriptor[]> { - const { useCache, responseMap, baseUrl } = { ...this.defaultOptions, ...options }; - - if (useCache && this.cache[path]) { - return Promise.resolve(this.cache[path]); - } - - return lastValueFrom( - getBackendSrv() - .fetch>({ - url: baseUrl + path, - method: 'GET', - }) - .pipe( - map((response) => { - const responsePropName = path.match(/([^\/]*)\/*$/)![1].split('?')[0]; - let res = []; - if (response && response.data && response.data[responsePropName]) { - res = response.data[responsePropName].map(responseMap); - } - - if (useCache) { - this.cache[path] = res; - } - - return res; - }), - catchError((error) => { - appEvents.emit(CoreEvents.dsRequestError, { - error: { data: { error: formatCloudMonitoringError(error) } }, - }); - return of([]); - }) - ) - ); - } - - post(data: Record): Observable> { - return getBackendSrv().fetch({ - url: '/api/ds/query', - method: 'POST', - data, - }); - } - - test(projectName: string) { - return lastValueFrom( - getBackendSrv().fetch({ - url: `${this.baseUrl}${projectName}/metricDescriptors`, - method: 'GET', - }) - ); - } -} diff --git a/public/app/plugins/datasource/cloud-monitoring/components/Project.tsx b/public/app/plugins/datasource/cloud-monitoring/components/Project.tsx index a69d08af2ed..ab2cd2664d8 100644 --- a/public/app/plugins/datasource/cloud-monitoring/components/Project.tsx +++ b/public/app/plugins/datasource/cloud-monitoring/components/Project.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { SelectableValue } from '@grafana/data'; import { Select } from '@grafana/ui'; import CloudMonitoringDatasource from '../datasource'; @@ -15,16 +15,20 @@ export interface Props { export function Project({ projectName, datasource, onChange, templateVariableOptions }: Props) { const [projects, setProjects] = useState>>([]); useEffect(() => { - datasource.getProjects().then((projects) => - setProjects([ - { - label: 'Template Variables', - options: templateVariableOptions, - }, - ...projects, - ]) - ); - }, [datasource, templateVariableOptions]); + datasource.getProjects().then((projects) => setProjects(projects)); + }, [datasource]); + + const projectsWithTemplateVariables = useMemo( + () => [ + projects, + { + label: 'Template Variables', + options: templateVariableOptions, + }, + ...projects, + ], + [projects, templateVariableOptions] + ); return ( @@ -34,7 +38,7 @@ export function Project({ projectName, datasource, onChange, templateVariableOpt allowCustomValue formatCreateLabel={(v) => `Use project: ${v}`} onChange={({ value }) => onChange(value!)} - options={projects} + options={projectsWithTemplateVariables} value={{ value: projectName, label: projectName }} placeholder="Select Project" /> diff --git a/public/app/plugins/datasource/cloud-monitoring/datasource.ts b/public/app/plugins/datasource/cloud-monitoring/datasource.ts index 028650688da..010fc3c2234 100644 --- a/public/app/plugins/datasource/cloud-monitoring/datasource.ts +++ b/public/app/plugins/datasource/cloud-monitoring/datasource.ts @@ -1,6 +1,6 @@ import { chunk, flatten, isString } from 'lodash'; -import { from, lastValueFrom, Observable, of, throwError } from 'rxjs'; -import { catchError, map, mergeMap } from 'rxjs/operators'; +import { from, lastValueFrom, Observable, of } from 'rxjs'; +import { map, mergeMap } from 'rxjs/operators'; import { DataQueryRequest, DataQueryResponse, @@ -8,19 +8,25 @@ import { ScopedVars, SelectableValue, } from '@grafana/data'; -import { DataSourceWithBackend, toDataQueryResponse } from '@grafana/runtime'; +import { DataSourceWithBackend, getBackendSrv, toDataQueryResponse } from '@grafana/runtime'; import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv'; import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv'; -import { CloudMonitoringOptions, CloudMonitoringQuery, EditorMode, Filter, MetricDescriptor, QueryType } from './types'; -import API from './api'; +import { + CloudMonitoringOptions, + CloudMonitoringQuery, + EditorMode, + Filter, + MetricDescriptor, + QueryType, + PostResponse, +} from './types'; import { CloudMonitoringVariableSupport } from './variables'; export default class CloudMonitoringDatasource extends DataSourceWithBackend< CloudMonitoringQuery, CloudMonitoringOptions > { - api: API; authenticationType: string; intervalMs: number; @@ -31,7 +37,6 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend< ) { super(instanceSettings); this.authenticationType = instanceSettings.jsonData.authenticationType || 'jwt'; - this.api = new API(`/api/datasources/${this.id}/resources/cloudmonitoring/v3/projects/`); this.variables = new CloudMonitoringVariableSupport(this); this.intervalMs = 0; } @@ -72,11 +77,15 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend< ]; return lastValueFrom( - this.api - .post({ - from: options.range.from.valueOf().toString(), - to: options.range.to.valueOf().toString(), - queries, + getBackendSrv() + .fetch({ + url: '/api/ds/query', + method: 'POST', + data: { + from: options.range.from.valueOf().toString(), + to: options.range.to.valueOf().toString(), + queries, + }, }) .pipe( map(({ data }) => { @@ -156,10 +165,14 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend< return lastValueFrom( from(this.ensureGCEDefaultProject()).pipe( mergeMap(() => { - return this.api.post({ - from: options.range.from.valueOf().toString(), - to: options.range.to.valueOf().toString(), - queries, + return getBackendSrv().fetch({ + url: '/api/ds/query', + method: 'POST', + data: { + from: options.range.from.valueOf().toString(), + to: options.range.to.valueOf().toString(), + queries, + }, }); }), map(({ data }) => { @@ -172,62 +185,8 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend< ); } - async testDatasource() { - let status, message; - const defaultErrorMessage = 'Cannot connect to Google Cloud Monitoring API'; - try { - await this.ensureGCEDefaultProject(); - const response = await this.api.test(this.getDefaultProject()); - if (response.status === 200) { - status = 'success'; - message = 'Successfully queried the Google Cloud Monitoring API.'; - } else { - status = 'error'; - message = response.statusText ? response.statusText : defaultErrorMessage; - } - } catch (error) { - status = 'error'; - if (isString(error)) { - message = error; - } else { - message = 'Google Cloud Monitoring: '; - message += error.statusText ? error.statusText : defaultErrorMessage; - if (error.data && error.data.error && error.data.error.code) { - message += ': ' + error.data.error.code + '. ' + error.data.error.message; - } - } - } finally { - return { - status, - message, - }; - } - } - async getGCEDefaultProject() { - return lastValueFrom( - this.api - .post({ - queries: [ - { - refId: 'getGCEDefaultProject', - type: 'getGCEDefaultProject', - datasource: this.getRef(), - }, - ], - }) - .pipe( - map(({ data }) => { - const dataQueryResponse = toDataQueryResponse({ - data: data, - }); - return dataQueryResponse?.data[0]?.meta?.custom?.defaultProject ?? ''; - }), - catchError((err) => { - return throwError(err.data.error); - }) - ) - ); + return this.getResource(`gceDefaultProject`); } getDefaultProject(): string { @@ -251,26 +210,13 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend< return []; } - return this.api.get(`${this.templateSrv.replace(projectName)}/metricDescriptors`, { - responseMap: (m: MetricDescriptor) => { - const [service] = m.type.split('/'); - const [serviceShortName] = service.split('.'); - m.service = service; - m.serviceShortName = serviceShortName; - m.displayName = m.displayName || m.type; - - return m; - }, - }) as Promise; + return this.getResource( + `metricDescriptors/v3/projects/${this.templateSrv.replace(projectName)}/metricDescriptors` + ) as Promise; } async getSLOServices(projectName: string): Promise>> { - return this.api.get(`${this.templateSrv.replace(projectName)}/services?pageSize=1000`, { - responseMap: ({ name, displayName }: { name: string; displayName: string }) => ({ - value: name.match(/([^\/]*)\/*$/)![1], - label: displayName || name.match(/([^\/]*)\/*$/)![1], - }), - }); + return this.getResource(`services/v3/projects/${this.templateSrv.replace(projectName)}/services?pageSize=1000`); } async getServiceLevelObjectives(projectName: string, serviceId: string): Promise>> { @@ -278,23 +224,11 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend< return Promise.resolve([]); } let { projectName: p, serviceId: s } = this.interpolateProps({ projectName, serviceId }); - return this.api.get(`${p}/services/${s}/serviceLevelObjectives`, { - responseMap: ({ name, displayName, goal }: { name: string; displayName: string; goal: number }) => ({ - value: name.match(/([^\/]*)\/*$/)![1], - label: displayName, - goal, - }), - }); + return this.getResource(`slo-services/v3/projects/${p}/services/${s}/serviceLevelObjectives`); } getProjects(): Promise>> { - return this.api.get(`projects`, { - responseMap: ({ projectId, name }: { projectId: string; name: string }) => ({ - value: projectId, - label: name, - }), - baseUrl: `/api/datasources/${this.id}/resources/cloudresourcemanager/v1/`, - }); + return this.getResource(`projects`); } migrateQuery(query: CloudMonitoringQuery): CloudMonitoringQuery { diff --git a/public/app/plugins/datasource/cloud-monitoring/specs/datasource.test.ts b/public/app/plugins/datasource/cloud-monitoring/specs/datasource.test.ts index 12b6dae9b35..db478db95d6 100644 --- a/public/app/plugins/datasource/cloud-monitoring/specs/datasource.test.ts +++ b/public/app/plugins/datasource/cloud-monitoring/specs/datasource.test.ts @@ -2,7 +2,6 @@ import { of, throwError } from 'rxjs'; import { DataSourceInstanceSettings, toUtc } from '@grafana/data'; import CloudMonitoringDataSource from '../datasource'; -import { metricDescriptors } from './testData'; import { TemplateSrv } from 'app/features/templating/template_srv'; import { CloudMonitoringOptions } from '../types'; import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__ @@ -41,47 +40,6 @@ function getTestcontext({ response = {}, throws = false, templateSrv = new Templ } describe('CloudMonitoringDataSource', () => { - describe('when performing testDataSource', () => { - describe('and call to cloud monitoring api succeeds', () => { - it('should return successfully', async () => { - const { ds } = getTestcontext(); - - const result = await ds.testDatasource(); - - expect(result.status).toBe('success'); - }); - }); - - describe('and a list of metricDescriptors are returned', () => { - it('should return status success', async () => { - const { ds } = getTestcontext({ response: metricDescriptors }); - - const result = await ds.testDatasource(); - - expect(result.status).toBe('success'); - }); - }); - - describe('and call to cloud monitoring api fails with 400 error', () => { - it('should return error status and a detailed error message', async () => { - const response = { - statusText: 'Bad Request', - data: { - error: { code: 400, message: 'Field interval.endTime had an invalid value' }, - }, - }; - const { ds } = getTestcontext({ response, throws: true }); - - const result = await ds.testDatasource(); - - expect(result.status).toEqual('error'); - expect(result.message).toBe( - 'Google Cloud Monitoring: Bad Request: 400. Field interval.endTime had an invalid value' - ); - }); - }); - }); - describe('When performing query', () => { describe('and no time series data is returned', () => { it('should return a list of datapoints', async () => { @@ -124,37 +82,6 @@ describe('CloudMonitoringDataSource', () => { }); }); - describe('when performing getMetricTypes', () => { - describe('and call to cloud monitoring api succeeds', () => { - it('should return successfully', async () => { - const response = { - metricDescriptors: [ - { - displayName: 'test metric name 1', - type: 'compute.googleapis.com/instance/cpu/test-metric-type-1', - description: 'A description', - }, - { - type: 'logging.googleapis.com/user/logbased-metric-with-no-display-name', - }, - ], - }; - const { ds } = getTestcontext({ response }); - - const result = await ds.getMetricTypes('proj'); - - expect(result.length).toBe(2); - expect(result[0].service).toBe('compute.googleapis.com'); - expect(result[0].serviceShortName).toBe('compute'); - expect(result[0].type).toBe('compute.googleapis.com/instance/cpu/test-metric-type-1'); - expect(result[0].displayName).toBe('test metric name 1'); - expect(result[0].description).toBe('A description'); - expect(result[1].type).toBe('logging.googleapis.com/user/logbased-metric-with-no-display-name'); - expect(result[1].displayName).toBe('logging.googleapis.com/user/logbased-metric-with-no-display-name'); - }); - }); - }); - describe('when interpolating a template variable for the filter', () => { describe('and is single value variable', () => { it('should replace the variable with the value', () => { diff --git a/public/app/plugins/datasource/cloud-monitoring/types.ts b/public/app/plugins/datasource/cloud-monitoring/types.ts index c9fa4aa44c0..135147f4db2 100644 --- a/public/app/plugins/datasource/cloud-monitoring/types.ts +++ b/public/app/plugins/datasource/cloud-monitoring/types.ts @@ -204,3 +204,7 @@ export interface CustomMetaData { perSeriesAligner?: string; alignmentPeriod?: string; } + +export interface PostResponse { + results: Record; +}