AzureMonitor: Migrate to backend checkHealth API (#50448)

* Add check health functions for each datasource and generic checkHealth function

* Log backend errors

* Update testDatasource function

- Remove unused testDatasource functions from pseudo datasources

* Switch datasource to extend DataSourceWithBackend

* Improve errors and responses from health endpoint

* Fix backend lint issues

* Remove unneeded frontend tests

* Remove unused/unnecessary datasource methods

* Update types

* Improve message construction

* Stubbing out checkHealth tests

* Update tests

- Remove comments
- Simplify structure

* Update log analytics health check to query data rather than retrieve workspace metadata

* Fix lint issue

* Fix frontend lint issues

* Update pkg/tsdb/azuremonitor/azuremonitor.go

Co-authored-by: Andres Martinez Gotor <andres.martinez@grafana.com>

* Updates based on PR comments

- Don't use deprecated default workspace field
- Handle situation if no workspace is found by notifying user
- Correctly handle health responses

* Remove debug line

* Make use of defined api versions

* Remove field validation functions

* Expose errors in frontend

* Update errors and tests

* Remove instanceSettings

* Update error handling

* Improve error handling and presentation

* Update tests and correctly check error type

* Refactor AzureHealthCheckError and update tests

* Fix lint errors

Co-authored-by: Andres Martinez Gotor <andres.martinez@grafana.com>
This commit is contained in:
Andreas Christou
2022-06-20 11:33:13 +01:00
committed by GitHub
co-authored by Andres Martinez Gotor
parent 62c2b1ec78
commit ecaa1dcbfd
10 changed files with 525 additions and 233 deletions
+203
View File
@@ -1,10 +1,14 @@
package azuremonitor
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
@@ -161,3 +165,202 @@ func (s *Service) newQueryMux() *datasource.QueryTypeMux {
}
return mux
}
func (s *Service) getDSInfo(pluginCtx backend.PluginContext) (types.DatasourceInfo, error) {
i, err := s.im.Get(pluginCtx)
if err != nil {
return types.DatasourceInfo{}, err
}
instance, ok := i.(types.DatasourceInfo)
if !ok {
return types.DatasourceInfo{}, fmt.Errorf("failed to cast datsource info")
}
return instance, nil
}
func checkAzureMonitorMetricsHealth(dsInfo types.DatasourceInfo) (*http.Response, error) {
url := fmt.Sprintf("%v/subscriptions?api-version=%v", dsInfo.Routes["Azure Monitor"].URL, metrics.AzureMonitorAPIVersion)
request, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
res, err := dsInfo.Services["Azure Monitor"].HTTPClient.Do(request)
if err != nil {
return nil, fmt.Errorf("%w: %v", types.ErrorAzureHealthCheck, err)
}
return res, nil
}
func checkAzureLogAnalyticsHealth(dsInfo types.DatasourceInfo) (*http.Response, error) {
workspacesUrl := fmt.Sprintf("%v/subscriptions/%v/providers/Microsoft.OperationalInsights/workspaces?api-version=2017-04-26-preview", dsInfo.Routes["Azure Monitor"].URL, dsInfo.Settings.SubscriptionId)
workspacesReq, err := http.NewRequest(http.MethodGet, workspacesUrl, nil)
if err != nil {
return nil, err
}
res, err := dsInfo.Services["Azure Monitor"].HTTPClient.Do(workspacesReq)
if err != nil {
return nil, fmt.Errorf("%w: %v", types.ErrorAzureHealthCheck, err)
}
var target struct {
Value []types.LogAnalyticsWorkspaceResponse
}
err = json.NewDecoder(res.Body).Decode(&target)
if err != nil {
return nil, err
}
if len(target.Value) == 0 {
return nil, errors.New("no default workspace found")
}
defaultWorkspaceId := target.Value[0].Properties.CustomerId
body, err := json.Marshal(map[string]interface{}{
"query": "AzureActivity | limit 1",
})
if err != nil {
return nil, err
}
workspaceUrl := fmt.Sprintf("%v/v1/workspaces/%v/query", dsInfo.Routes["Azure Log Analytics"].URL, defaultWorkspaceId)
workspaceReq, err := http.NewRequest(http.MethodPost, workspaceUrl, bytes.NewBuffer(body))
workspaceReq.Header.Set("Content-Type", "application/json")
if err != nil {
return nil, err
}
res, err = dsInfo.Services["Azure Log Analytics"].HTTPClient.Do(workspaceReq)
if err != nil {
return nil, fmt.Errorf("%w: %v", types.ErrorAzureHealthCheck, err)
}
return res, nil
}
func checkAzureMonitorResourceGraphHealth(dsInfo types.DatasourceInfo) (*http.Response, error) {
body, err := json.Marshal(map[string]interface{}{
"query": "Resources | project id | limit 1",
"subscriptions": []string{dsInfo.Settings.SubscriptionId},
})
if err != nil {
return nil, err
}
url := fmt.Sprintf("%v/providers/Microsoft.ResourceGraph/resources?api-version=%v", dsInfo.Routes["Azure Resource Graph"].URL, resourcegraph.ArgAPIVersion)
request, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(body))
request.Header.Set("Content-Type", "application/json")
if err != nil {
return nil, err
}
res, err := dsInfo.Services["Azure Resource Graph"].HTTPClient.Do(request)
if err != nil {
return nil, fmt.Errorf("%w: %v", types.ErrorAzureHealthCheck, err)
}
return res, nil
}
func (s *Service) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
dsInfo, err := s.getDSInfo(req.PluginContext)
if err != nil {
return nil, err
}
status := backend.HealthStatusOk
metricsLog := "Successfully connected to Azure Monitor endpoint."
logAnalyticsLog := "Successfully connected to Azure Log Analytics endpoint."
graphLog := "Successfully connected to Azure Resource Graph endpoint."
metricsRes, err := checkAzureMonitorMetricsHealth(dsInfo)
if err != nil || metricsRes.StatusCode != 200 {
status = backend.HealthStatusError
if err != nil {
if ok := errors.Is(err, types.ErrorAzureHealthCheck); ok {
metricsLog = fmt.Sprintf("Error connecting to Azure Monitor endpoint: %s", err.Error())
} else {
return nil, err
}
} else {
body, err := io.ReadAll(metricsRes.Body)
if err != nil {
return nil, err
}
metricsLog = fmt.Sprintf("Error connecting to Azure Monitor endpoint: %s", string(body))
}
}
logsRes, err := checkAzureLogAnalyticsHealth(dsInfo)
if err != nil || logsRes.StatusCode != 200 {
status = backend.HealthStatusError
if err != nil {
if err.Error() == "no default workspace found" {
status = backend.HealthStatusUnknown
logAnalyticsLog = "No Log Analytics workspaces found."
} else if ok := errors.Is(err, types.ErrorAzureHealthCheck); ok {
logAnalyticsLog = fmt.Sprintf("Error connecting to Azure Log Analytics endpoint: %s", err.Error())
} else {
return nil, err
}
} else {
body, err := io.ReadAll(logsRes.Body)
if err != nil {
return nil, err
}
logAnalyticsLog = fmt.Sprintf("Error connecting to Azure Log Analytics endpoint: %s", string(body))
}
}
resourceGraphRes, err := checkAzureMonitorResourceGraphHealth(dsInfo)
if err != nil || resourceGraphRes.StatusCode != 200 {
status = backend.HealthStatusError
if err != nil {
if ok := errors.Is(err, types.ErrorAzureHealthCheck); ok {
graphLog = fmt.Sprintf("Error connecting to Azure Resource Graph endpoint: %s", err.Error())
} else {
return nil, err
}
} else {
body, err := io.ReadAll(resourceGraphRes.Body)
if err != nil {
return nil, err
}
graphLog = fmt.Sprintf("Error connecting to Azure Resource Graph endpoint: %s", string(body))
}
}
defer func() {
if metricsRes != nil {
if err := metricsRes.Body.Close(); err != nil {
backend.Logger.Error("Failed to close response body", "err", err)
}
}
if logsRes != nil {
if err := logsRes.Body.Close(); logsRes != nil && err != nil {
backend.Logger.Error("Failed to close response body", "err", err)
}
}
if resourceGraphRes != nil {
if err := resourceGraphRes.Body.Close(); resourceGraphRes != nil && err != nil {
backend.Logger.Error("Failed to close response body", "err", err)
}
}
}()
if status == backend.HealthStatusOk {
return &backend.CheckHealthResult{
Status: status,
Message: "Successfully connected to all Azure Monitor endpoints.",
}, nil
}
return &backend.CheckHealthResult{
Status: status,
Message: "One or more health checks failed. See details below.",
JSONDetails: []byte(
fmt.Sprintf(`{"verboseMessage": %s }`, strconv.Quote(fmt.Sprintf("1. %s\n2. %s\n3. %s", metricsLog, logAnalyticsLog, graphLog))),
),
}, nil
}
+279
View File
@@ -1,8 +1,13 @@
package azuremonitor
import (
"bytes"
"context"
"encoding/json"
"errors"
"io/ioutil"
"net/http"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
@@ -16,6 +21,7 @@ import (
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -66,14 +72,18 @@ func TestNewInstanceSettings(t *testing.T) {
}
type fakeInstance struct {
cloud string
routes map[string]types.AzRoute
services map[string]types.DatasourceService
settings types.AzureMonitorSettings
}
func (f *fakeInstance) Get(pluginContext backend.PluginContext) (instancemgmt.Instance, error) {
return types.DatasourceInfo{
Cloud: f.cloud,
Routes: f.routes,
Services: f.services,
Settings: f.settings,
}, nil
}
@@ -158,3 +168,272 @@ func Test_newMux(t *testing.T) {
})
}
}
type RoundTripFunc func(req *http.Request) (*http.Response, error)
func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req)
}
func NewTestClient(fn RoundTripFunc) *http.Client {
return &http.Client{
Transport: fn,
}
}
func TestCheckHealth(t *testing.T) {
logAnalyticsResponse := func(empty bool) (*http.Response, error) {
if !empty {
body := struct {
Value []types.LogAnalyticsWorkspaceResponse
}{Value: []types.LogAnalyticsWorkspaceResponse{{
Id: "abcd-1234",
Location: "location",
Name: "test-workspace",
Properties: types.LogAnalyticsWorkspaceProperties{
CreatedDate: "",
CustomerId: "abcd-1234",
Features: types.LogAnalyticsWorkspaceFeatures{},
},
ProvisioningState: "provisioned",
PublicNetworkAccessForIngestion: "enabled",
PublicNetworkAccessForQuery: "disabled",
RetentionInDays: 0},
}}
bodyMarshal, err := json.Marshal(body)
if err != nil {
return nil, err
}
return &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewBuffer(bodyMarshal)),
Header: make(http.Header),
}, nil
} else {
body := struct {
Value []types.LogAnalyticsWorkspaceResponse
}{Value: []types.LogAnalyticsWorkspaceResponse{}}
bodyMarshal, err := json.Marshal(body)
if err != nil {
return nil, err
}
return &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewBuffer(bodyMarshal)),
Header: make(http.Header),
}, nil
}
}
azureMonitorClient := func(logAnalyticsEmpty bool, fail bool) *http.Client {
return NewTestClient(func(req *http.Request) (*http.Response, error) {
if strings.Contains(req.URL.String(), "workspaces") {
return logAnalyticsResponse(logAnalyticsEmpty)
} else {
if !fail {
return &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewBufferString("OK")),
Header: make(http.Header),
}, nil
} else {
return &http.Response{
StatusCode: 404,
Body: ioutil.NopCloser(bytes.NewBufferString("not found")),
Header: make(http.Header),
}, nil
}
}
})
}
okClient := NewTestClient(func(req *http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewBufferString("OK")),
Header: make(http.Header),
}, nil
})
failClient := func(azureHealthCheckError bool) *http.Client {
return NewTestClient(func(req *http.Request) (*http.Response, error) {
if azureHealthCheckError {
return nil, errors.New("not found")
}
return &http.Response{
StatusCode: 404,
Body: ioutil.NopCloser(bytes.NewBufferString("not found")),
Header: make(http.Header),
}, nil
})
}
cloud := "AzureCloud"
tests := []struct {
name string
errorExpected bool
expectedResult *backend.CheckHealthResult
customServices map[string]types.DatasourceService
}{
{
name: "Successfully queries all endpoints",
errorExpected: false,
expectedResult: &backend.CheckHealthResult{
Status: backend.HealthStatusOk,
Message: "Successfully connected to all Azure Monitor endpoints.",
},
customServices: map[string]types.DatasourceService{
azureMonitor: {
URL: routes[cloud]["Azure Monitor"].URL,
HTTPClient: azureMonitorClient(false, false),
},
azureLogAnalytics: {
URL: routes[cloud]["Azure Log Analytics"].URL,
HTTPClient: okClient,
},
azureResourceGraph: {
URL: routes[cloud]["Azure Resource Graph"].URL,
HTTPClient: okClient,
}},
},
{
name: "Successfully queries all endpoints except metrics",
errorExpected: false,
expectedResult: &backend.CheckHealthResult{
Status: backend.HealthStatusError,
Message: "One or more health checks failed. See details below.",
JSONDetails: []byte(
`{"verboseMessage": "1. Error connecting to Azure Monitor endpoint: not found\n2. Successfully connected to Azure Log Analytics endpoint.\n3. Successfully connected to Azure Resource Graph endpoint." }`),
},
customServices: map[string]types.DatasourceService{
azureMonitor: {
URL: routes[cloud]["Azure Monitor"].URL,
HTTPClient: azureMonitorClient(false, true),
},
azureLogAnalytics: {
URL: routes[cloud]["Azure Log Analytics"].URL,
HTTPClient: okClient,
},
azureResourceGraph: {
URL: routes[cloud]["Azure Resource Graph"].URL,
HTTPClient: okClient,
}},
},
{
name: "Successfully queries all endpoints except log analytics",
errorExpected: false,
expectedResult: &backend.CheckHealthResult{
Status: backend.HealthStatusError,
Message: "One or more health checks failed. See details below.",
JSONDetails: []byte(
`{"verboseMessage": "1. Successfully connected to Azure Monitor endpoint.\n2. Error connecting to Azure Log Analytics endpoint: not found\n3. Successfully connected to Azure Resource Graph endpoint." }`),
},
customServices: map[string]types.DatasourceService{
azureMonitor: {
URL: routes[cloud]["Azure Monitor"].URL,
HTTPClient: azureMonitorClient(false, false),
},
azureLogAnalytics: {
URL: routes[cloud]["Azure Log Analytics"].URL,
HTTPClient: failClient(false),
},
azureResourceGraph: {
URL: routes[cloud]["Azure Resource Graph"].URL,
HTTPClient: okClient,
}},
},
{
name: "Successfully queries all endpoints except resource graph",
errorExpected: false,
expectedResult: &backend.CheckHealthResult{
Status: backend.HealthStatusError,
Message: "One or more health checks failed. See details below.",
JSONDetails: []byte(
`{"verboseMessage": "1. Successfully connected to Azure Monitor endpoint.\n2. Successfully connected to Azure Log Analytics endpoint.\n3. Error connecting to Azure Resource Graph endpoint: not found" }`),
},
customServices: map[string]types.DatasourceService{
azureMonitor: {
URL: routes[cloud]["Azure Monitor"].URL,
HTTPClient: azureMonitorClient(false, false),
},
azureLogAnalytics: {
URL: routes[cloud]["Azure Log Analytics"].URL,
HTTPClient: okClient,
},
azureResourceGraph: {
URL: routes[cloud]["Azure Resource Graph"].URL,
HTTPClient: failClient(false),
}},
},
{
name: "Successfully returns UNKNOWN status if no log analytics workspace is found",
errorExpected: false,
expectedResult: &backend.CheckHealthResult{
Status: backend.HealthStatusUnknown,
Message: "One or more health checks failed. See details below.",
JSONDetails: []byte(
`{"verboseMessage": "1. Successfully connected to Azure Monitor endpoint.\n2. No Log Analytics workspaces found.\n3. Successfully connected to Azure Resource Graph endpoint." }`),
},
customServices: map[string]types.DatasourceService{
azureMonitor: {
URL: routes[cloud]["Azure Monitor"].URL,
HTTPClient: azureMonitorClient(true, false),
},
azureLogAnalytics: {
URL: routes[cloud]["Azure Log Analytics"].URL,
HTTPClient: okClient,
},
azureResourceGraph: {
URL: routes[cloud]["Azure Resource Graph"].URL,
HTTPClient: okClient,
}},
},
{
name: "Successfully returns Azure health check errors",
errorExpected: false,
expectedResult: &backend.CheckHealthResult{
Status: backend.HealthStatusError,
Message: "One or more health checks failed. See details below.",
JSONDetails: []byte(
`{"verboseMessage": "1. Error connecting to Azure Monitor endpoint: health check failed: Get \"https://management.azure.com/subscriptions?api-version=2018-01-01\": not found\n2. Error connecting to Azure Log Analytics endpoint: health check failed: Get \"https://management.azure.com/subscriptions//providers/Microsoft.OperationalInsights/workspaces?api-version=2017-04-26-preview\": not found\n3. Error connecting to Azure Resource Graph endpoint: health check failed: Post \"https://management.azure.com/providers/Microsoft.ResourceGraph/resources?api-version=2021-06-01-preview\": not found" }`),
},
customServices: map[string]types.DatasourceService{
azureMonitor: {
URL: routes[cloud]["Azure Monitor"].URL,
HTTPClient: failClient(true),
},
azureLogAnalytics: {
URL: routes[cloud]["Azure Log Analytics"].URL,
HTTPClient: failClient(true),
},
azureResourceGraph: {
URL: routes[cloud]["Azure Resource Graph"].URL,
HTTPClient: failClient(true),
}},
},
}
instance := &fakeInstance{
cloud: cloud,
routes: routes[cloud],
services: map[string]types.DatasourceService{},
settings: types.AzureMonitorSettings{
LogAnalyticsDefaultWorkspace: "workspace-id",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
instance.services = tt.customServices
s := &Service{
im: instance,
}
res, err := s.CheckHealth(context.Background(), &backend.CheckHealthRequest{
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
}})
if tt.errorExpected {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
assert.Equal(t, tt.expectedResult, res)
})
}
}
@@ -36,7 +36,7 @@ var (
resourceNameLandmark = regexp.MustCompile(`(?i)(/(?P<resourceName>[\w-\.]+)/providers/Microsoft\.Insights/metrics)`)
)
const azureMonitorAPIVersion = "2018-01-01"
const AzureMonitorAPIVersion = "2018-01-01"
func (e *AzureMonitorDatasource) ResourceRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client) {
e.Proxy.Do(rw, req, cli)
@@ -114,7 +114,7 @@ func (e *AzureMonitorDatasource) buildQueries(queries []backend.DataQuery, dsInf
}
params := url.Values{}
params.Add("api-version", azureMonitorAPIVersion)
params.Add("api-version", AzureMonitorAPIVersion)
params.Add("timespan", fmt.Sprintf("%v/%v", query.TimeRange.From.UTC().Format(time.RFC3339), query.TimeRange.To.UTC().Format(time.RFC3339)))
params.Add("interval", timeGrain)
params.Add("aggregation", azJSONModel.Aggregation)
@@ -46,7 +46,7 @@ type AzureResourceGraphQuery struct {
TimeRange backend.TimeRange
}
const argAPIVersion = "2021-06-01-preview"
const ArgAPIVersion = "2021-06-01-preview"
const argQueryProviderName = "/providers/Microsoft.ResourceGraph/resources"
func (e *AzureResourceGraphDatasource) ResourceRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client) {
@@ -123,7 +123,7 @@ func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *
dataResponse := backend.DataResponse{}
params := url.Values{}
params.Add("api-version", argAPIVersion)
params.Add("api-version", ArgAPIVersion)
dataResponseErrorWithExecuted := func(err error) backend.DataResponse {
dataResponse = backend.DataResponse{Error: err}
+26
View File
@@ -1,6 +1,7 @@
package types
import (
"errors"
"fmt"
"net/http"
"net/url"
@@ -190,3 +191,28 @@ type MetricVisualization struct {
type ServiceProxy interface {
Do(rw http.ResponseWriter, req *http.Request, cli *http.Client) http.ResponseWriter
}
type LogAnalyticsWorkspaceFeatures struct {
EnableLogAccessUsingOnlyResourcePermissions bool `json:"enableLogAccessUsingOnlyResourcePermissions"`
Legacy int `json:"legacy"`
SearchVersion int `json:"searchVersion"`
}
type LogAnalyticsWorkspaceProperties struct {
CreatedDate string `json:"createdDate"`
CustomerId string `json:"customerId"`
Features LogAnalyticsWorkspaceFeatures `json:"features"`
}
type LogAnalyticsWorkspaceResponse struct {
Id string `json:"id"`
Location string `json:"location"`
Name string `json:"name"`
Properties LogAnalyticsWorkspaceProperties `json:"properties"`
ProvisioningState string `json:"provisioningState"`
PublicNetworkAccessForIngestion string `json:"publicNetworkAccessForIngestion"`
PublicNetworkAccessForQuery string `json:"publicNetworkAccessForQuery"`
RetentionInDays int `json:"retentionInDays"`
}
var ErrorAzureHealthCheck = errors.New("health check failed")