IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

AzureMonitor: Use plugin SDK contracts (#34729)

Browse Source
This commit is contained in:
Andres Martinez Gotor 2021-06-07 14:54:51 +02:00 committed by GitHub
parent e8bc48a796
commit d225323049
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 652 additions and 735 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
pkg/tsdb/azuremonitor/applicationinsights-datasource.go
View File

@ -13,8 +13,10 @@ import (
"strings" "strings"
"time" "time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/pluginproxy" "github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
@ -26,8 +28,6 @@ import (
// ApplicationInsightsDatasource calls the application insights query API. // ApplicationInsightsDatasource calls the application insights query API.
type ApplicationInsightsDatasource struct { type ApplicationInsightsDatasource struct {
httpClient *http.Client
dsInfo *models.DataSource
pluginManager plugins.Manager pluginManager plugins.Manager
cfg *setting.Cfg cfg *setting.Cfg
} }
@ -36,7 +36,8 @@ type ApplicationInsightsDatasource struct {
// needed to make a metrics query to Application Insights, and the information // needed to make a metrics query to Application Insights, and the information
// used to parse the response. // used to parse the response.
type ApplicationInsightsQuery struct { type ApplicationInsightsQuery struct {
RefID string RefID string
TimeRange backend.TimeRange
// Text based raw query options. // Text based raw query options.
ApiURL string ApiURL string
@ -50,45 +51,31 @@ type ApplicationInsightsQuery struct {
aggregation string aggregation string
} }
// nolint:staticcheck // plugins.DataQueryResult deprecated
func (e *ApplicationInsightsDatasource) executeTimeSeriesQuery(ctx context.Context, func (e *ApplicationInsightsDatasource) executeTimeSeriesQuery(ctx context.Context,
originalQueries []plugins.DataSubQuery, originalQueries []backend.DataQuery, dsInfo datasourceInfo) (*backend.QueryDataResponse, error) {
timeRange plugins.DataTimeRange) (plugins.DataResponse, error) { result := backend.NewQueryDataResponse()
result := plugins.DataResponse{
Results: map[string]plugins.DataQueryResult{},
}
queries, err := e.buildQueries(originalQueries, timeRange) queries, err := e.buildQueries(originalQueries)
if err != nil { if err != nil {
return plugins.DataResponse{}, err return nil, err
} }
for _, query := range queries { for _, query := range queries {
queryRes, err := e.executeQuery(ctx, query) queryRes, err := e.executeQuery(ctx, query, dsInfo)
if err != nil { if err != nil {
return plugins.DataResponse{}, err return nil, err
} }
result.Results[query.RefID] = queryRes result.Responses[query.RefID] = queryRes
} }
return result, nil return result, nil
} }
func (e *ApplicationInsightsDatasource) buildQueries(queries []plugins.DataSubQuery, func (e *ApplicationInsightsDatasource) buildQueries(queries []backend.DataQuery) ([]*ApplicationInsightsQuery, error) {
timeRange plugins.DataTimeRange) ([]*ApplicationInsightsQuery, error) {
applicationInsightsQueries := []*ApplicationInsightsQuery{} applicationInsightsQueries := []*ApplicationInsightsQuery{}
startTime, err := timeRange.ParseFrom()
if err != nil {
return nil, err
}
endTime, err := timeRange.ParseTo()
if err != nil {
return nil, err
}
for _, query := range queries { for _, query := range queries {
queryBytes, err := query.Model.Encode() queryBytes, err := query.JSON.MarshalJSON()
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to re-encode the Azure Application Insights query into JSON: %w", err) return nil, fmt.Errorf("failed to re-encode the Azure Application Insights query into JSON: %w", err)
} }
@ -108,14 +95,14 @@ func (e *ApplicationInsightsDatasource) buildQueries(queries []plugins.DataSubQu
// Previous versions of the query model don't specify a time grain, so we // Previous versions of the query model don't specify a time grain, so we
// need to fallback to a default value // need to fallback to a default value
if timeGrain == "auto" || timeGrain == "" { if timeGrain == "auto" || timeGrain == "" {
timeGrain, err = setAutoTimeGrain(query.IntervalMS, timeGrains) timeGrain, err = setAutoTimeGrain(query.Interval.Milliseconds(), timeGrains)
if err != nil { if err != nil {
return nil, err return nil, err
} }
} }
params := url.Values{} params := url.Values{}
params.Add("timespan", fmt.Sprintf("%v/%v", startTime.UTC().Format(time.RFC3339), endTime.UTC().Format(time.RFC3339))) params.Add("timespan", fmt.Sprintf("%v/%v", query.TimeRange.From.UTC().Format(time.RFC3339), query.TimeRange.To.UTC().Format(time.RFC3339)))
if timeGrain != "none" { if timeGrain != "none" {
params.Add("interval", timeGrain) params.Add("interval", timeGrain)
} }
@ -131,6 +118,7 @@ func (e *ApplicationInsightsDatasource) buildQueries(queries []plugins.DataSubQu
} }
applicationInsightsQueries = append(applicationInsightsQueries, &ApplicationInsightsQuery{ applicationInsightsQueries = append(applicationInsightsQueries, &ApplicationInsightsQuery{
RefID: query.RefID, RefID: query.RefID,
TimeRange: query.TimeRange,
ApiURL: azureURL, ApiURL: azureURL,
Params: params, Params: params,
Alias: insightsJSONModel.Alias, Alias: insightsJSONModel.Alias,
@ -144,15 +132,14 @@ func (e *ApplicationInsightsDatasource) buildQueries(queries []plugins.DataSubQu
return applicationInsightsQueries, nil return applicationInsightsQueries, nil
} }
// nolint:staticcheck // plugins.DataQueryResult deprecated func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query *ApplicationInsightsQuery, dsInfo datasourceInfo) (
func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query *ApplicationInsightsQuery) ( backend.DataResponse, error) {
plugins.DataQueryResult, error) { dataResponse := backend.DataResponse{}
queryResult := plugins.DataQueryResult{Meta: simplejson.New(), RefID: query.RefID}
req, err := e.createRequest(ctx, e.dsInfo) req, err := e.createRequest(ctx, dsInfo)
if err != nil { if err != nil {
queryResult.Error = err dataResponse.Error = err
return queryResult, nil return dataResponse, nil
} }
req.URL.Path = path.Join(req.URL.Path, query.ApiURL) req.URL.Path = path.Join(req.URL.Path, query.ApiURL)
@ -160,8 +147,10 @@ func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query
span, ctx := opentracing.StartSpanFromContext(ctx, "application insights query") span, ctx := opentracing.StartSpanFromContext(ctx, "application insights query")
span.SetTag("target", query.Target) span.SetTag("target", query.Target)
span.SetTag("datasource_id", e.dsInfo.Id) span.SetTag("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond))
span.SetTag("org_id", e.dsInfo.OrgId) span.SetTag("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond))
span.SetTag("datasource_id", dsInfo.DatasourceID)
span.SetTag("org_id", dsInfo.OrgID)
defer span.Finish() defer span.Finish()
@ -175,10 +164,10 @@ func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query
} }
azlog.Debug("ApplicationInsights", "Request URL", req.URL.String()) azlog.Debug("ApplicationInsights", "Request URL", req.URL.String())
res, err := ctxhttp.Do(ctx, e.httpClient, req) res, err := ctxhttp.Do(ctx, dsInfo.HTTPClient, req)
if err != nil { if err != nil {
queryResult.Error = err dataResponse.Error = err
return queryResult, nil return dataResponse, nil
} }
body, err := ioutil.ReadAll(res.Body) body, err := ioutil.ReadAll(res.Body)
@ -188,48 +177,47 @@ func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query
} }
}() }()
if err != nil { if err != nil {
return plugins.DataQueryResult{}, err return backend.DataResponse{}, err
} }
if res.StatusCode/100 != 2 { if res.StatusCode/100 != 2 {
azlog.Debug("Request failed", "status", res.Status, "body", string(body)) azlog.Debug("Request failed", "status", res.Status, "body", string(body))
return plugins.DataQueryResult{}, fmt.Errorf("request failed, status: %s", res.Status) return backend.DataResponse{}, fmt.Errorf("request failed, status: %s", res.Status)
} }
mr := MetricsResult{} mr := MetricsResult{}
err = json.Unmarshal(body, &mr) err = json.Unmarshal(body, &mr)
if err != nil { if err != nil {
return plugins.DataQueryResult{}, err return backend.DataResponse{}, err
} }
frame, err := InsightsMetricsResultToFrame(mr, query.metricName, query.aggregation, query.dimensions) frame, err := InsightsMetricsResultToFrame(mr, query.metricName, query.aggregation, query.dimensions)
if err != nil { if err != nil {
queryResult.Error = err dataResponse.Error = err
return queryResult, nil return dataResponse, nil
} }
applyInsightsMetricAlias(frame, query.Alias) applyInsightsMetricAlias(frame, query.Alias)
queryResult.Dataframes = plugins.NewDecodedDataFrames(data.Frames{frame}) dataResponse.Frames = data.Frames{frame}
return queryResult, nil return dataResponse, nil
} }
func (e *ApplicationInsightsDatasource) createRequest(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) { func (e *ApplicationInsightsDatasource) createRequest(ctx context.Context, dsInfo datasourceInfo) (*http.Request, error) {
// find plugin // find plugin
plugin := e.pluginManager.GetDataSource(dsInfo.Type) plugin := e.pluginManager.GetDataSource(dsName)
if plugin == nil { if plugin == nil {
return nil, errors.New("unable to find datasource plugin Azure Application Insights") return nil, errors.New("unable to find datasource plugin Azure Application Insights")
} }
appInsightsRoute, routeName, err := e.getPluginRoute(plugin) appInsightsRoute, routeName, err := e.getPluginRoute(plugin, dsInfo)
if err != nil { if err != nil {
return nil, err return nil, err
} }
appInsightsAppID := dsInfo.JsonData.Get("appInsightsAppId").MustString() appInsightsAppID := dsInfo.Settings.AppInsightsAppId
proxyPass := fmt.Sprintf("%s/v1/apps/%s", routeName, appInsightsAppID)
u, err := url.Parse(dsInfo.Url) u, err := url.Parse(dsInfo.URL)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -241,13 +229,18 @@ func (e *ApplicationInsightsDatasource) createRequest(ctx context.Context, dsInf
return nil, errutil.Wrap("Failed to create request", err) return nil, errutil.Wrap("Failed to create request", err)
} }
pluginproxy.ApplyRoute(ctx, req, proxyPass, appInsightsRoute, dsInfo, e.cfg) // TODO: Use backend authentication instead
proxyPass := fmt.Sprintf("%s/v1/apps/%s", routeName, appInsightsAppID)
pluginproxy.ApplyRoute(ctx, req, proxyPass, appInsightsRoute, &models.DataSource{
JsonData: simplejson.NewFromAny(dsInfo.JSONData),
SecureJsonData: securejsondata.GetEncryptedJsonData(dsInfo.DecryptedSecureJSONData),
}, e.cfg)
return req, nil return req, nil
} }
func (e *ApplicationInsightsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin) (*plugins.AppPluginRoute, string, error) { func (e *ApplicationInsightsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, dsInfo datasourceInfo) (*plugins.AppPluginRoute, string, error) {
cloud, err := getAzureCloud(e.cfg, e.dsInfo.JsonData) cloud, err := getAzureCloud(e.cfg, dsInfo)
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }

140
pkg/tsdb/azuremonitor/applicationinsights-datasource_test.go
View File

@ -2,14 +2,12 @@ package azuremonitor
import ( import (
"encoding/json" "encoding/json"
"fmt"
"testing" "testing"
"time" "time"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-cmp/cmp/cmpopts"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -23,33 +21,28 @@ func TestApplicationInsightsDatasource(t *testing.T) {
Convey("Parse queries from frontend and build AzureMonitor API queries", func() { Convey("Parse queries from frontend and build AzureMonitor API queries", func() {
fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local)
tsdbQuery := plugins.DataQuery{ tsdbQuery := []backend.DataQuery{
TimeRange: &plugins.DataTimeRange{ {
From: fmt.Sprintf("%v", fromStart.Unix()*1000), TimeRange: backend.TimeRange{
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000), From: fromStart,
}, To: fromStart.Add(34 * time.Minute),
Queries: []plugins.DataSubQuery{
{
DataSource: &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{}),
},
Model: simplejson.NewFromAny(map[string]interface{}{
"appInsights": map[string]interface{}{
"rawQuery": false,
"timeGrain": "PT1M",
"aggregation": "Average",
"metricName": "server/exceptions",
"alias": "testalias",
"queryType": "Application Insights",
},
}),
RefID: "A",
IntervalMS: 1234,
}, },
JSON: []byte(`{
"appInsights": {
"rawQuery": false,
"timeGrain": "PT1M",
"aggregation": "Average",
"metricName": "server/exceptions",
"alias": "testalias",
"queryType": "Application Insights"
}
}`),
RefID: "A",
Interval: 1234,
}, },
} }
Convey("and is a normal query", func() { Convey("and is a normal query", func() {
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange) queries, err := datasource.buildQueries(tsdbQuery)
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(len(queries), ShouldEqual, 1) So(len(queries), ShouldEqual, 1)
@ -64,66 +57,68 @@ func TestApplicationInsightsDatasource(t *testing.T) {
}) })
Convey("and has a time grain set to auto", func() { Convey("and has a time grain set to auto", func() {
tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ tsdbQuery[0].JSON = []byte(`{
"appInsights": map[string]interface{}{ "appInsights": {
"rawQuery": false, "rawQuery": false,
"timeGrain": "auto", "timeGrain": "auto",
"aggregation": "Average", "aggregation": "Average",
"metricName": "Percentage CPU", "metricName": "Percentage CPU",
"alias": "testalias", "alias": "testalias",
"queryType": "Application Insights", "queryType": "Application Insights"
}, }
}) }`)
tsdbQuery.Queries[0].IntervalMS = 400000 var err error
tsdbQuery[0].Interval, err = time.ParseDuration("400s")
require.NoError(t, err)
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange) queries, err := datasource.buildQueries(tsdbQuery)
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(queries[0].Params["interval"][0], ShouldEqual, "PT15M") So(queries[0].Params["interval"][0], ShouldEqual, "PT15M")
}) })
Convey("and has an empty time grain", func() { Convey("and has an empty time grain", func() {
tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ tsdbQuery[0].JSON = []byte(`{
"appInsights": map[string]interface{}{ "appInsights": {
"rawQuery": false, "rawQuery": false,
"timeGrain": "", "timeGrain": "",
"aggregation": "Average", "aggregation": "Average",
"metricName": "Percentage CPU", "metricName": "Percentage CPU",
"alias": "testalias", "alias": "testalias",
"queryType": "Application Insights", "queryType": "Application Insights"
}, }
}) }`)
tsdbQuery.Queries[0].IntervalMS = 400000 tsdbQuery[0].Interval, _ = time.ParseDuration("400s")
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange) queries, err := datasource.buildQueries(tsdbQuery)
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(queries[0].Params["interval"][0], ShouldEqual, "PT15M") So(queries[0].Params["interval"][0], ShouldEqual, "PT15M")
}) })
Convey("and has a time grain set to auto and the metric has a limited list of allowed time grains", func() { Convey("and has a time grain set to auto and the metric has a limited list of allowed time grains", func() {
tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ tsdbQuery[0].JSON = []byte(`{
"appInsights": map[string]interface{}{ "appInsights": {
"rawQuery": false, "rawQuery": false,
"timeGrain": "auto", "timeGrain": "auto",
"aggregation": "Average", "aggregation": "Average",
"metricName": "Percentage CPU", "metricName": "Percentage CPU",
"alias": "testalias", "alias": "testalias",
"queryType": "Application Insights", "queryType": "Application Insights",
"allowedTimeGrainsMs": []int64{60000, 300000}, "allowedTimeGrainsMs": [60000, 300000]
}, }
}) }`)
tsdbQuery.Queries[0].IntervalMS = 400000 tsdbQuery[0].Interval, _ = time.ParseDuration("400s")
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange) queries, err := datasource.buildQueries(tsdbQuery)
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(queries[0].Params["interval"][0], ShouldEqual, "PT5M") So(queries[0].Params["interval"][0], ShouldEqual, "PT5M")
}) })
Convey("and has a dimension filter", func() { Convey("and has a dimension filter", func() {
tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ tsdbQuery[0].JSON = []byte(`{
"appInsights": map[string]interface{}{ "appInsights": {
"rawQuery": false, "rawQuery": false,
"timeGrain": "PT1M", "timeGrain": "PT1M",
"aggregation": "Average", "aggregation": "Average",
@ -131,11 +126,11 @@ func TestApplicationInsightsDatasource(t *testing.T) {
"alias": "testalias", "alias": "testalias",
"queryType": "Application Insights", "queryType": "Application Insights",
"dimension": "blob", "dimension": "blob",
"dimensionFilter": "blob eq '*'", "dimensionFilter": "blob eq '*'"
}, }
}) }`)
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange) queries, err := datasource.buildQueries(tsdbQuery)
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(queries[0].Target, ShouldEqual, "aggregation=Average&filter=blob+eq+%27%2A%27&interval=PT1M&segment=blob&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z") So(queries[0].Target, ShouldEqual, "aggregation=Average&filter=blob+eq+%27%2A%27&interval=PT1M&segment=blob&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z")
@ -143,19 +138,19 @@ func TestApplicationInsightsDatasource(t *testing.T) {
}) })
Convey("and has a dimension filter set to None", func() { Convey("and has a dimension filter set to None", func() {
tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{ tsdbQuery[0].JSON = []byte(`{
"appInsights": map[string]interface{}{ "appInsights": {
"rawQuery": false, "rawQuery": false,
"timeGrain": "PT1M", "timeGrain": "PT1M",
"aggregation": "Average", "aggregation": "Average",
"metricName": "Percentage CPU", "metricName": "Percentage CPU",
"alias": "testalias", "alias": "testalias",
"queryType": "Application Insights", "queryType": "Application Insights",
"dimension": "None", "dimension": "None"
}, }
}) }`)
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange) queries, err := datasource.buildQueries(tsdbQuery)
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(queries[0].Target, ShouldEqual, "aggregation=Average&interval=PT1M&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z") So(queries[0].Target, ShouldEqual, "aggregation=Average&interval=PT1M&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z")
@ -198,20 +193,21 @@ func TestAppInsightsPluginRoutes(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
datasource *ApplicationInsightsDatasource datasource *ApplicationInsightsDatasource
dsInfo datasourceInfo
expectedRouteName string expectedRouteName string
expectedRouteURL string expectedRouteURL string
Err require.ErrorAssertionFunc Err require.ErrorAssertionFunc
}{ }{
{ {
name: "plugin proxy route for the Azure public cloud", name: "plugin proxy route for the Azure public cloud",
dsInfo: datasourceInfo{
Settings: azureMonitorSettings{
AzureAuthType: AzureAuthClientSecret,
CloudName: "azuremonitor",
},
},
datasource: &ApplicationInsightsDatasource{ datasource: &ApplicationInsightsDatasource{
cfg: cfg, cfg: cfg,
dsInfo: &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{
"azureAuthType": AzureAuthClientSecret,
"cloudName": "azuremonitor",
}),
},
}, },
expectedRouteName: "appinsights", expectedRouteName: "appinsights",
expectedRouteURL: "https://api.applicationinsights.io", expectedRouteURL: "https://api.applicationinsights.io",
@ -219,14 +215,14 @@ func TestAppInsightsPluginRoutes(t *testing.T) {
}, },
{ {
name: "plugin proxy route for the Azure China cloud", name: "plugin proxy route for the Azure China cloud",
dsInfo: datasourceInfo{
Settings: azureMonitorSettings{
AzureAuthType: AzureAuthClientSecret,
CloudName: "chinaazuremonitor",
},
},
datasource: &ApplicationInsightsDatasource{ datasource: &ApplicationInsightsDatasource{
cfg: cfg, cfg: cfg,
dsInfo: &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{
"azureAuthType": AzureAuthClientSecret,
"cloudName": "chinaazuremonitor",
}),
},
}, },
expectedRouteName: "chinaappinsights", expectedRouteName: "chinaappinsights",
expectedRouteURL: "https://api.applicationinsights.azure.cn", expectedRouteURL: "https://api.applicationinsights.azure.cn",
@ -236,7 +232,7 @@ func TestAppInsightsPluginRoutes(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
route, routeName, err := tt.datasource.getPluginRoute(plugin) route, routeName, err := tt.datasource.getPluginRoute(plugin, tt.dsInfo)
tt.Err(t, err) tt.Err(t, err)
if diff := cmp.Diff(tt.expectedRouteURL, route.URL, cmpopts.EquateNaNs()); diff != "" { if diff := cmp.Diff(tt.expectedRouteURL, route.URL, cmpopts.EquateNaNs()); diff != "" {

130
pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go
View File

@ -12,9 +12,12 @@ import (
"net/url" "net/url"
"path" "path"
"regexp" "regexp"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/pluginproxy" "github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
@ -26,8 +29,6 @@ import (
// AzureLogAnalyticsDatasource calls the Azure Log Analytics API's // AzureLogAnalyticsDatasource calls the Azure Log Analytics API's
type AzureLogAnalyticsDatasource struct { type AzureLogAnalyticsDatasource struct {
httpClient *http.Client
dsInfo *models.DataSource
pluginManager plugins.Manager pluginManager plugins.Manager
cfg *setting.Cfg cfg *setting.Cfg
} }
@ -38,29 +39,26 @@ type AzureLogAnalyticsQuery struct {
RefID string RefID string
ResultFormat string ResultFormat string
URL string URL string
Model *simplejson.Json JSON json.RawMessage
Params url.Values Params url.Values
Target string Target string
TimeRange backend.TimeRange
} }
// executeTimeSeriesQuery does the following: // executeTimeSeriesQuery does the following:
// 1. build the AzureMonitor url and querystring for each query // 1. build the AzureMonitor url and querystring for each query
// 2. executes each query by calling the Azure Monitor API // 2. executes each query by calling the Azure Monitor API
// 3. parses the responses for each query into the timeseries format // 3. parses the responses for each query into data frames
//nolint: staticcheck // plugins.DataPlugin deprecated func (e *AzureLogAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo) (*backend.QueryDataResponse, error) {
func (e *AzureLogAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []plugins.DataSubQuery, result := backend.NewQueryDataResponse()
timeRange plugins.DataTimeRange) (plugins.DataResponse, error) {
result := plugins.DataResponse{
Results: map[string]plugins.DataQueryResult{},
}
queries, err := e.buildQueries(originalQueries, timeRange) queries, err := e.buildQueries(originalQueries, dsInfo)
if err != nil { if err != nil {
return plugins.DataResponse{}, err return nil, err
} }
for _, query := range queries { for _, query := range queries {
result.Results[query.RefID] = e.executeQuery(ctx, query, originalQueries, timeRange) result.Responses[query.RefID] = e.executeQuery(ctx, query, dsInfo)
} }
return result, nil return result, nil
@ -89,18 +87,12 @@ func getApiURL(queryJSONModel logJSONQuery) string {
} }
} }
func (e *AzureLogAnalyticsDatasource) buildQueries(queries []plugins.DataSubQuery, func (e *AzureLogAnalyticsDatasource) buildQueries(queries []backend.DataQuery, dsInfo datasourceInfo) ([]*AzureLogAnalyticsQuery, error) {
timeRange plugins.DataTimeRange) ([]*AzureLogAnalyticsQuery, error) {
azureLogAnalyticsQueries := []*AzureLogAnalyticsQuery{} azureLogAnalyticsQueries := []*AzureLogAnalyticsQuery{}
for _, query := range queries { for _, query := range queries {
queryBytes, err := query.Model.Encode()
if err != nil {
return nil, fmt.Errorf("failed to re-encode the Azure Log Analytics query into JSON: %w", err)
}
queryJSONModel := logJSONQuery{} queryJSONModel := logJSONQuery{}
err = json.Unmarshal(queryBytes, &queryJSONModel) err := json.Unmarshal(query.JSON, &queryJSONModel)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to decode the Azure Log Analytics query object from JSON: %w", err) return nil, fmt.Errorf("failed to decode the Azure Log Analytics query object from JSON: %w", err)
} }
@ -116,7 +108,7 @@ func (e *AzureLogAnalyticsDatasource) buildQueries(queries []plugins.DataSubQuer
apiURL := getApiURL(queryJSONModel) apiURL := getApiURL(queryJSONModel)
params := url.Values{} params := url.Values{}
rawQuery, err := KqlInterpolate(query, timeRange, azureLogAnalyticsTarget.Query, "TimeGenerated") rawQuery, err := KqlInterpolate(query, dsInfo, azureLogAnalyticsTarget.Query, "TimeGenerated")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -126,23 +118,22 @@ func (e *AzureLogAnalyticsDatasource) buildQueries(queries []plugins.DataSubQuer
RefID: query.RefID, RefID: query.RefID,
ResultFormat: resultFormat, ResultFormat: resultFormat,
URL: apiURL, URL: apiURL,
Model: query.Model, JSON: query.JSON,
Params: params, Params: params,
Target: params.Encode(), Target: params.Encode(),
TimeRange: query.TimeRange,
}) })
} }
return azureLogAnalyticsQueries, nil return azureLogAnalyticsQueries, nil
} }
//nolint: staticcheck // plugins.DataPlugin deprecated func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *AzureLogAnalyticsQuery, dsInfo datasourceInfo) backend.DataResponse {
func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *AzureLogAnalyticsQuery, dataResponse := backend.DataResponse{}
queries []plugins.DataSubQuery, timeRange plugins.DataTimeRange) plugins.DataQueryResult {
queryResult := plugins.DataQueryResult{RefID: query.RefID}
queryResultErrorWithExecuted := func(err error) plugins.DataQueryResult { dataResponseErrorWithExecuted := func(err error) backend.DataResponse {
queryResult.Error = err dataResponse.Error = err
frames := data.Frames{ dataResponse.Frames = data.Frames{
&data.Frame{ &data.Frame{
RefID: query.RefID, RefID: query.RefID,
Meta: &data.FrameMeta{ Meta: &data.FrameMeta{
@ -150,14 +141,13 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
}, },
}, },
} }
queryResult.Dataframes = plugins.NewDecodedDataFrames(frames) return dataResponse
return queryResult
} }
req, err := e.createRequest(ctx, e.dsInfo) req, err := e.createRequest(ctx, dsInfo)
if err != nil { if err != nil {
queryResult.Error = err dataResponse.Error = err
return queryResult return dataResponse
} }
req.URL.Path = path.Join(req.URL.Path, query.URL) req.URL.Path = path.Join(req.URL.Path, query.URL)
@ -165,10 +155,10 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
span, ctx := opentracing.StartSpanFromContext(ctx, "azure log analytics query") span, ctx := opentracing.StartSpanFromContext(ctx, "azure log analytics query")
span.SetTag("target", query.Target) span.SetTag("target", query.Target)
span.SetTag("from", timeRange.From) span.SetTag("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond))
span.SetTag("until", timeRange.To) span.SetTag("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond))
span.SetTag("datasource_id", e.dsInfo.Id) span.SetTag("datasource_id", dsInfo.DatasourceID)
span.SetTag("org_id", e.dsInfo.OrgId) span.SetTag("org_id", dsInfo.OrgID)
defer span.Finish() defer span.Finish()
@ -176,34 +166,39 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
span.Context(), span.Context(),
opentracing.HTTPHeaders, opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(req.Header)); err != nil { opentracing.HTTPHeadersCarrier(req.Header)); err != nil {
return queryResultErrorWithExecuted(err) return dataResponseErrorWithExecuted(err)
} }
azlog.Debug("AzureLogAnalytics", "Request ApiURL", req.URL.String()) azlog.Debug("AzureLogAnalytics", "Request ApiURL", req.URL.String())
res, err := ctxhttp.Do(ctx, e.httpClient, req) res, err := ctxhttp.Do(ctx, dsInfo.HTTPClient, req)
if err != nil { if err != nil {
return queryResultErrorWithExecuted(err) return dataResponseErrorWithExecuted(err)
} }
logResponse, err := e.unmarshalResponse(res) logResponse, err := e.unmarshalResponse(res)
if err != nil { if err != nil {
return queryResultErrorWithExecuted(err) return dataResponseErrorWithExecuted(err)
} }
t, err := logResponse.GetPrimaryResultTable() t, err := logResponse.GetPrimaryResultTable()
if err != nil { if err != nil {
return queryResultErrorWithExecuted(err) return dataResponseErrorWithExecuted(err)
} }
frame, err := ResponseTableToFrame(t) frame, err := ResponseTableToFrame(t)
if err != nil { if err != nil {
return queryResultErrorWithExecuted(err) return dataResponseErrorWithExecuted(err)
}
model, err := simplejson.NewJson(query.JSON)
if err != nil {
return dataResponseErrorWithExecuted(err)
} }
err = setAdditionalFrameMeta(frame, err = setAdditionalFrameMeta(frame,
query.Params.Get("query"), query.Params.Get("query"),
query.Model.Get("subscriptionId").MustString(), model.Get("subscriptionId").MustString(),
query.Model.Get("azureLogAnalytics").Get("workspace").MustString()) model.Get("azureLogAnalytics").Get("workspace").MustString())
if err != nil { if err != nil {
frame.AppendNotices(data.Notice{Severity: data.NoticeSeverityWarning, Text: "could not add custom metadata: " + err.Error()}) frame.AppendNotices(data.Notice{Severity: data.NoticeSeverityWarning, Text: "could not add custom metadata: " + err.Error()})
azlog.Warn("failed to add custom metadata to azure log analytics response", err) azlog.Warn("failed to add custom metadata to azure log analytics response", err)
@ -220,13 +215,23 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
} }
} }
} }
frames := data.Frames{frame} dataResponse.Frames = data.Frames{frame}
queryResult.Dataframes = plugins.NewDecodedDataFrames(frames) return dataResponse
return queryResult
} }
func (e *AzureLogAnalyticsDatasource) createRequest(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) { func (e *AzureLogAnalyticsDatasource) createRequest(ctx context.Context, dsInfo datasourceInfo) (*http.Request, error) {
u, err := url.Parse(dsInfo.Url) // find plugin
plugin := e.pluginManager.GetDataSource(dsName)
if plugin == nil {
return nil, errors.New("unable to find datasource plugin Azure Monitor")
}
logAnalyticsRoute, routeName, err := e.getPluginRoute(plugin, dsInfo)
if err != nil {
return nil, err
}
u, err := url.Parse(dsInfo.URL)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -240,24 +245,17 @@ func (e *AzureLogAnalyticsDatasource) createRequest(ctx context.Context, dsInfo
req.Header.Set("Content-Type", "application/json") req.Header.Set("Content-Type", "application/json")
// find plugin // TODO: Use backend authentication instead
plugin := e.pluginManager.GetDataSource(dsInfo.Type) pluginproxy.ApplyRoute(ctx, req, routeName, logAnalyticsRoute, &models.DataSource{
if plugin == nil { JsonData: simplejson.NewFromAny(dsInfo.JSONData),
return nil, errors.New("unable to find datasource plugin Azure Monitor") SecureJsonData: securejsondata.GetEncryptedJsonData(dsInfo.DecryptedSecureJSONData),
} }, e.cfg)
logAnalyticsRoute, routeName, err := e.getPluginRoute(plugin)
if err != nil {
return nil, err
}
pluginproxy.ApplyRoute(ctx, req, routeName, logAnalyticsRoute, dsInfo, e.cfg)
return req, nil return req, nil
} }
func (e *AzureLogAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin) (*plugins.AppPluginRoute, string, error) { func (e *AzureLogAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, dsInfo datasourceInfo) (*plugins.AppPluginRoute, string, error) {
cloud, err := getAzureCloud(e.cfg, e.dsInfo.JsonData) cloud, err := getAzureCloud(e.cfg, dsInfo)
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }

181
pkg/tsdb/azuremonitor/azure-log-analytics-datasource_test.go
View File

@ -8,8 +8,7 @@ import (
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-cmp/cmp/cmpopts"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -18,36 +17,28 @@ import (
func TestBuildingAzureLogAnalyticsQueries(t *testing.T) { func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
datasource := &AzureLogAnalyticsDatasource{} datasource := &AzureLogAnalyticsDatasource{}
fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local)
timeRange := backend.TimeRange{From: fromStart, To: fromStart.Add(34 * time.Minute)}
timeRange := plugins.DataTimeRange{
From: fmt.Sprintf("%v", fromStart.Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
}
tests := []struct { tests := []struct {
name string name string
queryModel []plugins.DataSubQuery queryModel []backend.DataQuery
timeRange plugins.DataTimeRange
azureLogAnalyticsQueries []*AzureLogAnalyticsQuery azureLogAnalyticsQueries []*AzureLogAnalyticsQuery
Err require.ErrorAssertionFunc Err require.ErrorAssertionFunc
}{ }{
{ {
name: "Query with macros should be interpolated", name: "Query with macros should be interpolated",
timeRange: timeRange, queryModel: []backend.DataQuery{
queryModel: []plugins.DataSubQuery{
{ {
DataSource: &models.DataSource{ JSON: []byte(fmt.Sprintf(`{
JsonData: simplejson.NewFromAny(map[string]interface{}{}),
},
Model: simplejson.NewFromAny(map[string]interface{}{
"queryType": "Azure Log Analytics", "queryType": "Azure Log Analytics",
"azureLogAnalytics": map[string]interface{}{ "azureLogAnalytics": {
"resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace", "resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "query=Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer", "query": "query=Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer",
"resultFormat": timeSeries, "resultFormat": "%s"
}, }
}), }`, timeSeries)),
RefID: "A", RefID: "A",
TimeRange: timeRange,
}, },
}, },
azureLogAnalyticsQueries: []*AzureLogAnalyticsQuery{ azureLogAnalyticsQueries: []*AzureLogAnalyticsQuery{
@ -55,36 +46,34 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
RefID: "A", RefID: "A",
ResultFormat: timeSeries, ResultFormat: timeSeries,
URL: "v1/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace/query", URL: "v1/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace/query",
Model: simplejson.NewFromAny(map[string]interface{}{ JSON: []byte(fmt.Sprintf(`{
"azureLogAnalytics": map[string]interface{}{ "queryType": "Azure Log Analytics",
"azureLogAnalytics": {
"resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "query=Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer", "query": "query=Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer",
"resultFormat": timeSeries, "resultFormat": "%s"
"workspace": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", }
}, }`, timeSeries)),
}), Params: url.Values{"query": {"query=Perf | where ['TimeGenerated'] >= datetime('2018-03-15T13:00:00Z') and ['TimeGenerated'] <= datetime('2018-03-15T13:34:00Z') | where ['Computer'] in ('comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, 34000ms), Computer"}},
Params: url.Values{"query": {"query=Perf | where ['TimeGenerated'] >= datetime('2018-03-15T13:00:00Z') and ['TimeGenerated'] <= datetime('2018-03-15T13:34:00Z') | where ['Computer'] in ('comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, 34000ms), Computer"}}, Target: "query=query%3DPerf+%7C+where+%5B%27TimeGenerated%27%5D+%3E%3D+datetime%28%272018-03-15T13%3A00%3A00Z%27%29+and+%5B%27TimeGenerated%27%5D+%3C%3D+datetime%28%272018-03-15T13%3A34%3A00Z%27%29+%7C+where+%5B%27Computer%27%5D+in+%28%27comp1%27%2C%27comp2%27%29+%7C+summarize+avg%28CounterValue%29+by+bin%28TimeGenerated%2C+34000ms%29%2C+Computer",
Target: "query=query%3DPerf+%7C+where+%5B%27TimeGenerated%27%5D+%3E%3D+datetime%28%272018-03-15T13%3A00%3A00Z%27%29+and+%5B%27TimeGenerated%27%5D+%3C%3D+datetime%28%272018-03-15T13%3A34%3A00Z%27%29+%7C+where+%5B%27Computer%27%5D+in+%28%27comp1%27%2C%27comp2%27%29+%7C+summarize+avg%28CounterValue%29+by+bin%28TimeGenerated%2C+34000ms%29%2C+Computer", TimeRange: timeRange,
}, },
}, },
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "Legacy queries with a workspace GUID should use workspace-centric url", name: "Legacy queries with a workspace GUID should use workspace-centric url",
timeRange: timeRange, queryModel: []backend.DataQuery{
queryModel: []plugins.DataSubQuery{
{ {
DataSource: &models.DataSource{ JSON: []byte(fmt.Sprintf(`{
JsonData: simplejson.NewFromAny(map[string]interface{}{}),
},
Model: simplejson.NewFromAny(map[string]interface{}{
"queryType": "Azure Log Analytics", "queryType": "Azure Log Analytics",
"azureLogAnalytics": map[string]interface{}{ "azureLogAnalytics": {
"workspace": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", "workspace": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
"query": "query=Perf", "query": "query=Perf",
"resultFormat": timeSeries, "resultFormat": "%s"
}, }
}), }`, timeSeries)),
RefID: "A", RefID: "A",
}, },
}, },
@ -93,13 +82,14 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
RefID: "A", RefID: "A",
ResultFormat: timeSeries, ResultFormat: timeSeries,
URL: "v1/workspaces/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/query", URL: "v1/workspaces/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/query",
Model: simplejson.NewFromAny(map[string]interface{}{ JSON: []byte(fmt.Sprintf(`{
"azureLogAnalytics": map[string]interface{}{ "queryType": "Azure Log Analytics",
"azureLogAnalytics": {
"workspace": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", "workspace": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
"query": "query=Perf", "query": "query=Perf",
"resultFormat": timeSeries, "resultFormat": "%s"
}, }
}), }`, timeSeries)),
Params: url.Values{"query": {"query=Perf"}}, Params: url.Values{"query": {"query=Perf"}},
Target: "query=query%3DPerf", Target: "query=query%3DPerf",
}, },
@ -108,21 +98,17 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
}, },
{ {
name: "Legacy workspace queries with a resource URI (from a template variable) should use resource-centric url", name: "Legacy workspace queries with a resource URI (from a template variable) should use resource-centric url",
timeRange: timeRange, queryModel: []backend.DataQuery{
queryModel: []plugins.DataSubQuery{
{ {
DataSource: &models.DataSource{ JSON: []byte(fmt.Sprintf(`{
JsonData: simplejson.NewFromAny(map[string]interface{}{}),
},
Model: simplejson.NewFromAny(map[string]interface{}{
"queryType": "Azure Log Analytics", "queryType": "Azure Log Analytics",
"azureLogAnalytics": map[string]interface{}{ "azureLogAnalytics": {
"workspace": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace", "workspace": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "query=Perf", "query": "query=Perf",
"resultFormat": timeSeries, "resultFormat": "%s"
}, }
}), }`, timeSeries)),
RefID: "A", RefID: "A",
}, },
}, },
@ -131,13 +117,14 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
RefID: "A", RefID: "A",
ResultFormat: timeSeries, ResultFormat: timeSeries,
URL: "v1/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace/query", URL: "v1/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace/query",
Model: simplejson.NewFromAny(map[string]interface{}{ JSON: []byte(fmt.Sprintf(`{
"azureLogAnalytics": map[string]interface{}{ "queryType": "Azure Log Analytics",
"azureLogAnalytics": {
"workspace": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace", "workspace": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "query=Perf", "query": "query=Perf",
"resultFormat": timeSeries, "resultFormat": "%s"
}, }
}), }`, timeSeries)),
Params: url.Values{"query": {"query=Perf"}}, Params: url.Values{"query": {"query=Perf"}},
Target: "query=query%3DPerf", Target: "query=query%3DPerf",
}, },
@ -146,21 +133,17 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
}, },
{ {
name: "Queries with a Resource should use resource-centric url", name: "Queries with a Resource should use resource-centric url",
timeRange: timeRange, queryModel: []backend.DataQuery{
queryModel: []plugins.DataSubQuery{
{ {
DataSource: &models.DataSource{ JSON: []byte(fmt.Sprintf(`{
JsonData: simplejson.NewFromAny(map[string]interface{}{}),
},
Model: simplejson.NewFromAny(map[string]interface{}{
"queryType": "Azure Log Analytics", "queryType": "Azure Log Analytics",
"azureLogAnalytics": map[string]interface{}{ "azureLogAnalytics": {
"resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace", "resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "query=Perf", "query": "query=Perf",
"resultFormat": timeSeries, "resultFormat": "%s"
}, }
}), }`, timeSeries)),
RefID: "A", RefID: "A",
}, },
}, },
@ -169,13 +152,14 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
RefID: "A", RefID: "A",
ResultFormat: timeSeries, ResultFormat: timeSeries,
URL: "v1/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace/query", URL: "v1/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace/query",
Model: simplejson.NewFromAny(map[string]interface{}{ JSON: []byte(fmt.Sprintf(`{
"azureLogAnalytics": map[string]interface{}{ "queryType": "Azure Log Analytics",
"azureLogAnalytics": {
"resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace", "resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "query=Perf", "query": "query=Perf",
"resultFormat": timeSeries, "resultFormat": "%s"
}, }
}), }`, timeSeries)),
Params: url.Values{"query": {"query=Perf"}}, Params: url.Values{"query": {"query=Perf"}},
Target: "query=query%3DPerf", Target: "query=query%3DPerf",
}, },
@ -186,9 +170,9 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
queries, err := datasource.buildQueries(tt.queryModel, tt.timeRange) queries, err := datasource.buildQueries(tt.queryModel, datasourceInfo{})
tt.Err(t, err) tt.Err(t, err)
if diff := cmp.Diff(tt.azureLogAnalyticsQueries, queries, cmpopts.IgnoreUnexported(simplejson.Json{})); diff != "" { if diff := cmp.Diff(tt.azureLogAnalyticsQueries[0], queries[0]); diff != "" {
t.Errorf("Result mismatch (-want +got):\n%s", diff) t.Errorf("Result mismatch (-want +got):\n%s", diff)
} }
}) })
@ -234,6 +218,7 @@ func TestPluginRoutes(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
dsInfo datasourceInfo
datasource *AzureLogAnalyticsDatasource datasource *AzureLogAnalyticsDatasource
expectedProxypass string expectedProxypass string
expectedRouteURL string expectedRouteURL string
@ -241,14 +226,14 @@ func TestPluginRoutes(t *testing.T) {
}{ }{
{ {
name: "plugin proxy route for the Azure public cloud", name: "plugin proxy route for the Azure public cloud",
dsInfo: datasourceInfo{
Settings: azureMonitorSettings{
AzureAuthType: AzureAuthClientSecret,
CloudName: "azuremonitor",
},
},
datasource: &AzureLogAnalyticsDatasource{ datasource: &AzureLogAnalyticsDatasource{
cfg: cfg, cfg: cfg,
dsInfo: &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{
"azureAuthType": AzureAuthClientSecret,
"cloudName": "azuremonitor",
}),
},
}, },
expectedProxypass: "loganalyticsazure", expectedProxypass: "loganalyticsazure",
expectedRouteURL: "https://api.loganalytics.io/", expectedRouteURL: "https://api.loganalytics.io/",
@ -256,14 +241,14 @@ func TestPluginRoutes(t *testing.T) {
}, },
{ {
name: "plugin proxy route for the Azure China cloud", name: "plugin proxy route for the Azure China cloud",
dsInfo: datasourceInfo{
Settings: azureMonitorSettings{
AzureAuthType: AzureAuthClientSecret,
CloudName: "chinaazuremonitor",
},
},
datasource: &AzureLogAnalyticsDatasource{ datasource: &AzureLogAnalyticsDatasource{
cfg: cfg, cfg: cfg,
dsInfo: &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{
"azureAuthType": AzureAuthClientSecret,
"cloudName": "chinaazuremonitor",
}),
},
}, },
expectedProxypass: "chinaloganalyticsazure", expectedProxypass: "chinaloganalyticsazure",
expectedRouteURL: "https://api.loganalytics.azure.cn/", expectedRouteURL: "https://api.loganalytics.azure.cn/",
@ -271,14 +256,14 @@ func TestPluginRoutes(t *testing.T) {
}, },
{ {
name: "plugin proxy route for the Azure Gov cloud", name: "plugin proxy route for the Azure Gov cloud",
dsInfo: datasourceInfo{
Settings: azureMonitorSettings{
AzureAuthType: AzureAuthClientSecret,
CloudName: "govazuremonitor",
},
},
datasource: &AzureLogAnalyticsDatasource{ datasource: &AzureLogAnalyticsDatasource{
cfg: cfg, cfg: cfg,
dsInfo: &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{
"azureAuthType": AzureAuthClientSecret,
"cloudName": "govazuremonitor",
}),
},
}, },
expectedProxypass: "govloganalyticsazure", expectedProxypass: "govloganalyticsazure",
expectedRouteURL: "https://api.loganalytics.us/", expectedRouteURL: "https://api.loganalytics.us/",
@ -288,7 +273,7 @@ func TestPluginRoutes(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
route, proxypass, err := tt.datasource.getPluginRoute(plugin) route, proxypass, err := tt.datasource.getPluginRoute(plugin, tt.dsInfo)
tt.Err(t, err) tt.Err(t, err)
if diff := cmp.Diff(tt.expectedRouteURL, route.URL, cmpopts.EquateNaNs()); diff != "" { if diff := cmp.Diff(tt.expectedRouteURL, route.URL, cmpopts.EquateNaNs()); diff != "" {

124
pkg/tsdb/azuremonitor/azure-resource-graph-datasource.go
View File

@ -2,6 +2,7 @@ package azuremonitor
import ( import (
"bytes" "bytes"
"time"
"context" "context"
"encoding/json" "encoding/json"
@ -15,6 +16,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/pluginproxy" "github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
@ -26,8 +28,6 @@ import (
// AzureResourceGraphDatasource calls the Azure Resource Graph API's // AzureResourceGraphDatasource calls the Azure Resource Graph API's
type AzureResourceGraphDatasource struct { type AzureResourceGraphDatasource struct {
httpClient *http.Client
dsInfo *models.DataSource
pluginManager plugins.Manager pluginManager plugins.Manager
cfg *setting.Cfg cfg *setting.Cfg
} }
@ -38,8 +38,9 @@ type AzureResourceGraphQuery struct {
RefID string RefID string
ResultFormat string ResultFormat string
URL string URL string
Model *simplejson.Json JSON json.RawMessage
InterpolatedQuery string InterpolatedQuery string
TimeRange backend.TimeRange
} }
const argAPIVersion = "2018-09-01-preview" const argAPIVersion = "2018-09-01-preview"
@ -48,37 +49,30 @@ const argQueryProviderName = "/providers/Microsoft.ResourceGraph/resources"
// executeTimeSeriesQuery does the following: // executeTimeSeriesQuery does the following:
// 1. builds the AzureMonitor url and querystring for each query // 1. builds the AzureMonitor url and querystring for each query
// 2. executes each query by calling the Azure Monitor API // 2. executes each query by calling the Azure Monitor API
// 3. parses the responses for each query into the timeseries format // 3. parses the responses for each query into data frames
func (e *AzureResourceGraphDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []plugins.DataSubQuery, func (e *AzureResourceGraphDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo) (*backend.QueryDataResponse, error) {
timeRange plugins.DataTimeRange) (backend.QueryDataResponse, error) { result := &backend.QueryDataResponse{
result := backend.QueryDataResponse{
Responses: map[string]backend.DataResponse{}, Responses: map[string]backend.DataResponse{},
} }
queries, err := e.buildQueries(originalQueries, timeRange) queries, err := e.buildQueries(originalQueries, dsInfo)
if err != nil { if err != nil {
return backend.QueryDataResponse{}, err return nil, err
} }
for _, query := range queries { for _, query := range queries {
result.Responses[query.RefID] = e.executeQuery(ctx, query, timeRange) result.Responses[query.RefID] = e.executeQuery(ctx, query, dsInfo)
} }
return result, nil return result, nil
} }
func (e *AzureResourceGraphDatasource) buildQueries(queries []plugins.DataSubQuery, func (e *AzureResourceGraphDatasource) buildQueries(queries []backend.DataQuery, dsInfo datasourceInfo) ([]*AzureResourceGraphQuery, error) {
timeRange plugins.DataTimeRange) ([]*AzureResourceGraphQuery, error) {
var azureResourceGraphQueries []*AzureResourceGraphQuery var azureResourceGraphQueries []*AzureResourceGraphQuery
for _, query := range queries { for _, query := range queries {
queryBytes, err := query.Model.Encode()
if err != nil {
return nil, fmt.Errorf("failed to re-encode the Azure Resource Graph query into JSON: %w", err)
}
queryJSONModel := argJSONQuery{} queryJSONModel := argJSONQuery{}
err = json.Unmarshal(queryBytes, &queryJSONModel) err := json.Unmarshal(query.JSON, &queryJSONModel)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to decode the Azure Resource Graph query object from JSON: %w", err) return nil, fmt.Errorf("failed to decode the Azure Resource Graph query object from JSON: %w", err)
} }
@ -91,7 +85,7 @@ func (e *AzureResourceGraphDatasource) buildQueries(queries []plugins.DataSubQue
resultFormat = "table" resultFormat = "table"
} }
interpolatedQuery, err := KqlInterpolate(query, timeRange, azureResourceGraphTarget.Query) interpolatedQuery, err := KqlInterpolate(query, dsInfo, azureResourceGraphTarget.Query)
if err != nil { if err != nil {
return nil, err return nil, err
@ -100,23 +94,23 @@ func (e *AzureResourceGraphDatasource) buildQueries(queries []plugins.DataSubQue
azureResourceGraphQueries = append(azureResourceGraphQueries, &AzureResourceGraphQuery{ azureResourceGraphQueries = append(azureResourceGraphQueries, &AzureResourceGraphQuery{
RefID: query.RefID, RefID: query.RefID,
ResultFormat: resultFormat, ResultFormat: resultFormat,
Model: query.Model, JSON: query.JSON,
InterpolatedQuery: interpolatedQuery, InterpolatedQuery: interpolatedQuery,
TimeRange: query.TimeRange,
}) })
} }
return azureResourceGraphQueries, nil return azureResourceGraphQueries, nil
} }
func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *AzureResourceGraphQuery, func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *AzureResourceGraphQuery, dsInfo datasourceInfo) backend.DataResponse {
timeRange plugins.DataTimeRange) backend.DataResponse { dataResponse := backend.DataResponse{}
queryResult := backend.DataResponse{}
params := url.Values{} params := url.Values{}
params.Add("api-version", argAPIVersion) params.Add("api-version", argAPIVersion)
queryResultErrorWithExecuted := func(err error) backend.DataResponse { dataResponseErrorWithExecuted := func(err error) backend.DataResponse {
queryResult = backend.DataResponse{Error: err} dataResponse = backend.DataResponse{Error: err}
frames := data.Frames{ frames := data.Frames{
&data.Frame{ &data.Frame{
RefID: query.RefID, RefID: query.RefID,
@ -125,25 +119,31 @@ func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *
}, },
}, },
} }
queryResult.Frames = frames dataResponse.Frames = frames
return queryResult return dataResponse
}
model, err := simplejson.NewJson(query.JSON)
if err != nil {
dataResponse.Error = err
return dataResponse
} }
reqBody, err := json.Marshal(map[string]interface{}{ reqBody, err := json.Marshal(map[string]interface{}{
"subscriptions": query.Model.Get("subscriptions").MustStringArray(), "subscriptions": model.Get("subscriptions").MustStringArray(),
"query": query.InterpolatedQuery, "query": query.InterpolatedQuery,
}) })
if err != nil { if err != nil {
queryResult.Error = err dataResponse.Error = err
return queryResult return dataResponse
} }
req, err := e.createRequest(ctx, e.dsInfo, reqBody) req, err := e.createRequest(ctx, dsInfo, reqBody)
if err != nil { if err != nil {
queryResult.Error = err dataResponse.Error = err
return queryResult return dataResponse
} }
req.URL.Path = path.Join(req.URL.Path, argQueryProviderName) req.URL.Path = path.Join(req.URL.Path, argQueryProviderName)
@ -151,10 +151,10 @@ func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *
span, ctx := opentracing.StartSpanFromContext(ctx, "azure resource graph query") span, ctx := opentracing.StartSpanFromContext(ctx, "azure resource graph query")
span.SetTag("interpolated_query", query.InterpolatedQuery) span.SetTag("interpolated_query", query.InterpolatedQuery)
span.SetTag("from", timeRange.From) span.SetTag("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond))
span.SetTag("until", timeRange.To) span.SetTag("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond))
span.SetTag("datasource_id", e.dsInfo.Id) span.SetTag("datasource_id", dsInfo.DatasourceID)
span.SetTag("org_id", e.dsInfo.OrgId) span.SetTag("org_id", dsInfo.OrgID)
defer span.Finish() defer span.Finish()
@ -162,35 +162,46 @@ func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *
span.Context(), span.Context(),
opentracing.HTTPHeaders, opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(req.Header)); err != nil { opentracing.HTTPHeadersCarrier(req.Header)); err != nil {
return queryResultErrorWithExecuted(err) return dataResponseErrorWithExecuted(err)
} }
azlog.Debug("AzureResourceGraph", "Request ApiURL", req.URL.String()) azlog.Debug("AzureResourceGraph", "Request ApiURL", req.URL.String())
res, err := ctxhttp.Do(ctx, e.httpClient, req) res, err := ctxhttp.Do(ctx, dsInfo.HTTPClient, req)
if err != nil { if err != nil {
return queryResultErrorWithExecuted(err) return dataResponseErrorWithExecuted(err)
} }
argResponse, err := e.unmarshalResponse(res) argResponse, err := e.unmarshalResponse(res)
if err != nil { if err != nil {
return queryResultErrorWithExecuted(err) return dataResponseErrorWithExecuted(err)
} }
frame, err := ResponseTableToFrame(&argResponse.Data) frame, err := ResponseTableToFrame(&argResponse.Data)
if err != nil { if err != nil {
return queryResultErrorWithExecuted(err) return dataResponseErrorWithExecuted(err)
} }
if frame.Meta == nil { if frame.Meta == nil {
frame.Meta = &data.FrameMeta{} frame.Meta = &data.FrameMeta{}
} }
frame.Meta.ExecutedQueryString = req.URL.RawQuery frame.Meta.ExecutedQueryString = req.URL.RawQuery
queryResult.Frames = data.Frames{frame} dataResponse.Frames = data.Frames{frame}
return queryResult return dataResponse
} }
func (e *AzureResourceGraphDatasource) createRequest(ctx context.Context, dsInfo *models.DataSource, reqBody []byte) (*http.Request, error) { func (e *AzureResourceGraphDatasource) createRequest(ctx context.Context, dsInfo datasourceInfo, reqBody []byte) (*http.Request, error) {
u, err := url.Parse(dsInfo.Url) // find plugin
plugin := e.pluginManager.GetDataSource(dsName)
if plugin == nil {
return nil, errors.New("unable to find datasource plugin Azure Monitor")
}
argRoute, routeName, err := e.getPluginRoute(plugin, dsInfo)
if err != nil {
return nil, err
}
u, err := url.Parse(dsInfo.URL)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -204,24 +215,17 @@ func (e *AzureResourceGraphDatasource) createRequest(ctx context.Context, dsInfo
req.Header.Set("Content-Type", "application/json") req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
// find plugin // TODO: Use backend authentication instead
plugin := e.pluginManager.GetDataSource(dsInfo.Type) pluginproxy.ApplyRoute(ctx, req, routeName, argRoute, &models.DataSource{
if plugin == nil { JsonData: simplejson.NewFromAny(dsInfo.JSONData),
return nil, errors.New("unable to find datasource plugin Azure Monitor") SecureJsonData: securejsondata.GetEncryptedJsonData(dsInfo.DecryptedSecureJSONData),
} }, e.cfg)
argRoute, routeName, err := e.getPluginRoute(plugin)
if err != nil {
return nil, err
}
pluginproxy.ApplyRoute(ctx, req, routeName, argRoute, dsInfo, e.cfg)
return req, nil return req, nil
} }
func (e *AzureResourceGraphDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin) (*plugins.AppPluginRoute, string, error) { func (e *AzureResourceGraphDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, dsInfo datasourceInfo) (*plugins.AppPluginRoute, string, error) {
cloud, err := getAzureCloud(e.cfg, e.dsInfo.JsonData) cloud, err := getAzureCloud(e.cfg, dsInfo)
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }

32
pkg/tsdb/azuremonitor/azure-resource-graph-datasource_test.go
View File

@ -7,8 +7,8 @@ import (
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-cmp/cmp/cmpopts"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -19,7 +19,7 @@ func TestBuildingAzureResourceGraphQueries(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
queryModel []plugins.DataSubQuery queryModel []backend.DataQuery
timeRange plugins.DataTimeRange timeRange plugins.DataTimeRange
azureResourceGraphQueries []*AzureResourceGraphQuery azureResourceGraphQueries []*AzureResourceGraphQuery
Err require.ErrorAssertionFunc Err require.ErrorAssertionFunc
@ -30,18 +30,15 @@ func TestBuildingAzureResourceGraphQueries(t *testing.T) {
From: fmt.Sprintf("%v", fromStart.Unix()*1000), From: fmt.Sprintf("%v", fromStart.Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000), To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
}, },
queryModel: []plugins.DataSubQuery{ queryModel: []backend.DataQuery{
{ {
DataSource: &models.DataSource{ JSON: []byte(`{
JsonData: simplejson.NewFromAny(map[string]interface{}{}),
},
Model: simplejson.NewFromAny(map[string]interface{}{
"queryType": "Azure Resource Graph", "queryType": "Azure Resource Graph",
"azureResourceGraph": map[string]interface{}{ "azureResourceGraph": {
"query": "resources | where $__contains(name,'res1','res2')", "query": "resources | where $__contains(name,'res1','res2')",
"resultFormat": "table", "resultFormat": "table"
}, }
}), }`),
RefID: "A", RefID: "A",
}, },
}, },
@ -50,12 +47,13 @@ func TestBuildingAzureResourceGraphQueries(t *testing.T) {
RefID: "A", RefID: "A",
ResultFormat: "table", ResultFormat: "table",
URL: "", URL: "",
Model: simplejson.NewFromAny(map[string]interface{}{ JSON: []byte(`{
"azureResourceGraph": map[string]interface{}{ "queryType": "Azure Resource Graph",
"azureResourceGraph": {
"query": "resources | where $__contains(name,'res1','res2')", "query": "resources | where $__contains(name,'res1','res2')",
"resultFormat": "table", "resultFormat": "table"
}, }
}), }`),
InterpolatedQuery: "resources | where ['name'] in ('res1','res2')", InterpolatedQuery: "resources | where ['name'] in ('res1','res2')",
}, },
}, },
@ -65,7 +63,7 @@ func TestBuildingAzureResourceGraphQueries(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
queries, err := datasource.buildQueries(tt.queryModel, tt.timeRange) queries, err := datasource.buildQueries(tt.queryModel, datasourceInfo{})
tt.Err(t, err) tt.Err(t, err)
if diff := cmp.Diff(tt.azureResourceGraphQueries, queries, cmpopts.IgnoreUnexported(simplejson.Json{})); diff != "" { if diff := cmp.Diff(tt.azureResourceGraphQueries, queries, cmpopts.IgnoreUnexported(simplejson.Json{})); diff != "" {
t.Errorf("Result mismatch (-want +got):\n%s", diff) t.Errorf("Result mismatch (-want +got):\n%s", diff)

113
pkg/tsdb/azuremonitor/azuremonitor-datasource.go
View File

@ -13,8 +13,11 @@ import (
"strings" "strings"
"time" "time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/pluginproxy" "github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
@ -25,8 +28,6 @@ import (
// AzureMonitorDatasource calls the Azure Monitor API - one of the four API's supported // AzureMonitorDatasource calls the Azure Monitor API - one of the four API's supported
type AzureMonitorDatasource struct { type AzureMonitorDatasource struct {
httpClient *http.Client
dsInfo *models.DataSource
pluginManager plugins.Manager pluginManager plugins.Manager
cfg *setting.Cfg cfg *setting.Cfg
} }
@ -41,58 +42,40 @@ const azureMonitorAPIVersion = "2018-01-01"
// executeTimeSeriesQuery does the following: // executeTimeSeriesQuery does the following:
// 1. build the AzureMonitor url and querystring for each query // 1. build the AzureMonitor url and querystring for each query
// 2. executes each query by calling the Azure Monitor API // 2. executes each query by calling the Azure Monitor API
// 3. parses the responses for each query into the timeseries format // 3. parses the responses for each query into data frames
//nolint: staticcheck // plugins.DataPlugin deprecated func (e *AzureMonitorDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo) (*backend.QueryDataResponse, error) {
func (e *AzureMonitorDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []plugins.DataSubQuery, result := backend.NewQueryDataResponse()
timeRange plugins.DataTimeRange) (plugins.DataResponse, error) {
result := plugins.DataResponse{
Results: map[string]plugins.DataQueryResult{},
}
queries, err := e.buildQueries(originalQueries, timeRange) queries, err := e.buildQueries(originalQueries, dsInfo)
if err != nil { if err != nil {
return plugins.DataResponse{}, err return nil, err
} }
for _, query := range queries { for _, query := range queries {
queryRes, resp, err := e.executeQuery(ctx, query, originalQueries, timeRange) queryRes, resp, err := e.executeQuery(ctx, query, dsInfo)
if err != nil { if err != nil {
return plugins.DataResponse{}, err return nil, err
} }
frames, err := e.parseResponse(resp, query) frames, err := e.parseResponse(resp, query)
if err != nil { if err != nil {
queryRes.Error = err queryRes.Error = err
} else { } else {
queryRes.Dataframes = frames queryRes.Frames = frames
} }
result.Results[query.RefID] = queryRes result.Responses[query.RefID] = queryRes
} }
return result, nil return result, nil
} }
func (e *AzureMonitorDatasource) buildQueries(queries []plugins.DataSubQuery, timeRange plugins.DataTimeRange) ([]*AzureMonitorQuery, error) { func (e *AzureMonitorDatasource) buildQueries(queries []backend.DataQuery, dsInfo datasourceInfo) ([]*AzureMonitorQuery, error) {
azureMonitorQueries := []*AzureMonitorQuery{} azureMonitorQueries := []*AzureMonitorQuery{}
startTime, err := timeRange.ParseFrom()
if err != nil {
return nil, err
}
endTime, err := timeRange.ParseTo()
if err != nil {
return nil, err
}
for _, query := range queries { for _, query := range queries {
var target string var target string
queryBytes, err := query.Model.Encode()
if err != nil {
return nil, fmt.Errorf("failed to re-encode the Azure Monitor query into JSON: %w", err)
}
queryJSONModel := azureMonitorJSONQuery{} queryJSONModel := azureMonitorJSONQuery{}
err = json.Unmarshal(queryBytes, &queryJSONModel) err := json.Unmarshal(query.JSON, &queryJSONModel)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to decode the Azure Monitor query object from JSON: %w", err) return nil, fmt.Errorf("failed to decode the Azure Monitor query object from JSON: %w", err)
} }
@ -106,7 +89,7 @@ func (e *AzureMonitorDatasource) buildQueries(queries []plugins.DataSubQuery, ti
urlComponents["resourceName"] = azJSONModel.ResourceName urlComponents["resourceName"] = azJSONModel.ResourceName
ub := urlBuilder{ ub := urlBuilder{
DefaultSubscription: query.DataSource.JsonData.Get("subscriptionId").MustString(), DefaultSubscription: dsInfo.Settings.SubscriptionId,
Subscription: queryJSONModel.Subscription, Subscription: queryJSONModel.Subscription,
ResourceGroup: queryJSONModel.AzureMonitor.ResourceGroup, ResourceGroup: queryJSONModel.AzureMonitor.ResourceGroup,
MetricDefinition: azJSONModel.MetricDefinition, MetricDefinition: azJSONModel.MetricDefinition,
@ -119,7 +102,7 @@ func (e *AzureMonitorDatasource) buildQueries(queries []plugins.DataSubQuery, ti
timeGrain := azJSONModel.TimeGrain timeGrain := azJSONModel.TimeGrain
timeGrains := azJSONModel.AllowedTimeGrainsMs timeGrains := azJSONModel.AllowedTimeGrainsMs
if timeGrain == "auto" { if timeGrain == "auto" {
timeGrain, err = setAutoTimeGrain(query.IntervalMS, timeGrains) timeGrain, err = setAutoTimeGrain(query.Interval.Milliseconds(), timeGrains)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -127,7 +110,7 @@ func (e *AzureMonitorDatasource) buildQueries(queries []plugins.DataSubQuery, ti
params := url.Values{} params := url.Values{}
params.Add("api-version", azureMonitorAPIVersion) params.Add("api-version", azureMonitorAPIVersion)
params.Add("timespan", fmt.Sprintf("%v/%v", startTime.UTC().Format(time.RFC3339), endTime.UTC().Format(time.RFC3339))) params.Add("timespan", fmt.Sprintf("%v/%v", query.TimeRange.From.UTC().Format(time.RFC3339), query.TimeRange.To.UTC().Format(time.RFC3339)))
params.Add("interval", timeGrain) params.Add("interval", timeGrain)
params.Add("aggregation", azJSONModel.Aggregation) params.Add("aggregation", azJSONModel.Aggregation)
params.Add("metricnames", azJSONModel.MetricName) // MetricName or MetricNames ? params.Add("metricnames", azJSONModel.MetricName) // MetricName or MetricNames ?
@ -168,21 +151,20 @@ func (e *AzureMonitorDatasource) buildQueries(queries []plugins.DataSubQuery, ti
Params: params, Params: params,
RefID: query.RefID, RefID: query.RefID,
Alias: alias, Alias: alias,
TimeRange: query.TimeRange,
}) })
} }
return azureMonitorQueries, nil return azureMonitorQueries, nil
} }
//nolint: staticcheck // plugins.DataPlugin deprecated func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *AzureMonitorQuery, dsInfo datasourceInfo) (backend.DataResponse, AzureMonitorResponse, error) {
func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *AzureMonitorQuery, queries []plugins.DataSubQuery, dataResponse := backend.DataResponse{}
timeRange plugins.DataTimeRange) (plugins.DataQueryResult, AzureMonitorResponse, error) {
queryResult := plugins.DataQueryResult{RefID: query.RefID}
req, err := e.createRequest(ctx, e.dsInfo) req, err := e.createRequest(ctx, dsInfo)
if err != nil { if err != nil {
queryResult.Error = err dataResponse.Error = err
return queryResult, AzureMonitorResponse{}, nil return dataResponse, AzureMonitorResponse{}, nil
} }
req.URL.Path = path.Join(req.URL.Path, query.URL) req.URL.Path = path.Join(req.URL.Path, query.URL)
@ -190,10 +172,10 @@ func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *AzureM
span, ctx := opentracing.StartSpanFromContext(ctx, "azuremonitor query") span, ctx := opentracing.StartSpanFromContext(ctx, "azuremonitor query")
span.SetTag("target", query.Target) span.SetTag("target", query.Target)
span.SetTag("from", timeRange.From) span.SetTag("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond))
span.SetTag("until", timeRange.To) span.SetTag("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond))
span.SetTag("datasource_id", e.dsInfo.Id) span.SetTag("datasource_id", dsInfo.DatasourceID)
span.SetTag("org_id", e.dsInfo.OrgId) span.SetTag("org_id", dsInfo.OrgID)
defer span.Finish() defer span.Finish()
@ -201,16 +183,16 @@ func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *AzureM
span.Context(), span.Context(),
opentracing.HTTPHeaders, opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(req.Header)); err != nil { opentracing.HTTPHeadersCarrier(req.Header)); err != nil {
queryResult.Error = err dataResponse.Error = err
return queryResult, AzureMonitorResponse{}, nil return dataResponse, AzureMonitorResponse{}, nil
} }
azlog.Debug("AzureMonitor", "Request ApiURL", req.URL.String()) azlog.Debug("AzureMonitor", "Request ApiURL", req.URL.String())
azlog.Debug("AzureMonitor", "Target", query.Target) azlog.Debug("AzureMonitor", "Target", query.Target)
res, err := ctxhttp.Do(ctx, e.httpClient, req) res, err := ctxhttp.Do(ctx, dsInfo.HTTPClient, req)
if err != nil { if err != nil {
queryResult.Error = err dataResponse.Error = err
return queryResult, AzureMonitorResponse{}, nil return dataResponse, AzureMonitorResponse{}, nil
} }
defer func() { defer func() {
if err := res.Body.Close(); err != nil { if err := res.Body.Close(); err != nil {
@ -220,28 +202,26 @@ func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *AzureM
data, err := e.unmarshalResponse(res) data, err := e.unmarshalResponse(res)
if err != nil { if err != nil {
queryResult.Error = err dataResponse.Error = err
return queryResult, AzureMonitorResponse{}, nil return dataResponse, AzureMonitorResponse{}, nil
} }
return queryResult, data, nil return dataResponse, data, nil
} }
func (e *AzureMonitorDatasource) createRequest(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) { func (e *AzureMonitorDatasource) createRequest(ctx context.Context, dsInfo datasourceInfo) (*http.Request, error) {
// find plugin // find plugin
plugin := e.pluginManager.GetDataSource(dsInfo.Type) plugin := e.pluginManager.GetDataSource(dsName)
if plugin == nil { if plugin == nil {
return nil, errors.New("unable to find datasource plugin Azure Monitor") return nil, errors.New("unable to find datasource plugin Azure Monitor")
} }
azureMonitorRoute, routeName, err := e.getPluginRoute(plugin) azureMonitorRoute, routeName, err := e.getPluginRoute(plugin, dsInfo)
if err != nil { if err != nil {
return nil, err return nil, err
} }
proxyPass := fmt.Sprintf("%s/subscriptions", routeName) u, err := url.Parse(dsInfo.URL)
u, err := url.Parse(dsInfo.Url)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -255,13 +235,18 @@ func (e *AzureMonitorDatasource) createRequest(ctx context.Context, dsInfo *mode
req.Header.Set("Content-Type", "application/json") req.Header.Set("Content-Type", "application/json")
pluginproxy.ApplyRoute(ctx, req, proxyPass, azureMonitorRoute, dsInfo, e.cfg) // TODO: Use backend authentication instead
proxyPass := fmt.Sprintf("%s/subscriptions", routeName)
pluginproxy.ApplyRoute(ctx, req, proxyPass, azureMonitorRoute, &models.DataSource{
JsonData: simplejson.NewFromAny(dsInfo.JSONData),
SecureJsonData: securejsondata.GetEncryptedJsonData(dsInfo.DecryptedSecureJSONData),
}, e.cfg)
return req, nil return req, nil
} }
func (e *AzureMonitorDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin) (*plugins.AppPluginRoute, string, error) { func (e *AzureMonitorDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, dsInfo datasourceInfo) (*plugins.AppPluginRoute, string, error) {
cloud, err := getAzureCloud(e.cfg, e.dsInfo.JsonData) cloud, err := getAzureCloud(e.cfg, dsInfo)
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }
@ -304,7 +289,7 @@ func (e *AzureMonitorDatasource) unmarshalResponse(res *http.Response) (AzureMon
} }
func (e *AzureMonitorDatasource) parseResponse(amr AzureMonitorResponse, query *AzureMonitorQuery) ( func (e *AzureMonitorDatasource) parseResponse(amr AzureMonitorResponse, query *AzureMonitorQuery) (
plugins.DataFrames, error) { data.Frames, error) {
if len(amr.Value) == 0 { if len(amr.Value) == 0 {
return nil, nil return nil, nil
} }
@ -364,7 +349,7 @@ func (e *AzureMonitorDatasource) parseResponse(amr AzureMonitorResponse, query *
frames = append(frames, frame) frames = append(frames, frame)
} }
return plugins.NewDecodedDataFrames(frames), nil return frames, nil
} }
// formatAzureMonitorLegendKey builds the legend key or timeseries name // formatAzureMonitorLegendKey builds the legend key or timeseries name

66
pkg/tsdb/azuremonitor/azuremonitor-datasource_test.go
View File

@ -11,24 +11,30 @@ import (
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-cmp/cmp/cmpopts"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
ptr "github.com/xorcare/pointer" ptr "github.com/xorcare/pointer"
) )
func TestAzureMonitorBuildQueries(t *testing.T) { func TestAzureMonitorBuildQueries(t *testing.T) {
datasource := &AzureMonitorDatasource{} datasource := &AzureMonitorDatasource{}
dsInfo := datasourceInfo{
Settings: azureMonitorSettings{
SubscriptionId: "default-subscription",
},
}
fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local)
duration, _ := time.ParseDuration("400s")
tests := []struct { tests := []struct {
name string name string
azureMonitorVariedProperties map[string]interface{} azureMonitorVariedProperties map[string]interface{}
azureMonitorQueryTarget string azureMonitorQueryTarget string
expectedInterval string expectedInterval string
queryIntervalMS int64 queryInterval time.Duration
}{ }{
{ {
name: "Parse queries from frontend and build AzureMonitor API queries", name: "Parse queries from frontend and build AzureMonitor API queries",
@ -45,7 +51,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) {
"timeGrain": "auto", "timeGrain": "auto",
"top": "10", "top": "10",
}, },
queryIntervalMS: 400000, queryInterval: duration,
expectedInterval: "PT15M", expectedInterval: "PT15M",
azureMonitorQueryTarget: "aggregation=Average&api-version=2018-01-01&interval=PT15M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z", azureMonitorQueryTarget: "aggregation=Average&api-version=2018-01-01&interval=PT15M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z",
}, },
@ -56,7 +62,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) {
"allowedTimeGrainsMs": []int64{60000, 300000}, "allowedTimeGrainsMs": []int64{60000, 300000},
"top": "10", "top": "10",
}, },
queryIntervalMS: 400000, queryInterval: duration,
expectedInterval: "PT5M", expectedInterval: "PT5M",
azureMonitorQueryTarget: "aggregation=Average&api-version=2018-01-01&interval=PT5M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z", azureMonitorQueryTarget: "aggregation=Average&api-version=2018-01-01&interval=PT5M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z",
}, },
@ -68,7 +74,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) {
"dimensionFilter": "*", "dimensionFilter": "*",
"top": "30", "top": "30",
}, },
queryIntervalMS: 400000, queryInterval: duration,
expectedInterval: "PT1M", expectedInterval: "PT1M",
azureMonitorQueryTarget: "%24filter=blob+eq+%27%2A%27&aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z&top=30", azureMonitorQueryTarget: "%24filter=blob+eq+%27%2A%27&aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z&top=30",
}, },
@ -80,7 +86,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) {
"dimensionFilter": "*", "dimensionFilter": "*",
"top": "10", "top": "10",
}, },
queryIntervalMS: 400000, queryInterval: duration,
expectedInterval: "PT1M", expectedInterval: "PT1M",
azureMonitorQueryTarget: "aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z", azureMonitorQueryTarget: "aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z",
}, },
@ -91,7 +97,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) {
"dimensionFilters": []azureMonitorDimensionFilter{{"blob", "eq", "*"}}, "dimensionFilters": []azureMonitorDimensionFilter{{"blob", "eq", "*"}},
"top": "30", "top": "30",
}, },
queryIntervalMS: 400000, queryInterval: duration,
expectedInterval: "PT1M", expectedInterval: "PT1M",
azureMonitorQueryTarget: "%24filter=blob+eq+%27%2A%27&aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z&top=30", azureMonitorQueryTarget: "%24filter=blob+eq+%27%2A%27&aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z&top=30",
}, },
@ -102,7 +108,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) {
"dimensionFilters": []azureMonitorDimensionFilter{{"blob", "eq", "*"}, {"tier", "eq", "*"}}, "dimensionFilters": []azureMonitorDimensionFilter{{"blob", "eq", "*"}, {"tier", "eq", "*"}},
"top": "30", "top": "30",
}, },
queryIntervalMS: 400000, queryInterval: duration,
expectedInterval: "PT1M", expectedInterval: "PT1M",
azureMonitorQueryTarget: "%24filter=blob+eq+%27%2A%27+and+tier+eq+%27%2A%27&aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z&top=30", azureMonitorQueryTarget: "%24filter=blob+eq+%27%2A%27+and+tier+eq+%27%2A%27&aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z&top=30",
}, },
@ -125,25 +131,18 @@ func TestAzureMonitorBuildQueries(t *testing.T) {
for k, v := range commonAzureModelProps { for k, v := range commonAzureModelProps {
tt.azureMonitorVariedProperties[k] = v tt.azureMonitorVariedProperties[k] = v
} }
tsdbQuery := plugins.DataQuery{ azureMonitorJSON, _ := json.Marshal(tt.azureMonitorVariedProperties)
TimeRange: &plugins.DataTimeRange{ tsdbQuery := []backend.DataQuery{
From: fmt.Sprintf("%v", fromStart.Unix()*1000), {
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000), JSON: []byte(fmt.Sprintf(`{
},
Queries: []plugins.DataSubQuery{
{
DataSource: &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{
"subscriptionId": "default-subscription",
}),
},
Model: simplejson.NewFromAny(map[string]interface{}{
"subscription": "12345678-aaaa-bbbb-cccc-123456789abc", "subscription": "12345678-aaaa-bbbb-cccc-123456789abc",
"azureMonitor": tt.azureMonitorVariedProperties, "azureMonitor": %s
}, }`, string(azureMonitorJSON))),
), RefID: "A",
RefID: "A", Interval: tt.queryInterval,
IntervalMS: tt.queryIntervalMS, TimeRange: backend.TimeRange{
From: fromStart,
To: fromStart.Add(34 * time.Minute),
}, },
}, },
} }
@ -159,9 +158,13 @@ func TestAzureMonitorBuildQueries(t *testing.T) {
Target: tt.azureMonitorQueryTarget, Target: tt.azureMonitorQueryTarget,
RefID: "A", RefID: "A",
Alias: "testalias", Alias: "testalias",
TimeRange: backend.TimeRange{
From: fromStart,
To: fromStart.Add(34 * time.Minute),
},
} }
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange) queries, err := datasource.buildQueries(tsdbQuery, dsInfo)
require.NoError(t, err) require.NoError(t, err)
if diff := cmp.Diff(azureMonitorQuery, queries[0], cmpopts.IgnoreUnexported(simplejson.Json{}), cmpopts.IgnoreFields(AzureMonitorQuery{}, "Params")); diff != "" { if diff := cmp.Diff(azureMonitorQuery, queries[0], cmpopts.IgnoreUnexported(simplejson.Json{}), cmpopts.IgnoreFields(AzureMonitorQuery{}, "Params")); diff != "" {
t.Errorf("Result mismatch (-want +got):\n%s", diff) t.Errorf("Result mismatch (-want +got):\n%s", diff)
@ -433,16 +436,11 @@ func TestAzureMonitorParseResponse(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
azData := loadTestFile(t, "azuremonitor/"+tt.responseFile) azData := loadTestFile(t, "azuremonitor/"+tt.responseFile)
//nolint: staticcheck // plugins.DataPlugin deprecated
res := plugins.DataQueryResult{Meta: simplejson.New(), RefID: "A"}
require.NotNil(t, res)
dframes, err := datasource.parseResponse(azData, tt.mockQuery) dframes, err := datasource.parseResponse(azData, tt.mockQuery)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, dframes) require.NotNil(t, dframes)
frames, err := dframes.Decoded() if diff := cmp.Diff(tt.expectedFrames, dframes, data.FrameTestCompareOptions()...); diff != "" {
require.NoError(t, err)
if diff := cmp.Diff(tt.expectedFrames, frames, data.FrameTestCompareOptions()...); diff != "" {
t.Errorf("Result mismatch (-want +got):\n%s", diff) t.Errorf("Result mismatch (-want +got):\n%s", diff)
} }
}) })

254
pkg/tsdb/azuremonitor/azuremonitor.go
View File

@ -2,19 +2,27 @@ package azuremonitor
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"net/http" "net/http"
"regexp" "regexp"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
"github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
"github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
) )
const timeSeries = "time_series" const (
timeSeries = "time_series"
dsName = "grafana-azure-monitor-datasource"
)
var ( var (
azlog = log.New("tsdb.azuremonitor") azlog = log.New("tsdb.azuremonitor")
@ -30,147 +38,111 @@ func init() {
} }
type Service struct { type Service struct {
PluginManager plugins.Manager `inject:""` PluginManager plugins.Manager `inject:""`
HTTPClientProvider httpclient.Provider `inject:""` HTTPClientProvider httpclient.Provider `inject:""`
Cfg *setting.Cfg `inject:""` Cfg *setting.Cfg `inject:""`
BackendPluginManager backendplugin.Manager `inject:""`
}
type azureMonitorSettings struct {
AppInsightsAppId string `json:"appInsightsAppId"`
AzureLogAnalyticsSameAs bool `json:"azureLogAnalyticsSameAs"`
ClientId string `json:"clientId"`
CloudName string `json:"cloudName"`
LogAnalyticsClientId string `json:"logAnalyticsClientId"`
LogAnalyticsDefaultWorkspace string `json:"logAnalyticsDefaultWorkspace"`
LogAnalyticsSubscriptionId string `json:"logAnalyticsSubscriptionId"`
LogAnalyticsTenantId string `json:"logAnalyticsTenantId"`
SubscriptionId string `json:"subscriptionId"`
TenantId string `json:"tenantId"`
AzureAuthType string `json:"azureAuthType,omitempty"`
}
type datasourceInfo struct {
Settings azureMonitorSettings
HTTPClient *http.Client
URL string
JSONData map[string]interface{}
DecryptedSecureJSONData map[string]string
DatasourceID int64
OrgID int64
}
func NewInstanceSettings(httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc {
return func(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
opts, err := settings.HTTPClientOptions()
if err != nil {
return nil, err
}
client, err := httpClientProvider.New(opts)
if err != nil {
return nil, err
}
jsonData := map[string]interface{}{}
err = json.Unmarshal(settings.JSONData, &jsonData)
if err != nil {
return nil, fmt.Errorf("error reading settings: %w", err)
}
azMonitorSettings := azureMonitorSettings{}
err = json.Unmarshal(settings.JSONData, &azMonitorSettings)
if err != nil {
return nil, fmt.Errorf("error reading settings: %w", err)
}
model := datasourceInfo{
Settings: azMonitorSettings,
HTTPClient: client,
URL: settings.URL,
JSONData: jsonData,
DecryptedSecureJSONData: settings.DecryptedSecureJSONData,
DatasourceID: settings.ID,
}
return model, nil
}
}
type azDatasourceExecutor interface {
executeTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo datasourceInfo) (*backend.QueryDataResponse, error)
}
func newExecutor(im instancemgmt.InstanceManager, pm plugins.Manager, httpC httpclient.Provider, cfg *setting.Cfg) *datasource.QueryTypeMux {
mux := datasource.NewQueryTypeMux()
executors := map[string]azDatasourceExecutor{
"Azure Monitor": &AzureMonitorDatasource{pm, cfg},
"Application Insights": &ApplicationInsightsDatasource{pm, cfg},
"Azure Log Analytics": &AzureLogAnalyticsDatasource{pm, cfg},
"Insights Analytics": &InsightsAnalyticsDatasource{pm, cfg},
"Azure Resource Graph": &AzureResourceGraphDatasource{pm, cfg},
}
for dsType := range executors {
// Make a copy of the string to keep the reference after the iterator
dst := dsType
mux.HandleFunc(dsType, func(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
i, err := im.Get(req.PluginContext)
if err != nil {
return nil, err
}
dsInfo := i.(datasourceInfo)
dsInfo.OrgID = req.PluginContext.OrgID
ds := executors[dst]
return ds.executeTimeSeriesQuery(ctx, req.Queries, dsInfo)
})
}
return mux
} }
func (s *Service) Init() error { func (s *Service) Init() error {
im := datasource.NewInstanceManager(NewInstanceSettings(s.HTTPClientProvider))
factory := coreplugin.New(backend.ServeOpts{
QueryDataHandler: newExecutor(im, s.PluginManager, s.HTTPClientProvider, s.Cfg),
})
if err := s.BackendPluginManager.Register(dsName, factory); err != nil {
azlog.Error("Failed to register plugin", "error", err)
}
return nil return nil
} }
// AzureMonitorExecutor executes queries for the Azure Monitor datasource - all four services
type AzureMonitorExecutor struct {
httpClient *http.Client
dsInfo *models.DataSource
pluginManager plugins.Manager
cfg *setting.Cfg
}
// NewAzureMonitorExecutor initializes a http client
//nolint: staticcheck // plugins.DataPlugin deprecated
func (s *Service) NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
httpClient, err := dsInfo.GetHTTPClient(s.HTTPClientProvider)
if err != nil {
return nil, err
}
return &AzureMonitorExecutor{
httpClient: httpClient,
dsInfo: dsInfo,
pluginManager: s.PluginManager,
cfg: s.Cfg,
}, nil
}
// Query takes in the frontend queries, parses them into the query format
// expected by chosen Azure Monitor service (Azure Monitor, App Insights etc.)
// executes the queries against the API and parses the response into
// the right format
//nolint: staticcheck // plugins.DataPlugin deprecated
func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSource,
tsdbQuery plugins.DataQuery) (plugins.DataResponse, error) {
var err error
var azureMonitorQueries []plugins.DataSubQuery
var applicationInsightsQueries []plugins.DataSubQuery
var azureLogAnalyticsQueries []plugins.DataSubQuery
var insightsAnalyticsQueries []plugins.DataSubQuery
var azureResourceGraphQueries []plugins.DataSubQuery
for _, query := range tsdbQuery.Queries {
queryType := query.Model.Get("queryType").MustString("")
switch queryType {
case "Azure Monitor":
azureMonitorQueries = append(azureMonitorQueries, query)
case "Application Insights":
applicationInsightsQueries = append(applicationInsightsQueries, query)
case "Azure Log Analytics":
azureLogAnalyticsQueries = append(azureLogAnalyticsQueries, query)
case "Insights Analytics":
insightsAnalyticsQueries = append(insightsAnalyticsQueries, query)
case "Azure Resource Graph":
azureResourceGraphQueries = append(azureResourceGraphQueries, query)
default:
return plugins.DataResponse{}, fmt.Errorf("alerting not supported for %q", queryType)
}
}
azDatasource := &AzureMonitorDatasource{
httpClient: e.httpClient,
dsInfo: e.dsInfo,
pluginManager: e.pluginManager,
cfg: e.cfg,
}
aiDatasource := &ApplicationInsightsDatasource{
httpClient: e.httpClient,
dsInfo: e.dsInfo,
pluginManager: e.pluginManager,
cfg: e.cfg,
}
alaDatasource := &AzureLogAnalyticsDatasource{
httpClient: e.httpClient,
dsInfo: e.dsInfo,
pluginManager: e.pluginManager,
cfg: e.cfg,
}
iaDatasource := &InsightsAnalyticsDatasource{
httpClient: e.httpClient,
dsInfo: e.dsInfo,
pluginManager: e.pluginManager,
cfg: e.cfg,
}
argDatasource := &AzureResourceGraphDatasource{
httpClient: e.httpClient,
dsInfo: e.dsInfo,
pluginManager: e.pluginManager,
}
azResult, err := azDatasource.executeTimeSeriesQuery(ctx, azureMonitorQueries, *tsdbQuery.TimeRange)
if err != nil {
return plugins.DataResponse{}, err
}
aiResult, err := aiDatasource.executeTimeSeriesQuery(ctx, applicationInsightsQueries, *tsdbQuery.TimeRange)
if err != nil {
return plugins.DataResponse{}, err
}
alaResult, err := alaDatasource.executeTimeSeriesQuery(ctx, azureLogAnalyticsQueries, *tsdbQuery.TimeRange)
if err != nil {
return plugins.DataResponse{}, err
}
iaResult, err := iaDatasource.executeTimeSeriesQuery(ctx, insightsAnalyticsQueries, *tsdbQuery.TimeRange)
if err != nil {
return plugins.DataResponse{}, err
}
argResult, err := argDatasource.executeTimeSeriesQuery(ctx, azureResourceGraphQueries, *tsdbQuery.TimeRange)
if err != nil {
return plugins.DataResponse{}, err
}
for k, v := range aiResult.Results {
azResult.Results[k] = v
}
for k, v := range alaResult.Results {
azResult.Results[k] = v
}
for k, v := range iaResult.Results {
azResult.Results[k] = v
}
for k, v := range argResult.Responses {
azResult.Results[k] = plugins.DataQueryResult{Error: v.Error, Dataframes: plugins.NewDecodedDataFrames(v.Frames)}
}
return azResult, nil
}

19
pkg/tsdb/azuremonitor/credentials.go
View File

@ -3,7 +3,6 @@ package azuremonitor
import ( import (
"fmt" "fmt"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
) )
@ -20,12 +19,12 @@ const (
azureMonitorGermany = "germanyazuremonitor" azureMonitorGermany = "germanyazuremonitor"
) )
func getAuthType(cfg *setting.Cfg, pluginData *simplejson.Json) string { func getAuthType(cfg *setting.Cfg, dsInfo datasourceInfo) string {
if authType := pluginData.Get("azureAuthType").MustString(); authType != "" { if dsInfo.Settings.AzureAuthType != "" {
return authType return dsInfo.Settings.AzureAuthType
} else { } else {
tenantId := pluginData.Get("tenantId").MustString() tenantId := dsInfo.Settings.TenantId
clientId := pluginData.Get("clientId").MustString() clientId := dsInfo.Settings.ClientId
// If authentication type isn't explicitly specified and datasource has client credentials, // If authentication type isn't explicitly specified and datasource has client credentials,
// then this is existing datasource which is configured for app registration (client secret) // then this is existing datasource which is configured for app registration (client secret)
@ -59,15 +58,15 @@ func getDefaultAzureCloud(cfg *setting.Cfg) (string, error) {
} }
} }
func getAzureCloud(cfg *setting.Cfg, pluginData *simplejson.Json) (string, error) { func getAzureCloud(cfg *setting.Cfg, dsInfo datasourceInfo) (string, error) {
authType := getAuthType(cfg, pluginData) authType := getAuthType(cfg, dsInfo)
switch authType { switch authType {
case AzureAuthManagedIdentity: case AzureAuthManagedIdentity:
// In case of managed identity, the cloud is always same as where Grafana is hosted // In case of managed identity, the cloud is always same as where Grafana is hosted
return getDefaultAzureCloud(cfg) return getDefaultAzureCloud(cfg)
case AzureAuthClientSecret: case AzureAuthClientSecret:
if cloud := pluginData.Get("cloudName").MustString(); cloud != "" { if dsInfo.Settings.CloudName != "" {
return cloud, nil return dsInfo.Settings.CloudName, nil
} else { } else {
return getDefaultAzureCloud(cfg) return getDefaultAzureCloud(cfg)
} }

90
pkg/tsdb/azuremonitor/insights-analytics-datasource.go
View File

@ -11,8 +11,11 @@ import (
"net/url" "net/url"
"path" "path"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/pluginproxy" "github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
@ -22,8 +25,6 @@ import (
) )
type InsightsAnalyticsDatasource struct { type InsightsAnalyticsDatasource struct {
httpClient *http.Client
dsInfo *models.DataSource
pluginManager plugins.Manager pluginManager plugins.Manager
cfg *setting.Cfg cfg *setting.Cfg
} }
@ -40,38 +41,29 @@ type InsightsAnalyticsQuery struct {
Target string Target string
} }
//nolint: staticcheck // plugins.DataPlugin deprecated
func (e *InsightsAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context, func (e *InsightsAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context,
originalQueries []plugins.DataSubQuery, timeRange plugins.DataTimeRange) (plugins.DataResponse, error) { originalQueries []backend.DataQuery, dsInfo datasourceInfo) (*backend.QueryDataResponse, error) {
result := plugins.DataResponse{ result := backend.NewQueryDataResponse()
Results: map[string]plugins.DataQueryResult{},
}
queries, err := e.buildQueries(originalQueries, timeRange) queries, err := e.buildQueries(originalQueries, dsInfo)
if err != nil { if err != nil {
return plugins.DataResponse{}, err return nil, err
} }
for _, query := range queries { for _, query := range queries {
result.Results[query.RefID] = e.executeQuery(ctx, query) result.Responses[query.RefID] = e.executeQuery(ctx, query, dsInfo)
} }
return result, nil return result, nil
} }
func (e *InsightsAnalyticsDatasource) buildQueries(queries []plugins.DataSubQuery, func (e *InsightsAnalyticsDatasource) buildQueries(queries []backend.DataQuery, dsInfo datasourceInfo) ([]*InsightsAnalyticsQuery, error) {
timeRange plugins.DataTimeRange) ([]*InsightsAnalyticsQuery, error) {
iaQueries := []*InsightsAnalyticsQuery{} iaQueries := []*InsightsAnalyticsQuery{}
for _, query := range queries { for _, query := range queries {
queryBytes, err := query.Model.Encode()
if err != nil {
return nil, fmt.Errorf("failed to re-encode the Azure Application Insights Analytics query into JSON: %w", err)
}
qm := InsightsAnalyticsQuery{} qm := InsightsAnalyticsQuery{}
queryJSONModel := insightsAnalyticsJSONQuery{} queryJSONModel := insightsAnalyticsJSONQuery{}
err = json.Unmarshal(queryBytes, &queryJSONModel) err := json.Unmarshal(query.JSON, &queryJSONModel)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to decode the Azure Application Insights Analytics query object from JSON: %w", err) return nil, fmt.Errorf("failed to decode the Azure Application Insights Analytics query object from JSON: %w", err)
} }
@ -84,7 +76,7 @@ func (e *InsightsAnalyticsDatasource) buildQueries(queries []plugins.DataSubQuer
return nil, fmt.Errorf("query is missing query string property") return nil, fmt.Errorf("query is missing query string property")
} }
qm.InterpolatedQuery, err = KqlInterpolate(query, timeRange, qm.RawQuery) qm.InterpolatedQuery, err = KqlInterpolate(query, dsInfo, qm.RawQuery)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -98,26 +90,25 @@ func (e *InsightsAnalyticsDatasource) buildQueries(queries []plugins.DataSubQuer
return iaQueries, nil return iaQueries, nil
} }
//nolint: staticcheck // plugins.DataPlugin deprecated func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *InsightsAnalyticsQuery, dsInfo datasourceInfo) backend.DataResponse {
func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *InsightsAnalyticsQuery) plugins.DataQueryResult { dataResponse := backend.DataResponse{}
queryResult := plugins.DataQueryResult{RefID: query.RefID}
queryResultError := func(err error) plugins.DataQueryResult { dataResponseError := func(err error) backend.DataResponse {
queryResult.Error = err dataResponse.Error = err
return queryResult return dataResponse
} }
req, err := e.createRequest(ctx, e.dsInfo) req, err := e.createRequest(ctx, dsInfo)
if err != nil { if err != nil {
return queryResultError(err) return dataResponseError(err)
} }
req.URL.Path = path.Join(req.URL.Path, "query") req.URL.Path = path.Join(req.URL.Path, "query")
req.URL.RawQuery = query.Params.Encode() req.URL.RawQuery = query.Params.Encode()
span, ctx := opentracing.StartSpanFromContext(ctx, "application insights analytics query") span, ctx := opentracing.StartSpanFromContext(ctx, "application insights analytics query")
span.SetTag("target", query.Target) span.SetTag("target", query.Target)
span.SetTag("datasource_id", e.dsInfo.Id) span.SetTag("datasource_id", dsInfo.DatasourceID)
span.SetTag("org_id", e.dsInfo.OrgId) span.SetTag("org_id", dsInfo.OrgID)
defer span.Finish() defer span.Finish()
@ -131,14 +122,14 @@ func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *I
} }
azlog.Debug("ApplicationInsights", "Request URL", req.URL.String()) azlog.Debug("ApplicationInsights", "Request URL", req.URL.String())
res, err := ctxhttp.Do(ctx, e.httpClient, req) res, err := ctxhttp.Do(ctx, dsInfo.HTTPClient, req)
if err != nil { if err != nil {
return queryResultError(err) return dataResponseError(err)
} }
body, err := ioutil.ReadAll(res.Body) body, err := ioutil.ReadAll(res.Body)
if err != nil { if err != nil {
return queryResultError(err) return dataResponseError(err)
} }
defer func() { defer func() {
if err := res.Body.Close(); err != nil { if err := res.Body.Close(); err != nil {
@ -148,24 +139,24 @@ func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *I
if res.StatusCode/100 != 2 { if res.StatusCode/100 != 2 {
azlog.Debug("Request failed", "status", res.Status, "body", string(body)) azlog.Debug("Request failed", "status", res.Status, "body", string(body))
return queryResultError(fmt.Errorf("request failed, status: %s, body: %s", res.Status, body)) return dataResponseError(fmt.Errorf("request failed, status: %s, body: %s", res.Status, body))
} }
var logResponse AzureLogAnalyticsResponse var logResponse AzureLogAnalyticsResponse
d := json.NewDecoder(bytes.NewReader(body)) d := json.NewDecoder(bytes.NewReader(body))
d.UseNumber() d.UseNumber()
err = d.Decode(&logResponse) err = d.Decode(&logResponse)
if err != nil { if err != nil {
return queryResultError(err) return dataResponseError(err)
} }
t, err := logResponse.GetPrimaryResultTable() t, err := logResponse.GetPrimaryResultTable()
if err != nil { if err != nil {
return queryResultError(err) return dataResponseError(err)
} }
frame, err := ResponseTableToFrame(t) frame, err := ResponseTableToFrame(t)
if err != nil { if err != nil {
return queryResultError(err) return dataResponseError(err)
} }
if query.ResultFormat == timeSeries { if query.ResultFormat == timeSeries {
@ -182,28 +173,26 @@ func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *I
} }
} }
} }
frames := data.Frames{frame} dataResponse.Frames = data.Frames{frame}
queryResult.Dataframes = plugins.NewDecodedDataFrames(frames)
return queryResult return dataResponse
} }
func (e *InsightsAnalyticsDatasource) createRequest(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) { func (e *InsightsAnalyticsDatasource) createRequest(ctx context.Context, dsInfo datasourceInfo) (*http.Request, error) {
// find plugin // find plugin
plugin := e.pluginManager.GetDataSource(dsInfo.Type) plugin := e.pluginManager.GetDataSource(dsName)
if plugin == nil { if plugin == nil {
return nil, errors.New("unable to find datasource plugin Azure Application Insights") return nil, errors.New("unable to find datasource plugin Azure Application Insights")
} }
appInsightsRoute, routeName, err := e.getPluginRoute(plugin) appInsightsRoute, routeName, err := e.getPluginRoute(plugin, dsInfo)
if err != nil { if err != nil {
return nil, err return nil, err
} }
appInsightsAppID := dsInfo.JsonData.Get("appInsightsAppId").MustString() appInsightsAppID := dsInfo.Settings.AppInsightsAppId
proxyPass := fmt.Sprintf("%s/v1/apps/%s", routeName, appInsightsAppID)
u, err := url.Parse(dsInfo.Url) u, err := url.Parse(dsInfo.URL)
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to parse url for Application Insights Analytics datasource: %w", err) return nil, fmt.Errorf("unable to parse url for Application Insights Analytics datasource: %w", err)
} }
@ -215,13 +204,18 @@ func (e *InsightsAnalyticsDatasource) createRequest(ctx context.Context, dsInfo
return nil, errutil.Wrap("Failed to create request", err) return nil, errutil.Wrap("Failed to create request", err)
} }
pluginproxy.ApplyRoute(ctx, req, proxyPass, appInsightsRoute, dsInfo, e.cfg) // TODO: Use backend authentication instead
proxyPass := fmt.Sprintf("%s/v1/apps/%s", routeName, appInsightsAppID)
pluginproxy.ApplyRoute(ctx, req, proxyPass, appInsightsRoute, &models.DataSource{
JsonData: simplejson.NewFromAny(dsInfo.JSONData),
SecureJsonData: securejsondata.GetEncryptedJsonData(dsInfo.DecryptedSecureJSONData),
}, e.cfg)
return req, nil return req, nil
} }
func (e *InsightsAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin) (*plugins.AppPluginRoute, string, error) { func (e *InsightsAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, dsInfo datasourceInfo) (*plugins.AppPluginRoute, string, error) {
cloud, err := getAzureCloud(e.cfg, e.dsInfo.JsonData) cloud, err := getAzureCloud(e.cfg, dsInfo)
if err != nil { if err != nil {
return nil, "", err return nil, "", err
} }

49
pkg/tsdb/azuremonitor/macros.go
View File

@ -6,7 +6,9 @@ import (
"strings" "strings"
"time" "time"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb/interval" "github.com/grafana/grafana/pkg/tsdb/interval"
) )
@ -15,8 +17,8 @@ const sExpr = `\$` + rsIdentifier + `(?:\(([^\)]*)\))?`
const escapeMultiExpr = `\$__escapeMulti\(('.*')\)` const escapeMultiExpr = `\$__escapeMulti\(('.*')\)`
type kqlMacroEngine struct { type kqlMacroEngine struct {
timeRange plugins.DataTimeRange timeRange backend.TimeRange
query plugins.DataSubQuery query backend.DataQuery
} }
// Macros: // Macros:
@ -29,18 +31,18 @@ type kqlMacroEngine struct {
// - $__escapeMulti('\\vm\eth0\Total','\\vm\eth2\Total') -> @'\\vm\eth0\Total',@'\\vm\eth2\Total' // - $__escapeMulti('\\vm\eth0\Total','\\vm\eth2\Total') -> @'\\vm\eth0\Total',@'\\vm\eth2\Total'
// KqlInterpolate interpolates macros for Kusto Query Language (KQL) queries // KqlInterpolate interpolates macros for Kusto Query Language (KQL) queries
func KqlInterpolate(query plugins.DataSubQuery, timeRange plugins.DataTimeRange, kql string, defaultTimeField ...string) (string, error) { func KqlInterpolate(query backend.DataQuery, dsInfo datasourceInfo, kql string, defaultTimeField ...string) (string, error) {
engine := kqlMacroEngine{} engine := kqlMacroEngine{}
defaultTimeFieldForAllDatasources := "timestamp" defaultTimeFieldForAllDatasources := "timestamp"
if len(defaultTimeField) > 0 { if len(defaultTimeField) > 0 {
defaultTimeFieldForAllDatasources = defaultTimeField[0] defaultTimeFieldForAllDatasources = defaultTimeField[0]
} }
return engine.Interpolate(query, timeRange, kql, defaultTimeFieldForAllDatasources) return engine.Interpolate(query, dsInfo, kql, defaultTimeFieldForAllDatasources)
} }
func (m *kqlMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plugins.DataTimeRange, kql string, defaultTimeField string) (string, error) { func (m *kqlMacroEngine) Interpolate(query backend.DataQuery, dsInfo datasourceInfo, kql string, defaultTimeField string) (string, error) {
m.timeRange = timeRange m.timeRange = query.TimeRange
m.query = query m.query = query
rExp, _ := regexp.Compile(sExpr) rExp, _ := regexp.Compile(sExpr)
escapeMultiRegex, _ := regexp.Compile(escapeMultiExpr) escapeMultiRegex, _ := regexp.Compile(escapeMultiExpr)
@ -69,7 +71,7 @@ func (m *kqlMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plugi
for i, arg := range args { for i, arg := range args {
args[i] = strings.Trim(arg, " ") args[i] = strings.Trim(arg, " ")
} }
res, err := m.evaluateMacro(groups[1], defaultTimeField, args) res, err := m.evaluateMacro(groups[1], defaultTimeField, args, dsInfo)
if err != nil && macroError == nil { if err != nil && macroError == nil {
macroError = err macroError = err
return "macro_error()" return "macro_error()"
@ -84,7 +86,7 @@ func (m *kqlMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plugi
return kql, nil return kql, nil
} }
func (m *kqlMacroEngine) evaluateMacro(name string, defaultTimeField string, args []string) (string, error) { func (m *kqlMacroEngine) evaluateMacro(name string, defaultTimeField string, args []string, dsInfo datasourceInfo) (string, error) {
switch name { switch name {
case "timeFilter": case "timeFilter":
timeColumn := defaultTimeField timeColumn := defaultTimeField
@ -92,27 +94,34 @@ func (m *kqlMacroEngine) evaluateMacro(name string, defaultTimeField string, arg
timeColumn = args[0] timeColumn = args[0]
} }
return fmt.Sprintf("['%s'] >= datetime('%s') and ['%s'] <= datetime('%s')", timeColumn, return fmt.Sprintf("['%s'] >= datetime('%s') and ['%s'] <= datetime('%s')", timeColumn,
m.timeRange.GetFromAsTimeUTC().Format(time.RFC3339), timeColumn, m.timeRange.From.UTC().Format(time.RFC3339), timeColumn,
m.timeRange.GetToAsTimeUTC().Format(time.RFC3339)), nil m.timeRange.To.UTC().Format(time.RFC3339)), nil
case "timeFrom", "__from": case "timeFrom", "__from":
return fmt.Sprintf("datetime('%s')", m.timeRange.GetFromAsTimeUTC().Format(time.RFC3339)), nil return fmt.Sprintf("datetime('%s')", m.timeRange.From.UTC().Format(time.RFC3339)), nil
case "timeTo", "__to": case "timeTo", "__to":
return fmt.Sprintf("datetime('%s')", m.timeRange.GetToAsTimeUTC().Format(time.RFC3339)), nil return fmt.Sprintf("datetime('%s')", m.timeRange.To.UTC().Format(time.RFC3339)), nil
case "interval": case "interval":
var it time.Duration var it time.Duration
if m.query.IntervalMS == 0 { if m.query.Interval.Milliseconds() == 0 {
to := m.timeRange.MustGetTo().UnixNano() to := m.timeRange.To.UnixNano()
from := m.timeRange.MustGetFrom().UnixNano() from := m.timeRange.From.UnixNano()
// default to "100 datapoints" if nothing in the query is more specific // default to "100 datapoints" if nothing in the query is more specific
defaultInterval := time.Duration((to - from) / 60) defaultInterval := time.Duration((to - from) / 60)
var err error model, err := simplejson.NewJson(m.query.JSON)
it, err = interval.GetIntervalFrom(m.query.DataSource, m.query.Model, defaultInterval)
if err != nil { if err != nil {
azlog.Warn("Unable to get interval from query", "datasource", m.query.DataSource, "model", m.query.Model) azlog.Warn("Unable to parse model from query", "JSON", m.query.JSON)
it = defaultInterval it = defaultInterval
} else {
it, err = interval.GetIntervalFrom(&models.DataSource{
JsonData: simplejson.NewFromAny(dsInfo.JSONData),
}, model, defaultInterval)
if err != nil {
azlog.Warn("Unable to get interval from query", "model", model)
it = defaultInterval
}
} }
} else { } else {
it = time.Millisecond * time.Duration(m.query.IntervalMS) it = time.Millisecond * time.Duration(m.query.Interval.Milliseconds())
} }
return fmt.Sprintf("%dms", int(it/time.Millisecond)), nil return fmt.Sprintf("%dms", int(it/time.Millisecond)), nil
case "contains": case "contains":

80
pkg/tsdb/azuremonitor/macros_test.go
View File

@ -1,137 +1,121 @@
package azuremonitor package azuremonitor
import ( import (
"fmt"
"testing" "testing"
"time" "time"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-cmp/cmp/cmpopts"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestAzureLogAnalyticsMacros(t *testing.T) { func TestAzureLogAnalyticsMacros(t *testing.T) {
fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local)
timeRange := plugins.DataTimeRange{ timeRange := backend.TimeRange{
From: fmt.Sprintf("%v", fromStart.Unix()*1000), From: fromStart,
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000), To: fromStart.Add(34 * time.Minute),
} }
tests := []struct { tests := []struct {
name string name string
query plugins.DataSubQuery query backend.DataQuery
timeRange plugins.DataTimeRange kql string
kql string expected string
expected string Err require.ErrorAssertionFunc
Err require.ErrorAssertionFunc
}{ }{
{ {
name: "invalid macro should be ignored", name: "invalid macro should be ignored",
query: plugins.DataSubQuery{}, query: backend.DataQuery{},
kql: "$__invalid()", kql: "$__invalid()",
expected: "$__invalid()", expected: "$__invalid()",
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "Kusto variables should be ignored", name: "Kusto variables should be ignored",
query: plugins.DataSubQuery{}, query: backend.DataQuery{},
kql: ") on $left.b == $right.y", kql: ") on $left.b == $right.y",
expected: ") on $left.b == $right.y", expected: ") on $left.b == $right.y",
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "$__contains macro with a multi template variable that has multiple selected values as a parameter should build in clause", name: "$__contains macro with a multi template variable that has multiple selected values as a parameter should build in clause",
query: plugins.DataSubQuery{}, query: backend.DataQuery{},
kql: "$__contains(col, 'val1','val2')", kql: "$__contains(col, 'val1','val2')",
expected: "['col'] in ('val1','val2')", expected: "['col'] in ('val1','val2')",
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "$__contains macro with a multi template variable that has a single selected value as a parameter should build in clause", name: "$__contains macro with a multi template variable that has a single selected value as a parameter should build in clause",
query: plugins.DataSubQuery{}, query: backend.DataQuery{},
kql: "$__contains(col, 'val1' )", kql: "$__contains(col, 'val1' )",
expected: "['col'] in ('val1')", expected: "['col'] in ('val1')",
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "$__contains macro with multi template variable has custom All value as a parameter should return a true expression", name: "$__contains macro with multi template variable has custom All value as a parameter should return a true expression",
query: plugins.DataSubQuery{}, query: backend.DataQuery{},
kql: "$__contains(col, all)", kql: "$__contains(col, all)",
expected: "1 == 1", expected: "1 == 1",
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "$__timeFilter has no column parameter should use default time field", name: "$__timeFilter has no column parameter should use default time field",
query: plugins.DataSubQuery{}, query: backend.DataQuery{TimeRange: timeRange},
kql: "$__timeFilter()", kql: "$__timeFilter()",
expected: "['TimeGenerated'] >= datetime('2018-03-15T13:00:00Z') and ['TimeGenerated'] <= datetime('2018-03-15T13:34:00Z')", expected: "['TimeGenerated'] >= datetime('2018-03-15T13:00:00Z') and ['TimeGenerated'] <= datetime('2018-03-15T13:34:00Z')",
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "$__timeFilter has time field parameter", name: "$__timeFilter has time field parameter",
query: plugins.DataSubQuery{}, query: backend.DataQuery{TimeRange: timeRange},
kql: "$__timeFilter(myTimeField)", kql: "$__timeFilter(myTimeField)",
expected: "['myTimeField'] >= datetime('2018-03-15T13:00:00Z') and ['myTimeField'] <= datetime('2018-03-15T13:34:00Z')", expected: "['myTimeField'] >= datetime('2018-03-15T13:00:00Z') and ['myTimeField'] <= datetime('2018-03-15T13:34:00Z')",
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "$__timeFrom and $__timeTo is in the query and range is a specific interval", name: "$__timeFrom and $__timeTo is in the query and range is a specific interval",
query: plugins.DataSubQuery{}, query: backend.DataQuery{TimeRange: timeRange},
kql: "myTimeField >= $__timeFrom() and myTimeField <= $__timeTo()", kql: "myTimeField >= $__timeFrom() and myTimeField <= $__timeTo()",
expected: "myTimeField >= datetime('2018-03-15T13:00:00Z') and myTimeField <= datetime('2018-03-15T13:34:00Z')", expected: "myTimeField >= datetime('2018-03-15T13:00:00Z') and myTimeField <= datetime('2018-03-15T13:34:00Z')",
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "$__interval should use the defined interval from the query", name: "$__interval should use the defined interval from the query",
timeRange: timeRange, query: backend.DataQuery{
query: plugins.DataSubQuery{ JSON: []byte(`{
Model: simplejson.NewFromAny(map[string]interface{}{ "interval": "5m"
"interval": "5m", }`),
}), TimeRange: timeRange,
}, },
kql: "bin(TimeGenerated, $__interval)", kql: "bin(TimeGenerated, $__interval)",
expected: "bin(TimeGenerated, 300000ms)", expected: "bin(TimeGenerated, 300000ms)",
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "$__interval should use the default interval if none is specified", name: "$__interval should use the default interval if none is specified",
query: plugins.DataSubQuery{ query: backend.DataQuery{TimeRange: timeRange},
DataSource: &models.DataSource{},
Model: simplejson.NewFromAny(map[string]interface{}{}),
},
kql: "bin(TimeGenerated, $__interval)", kql: "bin(TimeGenerated, $__interval)",
expected: "bin(TimeGenerated, 34000ms)", expected: "bin(TimeGenerated, 34000ms)",
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "$__escapeMulti with multi template variable should replace values with KQL style escaped strings", name: "$__escapeMulti with multi template variable should replace values with KQL style escaped strings",
query: plugins.DataSubQuery{ query: backend.DataQuery{},
DataSource: &models.DataSource{},
Model: simplejson.NewFromAny(map[string]interface{}{}),
},
kql: `CounterPath in ($__escapeMulti('\\grafana-vm\Network(eth0)\Total','\\grafana-vm\Network(eth1)\Total'))`, kql: `CounterPath in ($__escapeMulti('\\grafana-vm\Network(eth0)\Total','\\grafana-vm\Network(eth1)\Total'))`,
expected: `CounterPath in (@'\\grafana-vm\Network(eth0)\Total', @'\\grafana-vm\Network(eth1)\Total')`, expected: `CounterPath in (@'\\grafana-vm\Network(eth0)\Total', @'\\grafana-vm\Network(eth1)\Total')`,
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "$__escapeMulti with multi template variable and has one selected value that contains comma", name: "$__escapeMulti with multi template variable and has one selected value that contains comma",
query: plugins.DataSubQuery{ query: backend.DataQuery{},
DataSource: &models.DataSource{},
Model: simplejson.NewFromAny(map[string]interface{}{}),
},
kql: `$__escapeMulti('\\grafana-vm,\Network(eth0)\Total Bytes Received')`, kql: `$__escapeMulti('\\grafana-vm,\Network(eth0)\Total Bytes Received')`,
expected: `@'\\grafana-vm,\Network(eth0)\Total Bytes Received'`, expected: `@'\\grafana-vm,\Network(eth0)\Total Bytes Received'`,
Err: require.NoError, Err: require.NoError,
}, },
{ {
name: "$__escapeMulti with multi template variable and is not wrapped in single quotes should fail", name: "$__escapeMulti with multi template variable and is not wrapped in single quotes should fail",
query: plugins.DataSubQuery{ query: backend.DataQuery{},
DataSource: &models.DataSource{},
Model: simplejson.NewFromAny(map[string]interface{}{}),
},
kql: `$__escapeMulti(\\grafana-vm,\Network(eth0)\Total Bytes Received)`, kql: `$__escapeMulti(\\grafana-vm,\Network(eth0)\Total Bytes Received)`,
expected: "", expected: "",
Err: require.Error, Err: require.Error,
@ -141,7 +125,7 @@ func TestAzureLogAnalyticsMacros(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
defaultTimeField := "TimeGenerated" defaultTimeField := "TimeGenerated"
rawQuery, err := KqlInterpolate(tt.query, timeRange, tt.kql, defaultTimeField) rawQuery, err := KqlInterpolate(tt.query, datasourceInfo{}, tt.kql, defaultTimeField)
tt.Err(t, err) tt.Err(t, err)
if diff := cmp.Diff(tt.expected, rawQuery, cmpopts.EquateNaNs()); diff != "" { if diff := cmp.Diff(tt.expected, rawQuery, cmpopts.EquateNaNs()); diff != "" {
t.Errorf("Result mismatch (-want +got):\n%s", diff) t.Errorf("Result mismatch (-want +got):\n%s", diff)

3
pkg/tsdb/azuremonitor/types.go
View File

@ -6,6 +6,8 @@ import (
"net/url" "net/url"
"strings" "strings"
"time" "time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
) )
// AzureMonitorQuery is the query for all the services as they have similar queries // AzureMonitorQuery is the query for all the services as they have similar queries
@ -17,6 +19,7 @@ type AzureMonitorQuery struct {
Params url.Values Params url.Values
RefID string RefID string
Alias string Alias string
TimeRange backend.TimeRange
} }
// AzureMonitorResponse is the json response from the Azure Monitor API // AzureMonitorResponse is the json response from the Azure Monitor API

1
pkg/tsdb/service.go
View File

@ -65,7 +65,6 @@ func (s *Service) Init() error {
s.registry["mysql"] = mysql.New(s.HTTPClientProvider) s.registry["mysql"] = mysql.New(s.HTTPClientProvider)
s.registry["elasticsearch"] = elasticsearch.New(s.HTTPClientProvider) s.registry["elasticsearch"] = elasticsearch.New(s.HTTPClientProvider)
s.registry["stackdriver"] = s.CloudMonitoringService.NewExecutor s.registry["stackdriver"] = s.CloudMonitoringService.NewExecutor
s.registry["grafana-azure-monitor-datasource"] = s.AzureMonitorService.NewExecutor
s.registry["loki"] = loki.New(s.HTTPClientProvider) s.registry["loki"] = loki.New(s.HTTPClientProvider)
s.registry["tempo"] = tempo.New(s.HTTPClientProvider) s.registry["tempo"] = tempo.New(s.HTTPClientProvider)
return nil return nil