AzureMonitor: Add Azure Resource Graph (#33293)

* Add Azure Resource Graph in Azure Plugin

* fix lodash import

* Fix mock queries

* use "backend" sdk

* Address comments

* add converter for object type

* Query error should cause 400 & apply template var

* fix backend test & add documentation

* update image

* Address comments

* marshal body from map

* use interpolated query instead of raw query

* fix test

* filter out empty queries

* fix go linting problem

* use new query field language name

* improve variable tests

* add better tests for interpolate variable

Co-authored-by: joshhunt <josh@trtr.co>
Co-authored-by: Erik Sundell <erik.sundell87@gmail.com>
This commit is contained in:
shuotli 2021-05-19 01:31:27 -07:00 committed by GitHub
parent 4e31169a43
commit 71fd0981ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 934 additions and 25 deletions

View File

@ -69,6 +69,7 @@ In the query editor for a panel, after choosing your Azure Monitor data source,
- `Application Insights`
- `Logs`
- `Insights Analytics`
- `Azure Resource Graph`
The query editor changes depending on which one you pick. Metrics is the default.
@ -413,3 +414,65 @@ Application Insights and Insights Analytics are two ways to query the same Azure
> **Note** In Grafana 8.0, Application Insights and Insights Analytics will be deprecated and made read-only in favor of querying this data through Metrics and Logs. Existing queries will continue to work, but you cannot edit them.
To prepare for this upcoming change, Application Insights queries can now be made in Metrics, under the "microsoft.insights/components" Namespace. Insights Analytics queries cannot be made within Logs with KQL at this time.
## Query Azure Resource Graph service
Azure Resource Graph (ARG) is a service in Azure that is designed to extend Azure Resource Management by providing efficient and performant resource exploration with the ability to query at scale across a given set of subscriptions so that you can effectively govern your environment. By querying ARG, you can query resources with complex filtering, iteratively explore resources based on governance requirements, and assess the impact of applying policies in a vast cloud environment.
{{< docs-imagebox img="/img/docs/azure-monitor/azure-resource-graph-8-0.png" class="docs-image--no-shadow" caption="Azure Resource Graph Editor" >}}
### Table queries
Queries are written in the [Kusto Query Language](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/). An Azure Resource Graph query is formatted as table data.
If your credentials give you access to multiple subscriptions, then you can choose multiple subscriptions before entering queries.
### Sort results by resource properties
Here is an example query that returns any type of resource, but only the name, type, and location properties:
```kusto
Resources
| project name, type, location
| order by name asc
```
The query uses `order by` to sort the properties by the `name` property in ascending (asc) order. You can change what property to sort by and the order (`asc` or `desc`). The query uses `project` to show the listed properties in the results. You can add or remove properties.
### Query resources with complex filtering
Filtering for Azure resources with a tag name of `Environment` that have a value of `Internal`. You can change these to any desired tag key and value. The `=~` in the `type` match tells Resource Graph to be case insensitive. You can project by other properties or add/ remove more.
The tag key is case sensitive. `Environment` and `environment` will give different results. For example, a query that returns a list of resources with a specified tag value:
```kusto
Resources
| where tags.environment=~'internal'
| project name
```
### Group and aggregate the values by property
You can also use `summarize` and `count` to define how to group and aggregate the values by property. For example, returning count of healthy, unhealthy, and not applicable resources per recommendation.:
```kusto
securityresources
| where type == 'microsoft.security/assessments'
| extend resourceId=id,
    recommendationId=name,
    resourceType=type,
    recommendationName=properties.displayName,
    source=properties.resourceDetails.Source,
    recommendationState=properties.status.code,
    description=properties.metadata.description,
    assessmentType=properties.metadata.assessmentType,
    remediationDescription=properties.metadata.remediationDescription,
    policyDefinitionId=properties.metadata.policyDefinitionId,
    implementationEffort=properties.metadata.implementationEffort,
    recommendationSeverity=properties.metadata.severity,
    category=properties.metadata.categories,
    userImpact=properties.metadata.userImpact,
    threats=properties.metadata.threats,
    portalLink=properties.links.azurePortal
| summarize numberOfResources=count(resourceId) by tostring(recommendationName), tostring(recommendationState)
```

View File

@ -86,7 +86,7 @@ func (e *AzureLogAnalyticsDatasource) buildQueries(queries []plugins.DataSubQuer
resultFormat := azureLogAnalyticsTarget.ResultFormat
if resultFormat == "" {
resultFormat = "time_series"
resultFormat = timeSeries
}
urlComponents := map[string]string{}
@ -173,7 +173,7 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
return queryResultErrorWithExecuted(err)
}
frame, err := LogTableToFrame(t)
frame, err := ResponseTableToFrame(t)
if err != nil {
return queryResultErrorWithExecuted(err)
}
@ -187,7 +187,7 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
azlog.Warn("failed to add custom metadata to azure log analytics response", err)
}
if query.ResultFormat == "time_series" {
if query.ResultFormat == timeSeries {
tsSchema := frame.TimeSeriesSchema()
if tsSchema.Type == data.TimeSeriesTypeLong {
wideFrame, err := data.LongToWide(frame, nil)
@ -259,7 +259,7 @@ func (e *AzureLogAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourceP
// GetPrimaryResultTable returns the first table in the response named "PrimaryResult", or an
// error if there is no table by that name.
func (ar *AzureLogAnalyticsResponse) GetPrimaryResultTable() (*AzureLogAnalyticsTable, error) {
func (ar *AzureLogAnalyticsResponse) GetPrimaryResultTable() (*AzureResponseTable, error) {
for _, t := range ar.Tables {
if t.Name == "PrimaryResult" {
return &t, nil

View File

@ -41,7 +41,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"azureLogAnalytics": map[string]interface{}{
"workspace": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
"query": "query=Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer",
"resultFormat": "time_series",
"resultFormat": timeSeries,
},
}),
RefID: "A",
@ -50,12 +50,12 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
azureLogAnalyticsQueries: []*AzureLogAnalyticsQuery{
{
RefID: "A",
ResultFormat: "time_series",
ResultFormat: timeSeries,
URL: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/query",
Model: simplejson.NewFromAny(map[string]interface{}{
"azureLogAnalytics": map[string]interface{}{
"query": "query=Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer",
"resultFormat": "time_series",
"resultFormat": timeSeries,
"workspace": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
},
}),

View File

@ -0,0 +1,271 @@
package azuremonitor
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"path"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util/errutil"
"github.com/opentracing/opentracing-go"
"golang.org/x/net/context/ctxhttp"
)
// AzureResourceGraphDatasource calls the Azure Resource Graph API's
type AzureResourceGraphDatasource struct {
httpClient *http.Client
dsInfo *models.DataSource
pluginManager plugins.Manager
cfg *setting.Cfg
}
// AzureResourceGraphQuery is the query request that is built from the saved values for
// from the UI
type AzureResourceGraphQuery struct {
RefID string
ResultFormat string
URL string
Model *simplejson.Json
InterpolatedQuery string
}
const argAPIVersion = "2018-09-01-preview"
const argQueryProviderName = "/providers/Microsoft.ResourceGraph/resources"
// executeTimeSeriesQuery does the following:
// 1. builds the AzureMonitor url and querystring for each query
// 2. executes each query by calling the Azure Monitor API
// 3. parses the responses for each query into the timeseries format
func (e *AzureResourceGraphDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []plugins.DataSubQuery,
timeRange plugins.DataTimeRange) (backend.QueryDataResponse, error) {
result := backend.QueryDataResponse{
Responses: map[string]backend.DataResponse{},
}
queries, err := e.buildQueries(originalQueries, timeRange)
if err != nil {
return backend.QueryDataResponse{}, err
}
for _, query := range queries {
result.Responses[query.RefID] = e.executeQuery(ctx, query, timeRange)
}
return result, nil
}
func (e *AzureResourceGraphDatasource) buildQueries(queries []plugins.DataSubQuery,
timeRange plugins.DataTimeRange) ([]*AzureResourceGraphQuery, error) {
var azureResourceGraphQueries []*AzureResourceGraphQuery
for _, query := range queries {
queryBytes, err := query.Model.Encode()
if err != nil {
return nil, fmt.Errorf("failed to re-encode the Azure Resource Graph query into JSON: %w", err)
}
queryJSONModel := argJSONQuery{}
err = json.Unmarshal(queryBytes, &queryJSONModel)
if err != nil {
return nil, fmt.Errorf("failed to decode the Azure Resource Graph query object from JSON: %w", err)
}
azureResourceGraphTarget := queryJSONModel.AzureResourceGraph
azlog.Debug("AzureResourceGraph", "target", azureResourceGraphTarget)
resultFormat := azureResourceGraphTarget.ResultFormat
if resultFormat == "" {
resultFormat = "table"
}
interpolatedQuery, err := KqlInterpolate(query, timeRange, azureResourceGraphTarget.Query)
if err != nil {
return nil, err
}
azureResourceGraphQueries = append(azureResourceGraphQueries, &AzureResourceGraphQuery{
RefID: query.RefID,
ResultFormat: resultFormat,
Model: query.Model,
InterpolatedQuery: interpolatedQuery,
})
}
return azureResourceGraphQueries, nil
}
func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *AzureResourceGraphQuery,
timeRange plugins.DataTimeRange) backend.DataResponse {
queryResult := backend.DataResponse{}
params := url.Values{}
params.Add("api-version", argAPIVersion)
queryResultErrorWithExecuted := func(err error) backend.DataResponse {
queryResult = backend.DataResponse{Error: err}
frames := data.Frames{
&data.Frame{
RefID: query.RefID,
Meta: &data.FrameMeta{
ExecutedQueryString: query.InterpolatedQuery,
},
},
}
queryResult.Frames = frames
return queryResult
}
reqBody, err := json.Marshal(map[string]interface{}{
"subscriptions": query.Model.Get("subscriptions").MustStringArray(),
"query": query.InterpolatedQuery,
})
if err != nil {
queryResult.Error = err
return queryResult
}
req, err := e.createRequest(ctx, e.dsInfo, reqBody)
if err != nil {
queryResult.Error = err
return queryResult
}
req.URL.Path = path.Join(req.URL.Path, argQueryProviderName)
req.URL.RawQuery = params.Encode()
span, ctx := opentracing.StartSpanFromContext(ctx, "azure resource graph query")
span.SetTag("interpolated_query", query.InterpolatedQuery)
span.SetTag("from", timeRange.From)
span.SetTag("until", timeRange.To)
span.SetTag("datasource_id", e.dsInfo.Id)
span.SetTag("org_id", e.dsInfo.OrgId)
defer span.Finish()
if err := opentracing.GlobalTracer().Inject(
span.Context(),
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(req.Header)); err != nil {
return queryResultErrorWithExecuted(err)
}
azlog.Debug("AzureResourceGraph", "Request ApiURL", req.URL.String())
res, err := ctxhttp.Do(ctx, e.httpClient, req)
if err != nil {
return queryResultErrorWithExecuted(err)
}
argResponse, err := e.unmarshalResponse(res)
if err != nil {
return queryResultErrorWithExecuted(err)
}
frame, err := ResponseTableToFrame(&argResponse.Data)
if err != nil {
return queryResultErrorWithExecuted(err)
}
if frame.Meta == nil {
frame.Meta = &data.FrameMeta{}
}
frame.Meta.ExecutedQueryString = req.URL.RawQuery
queryResult.Frames = data.Frames{frame}
return queryResult
}
func (e *AzureResourceGraphDatasource) createRequest(ctx context.Context, dsInfo *models.DataSource, reqBody []byte) (*http.Request, error) {
u, err := url.Parse(dsInfo.Url)
if err != nil {
return nil, err
}
u.Path = path.Join(u.Path, "render")
req, err := http.NewRequest(http.MethodPost, u.String(), bytes.NewBuffer(reqBody))
if err != nil {
azlog.Debug("Failed to create request", "error", err)
return nil, errutil.Wrap("failed to create request", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
// find plugin
plugin := e.pluginManager.GetDataSource(dsInfo.Type)
if plugin == nil {
return nil, errors.New("unable to find datasource plugin Azure Monitor")
}
cloudName := dsInfo.JsonData.Get("cloudName").MustString("azuremonitor")
argRoute, proxypass, err := e.getPluginRoute(plugin, cloudName)
if err != nil {
return nil, err
}
pluginproxy.ApplyRoute(ctx, req, proxypass, argRoute, dsInfo, e.cfg)
return req, nil
}
func (e *AzureResourceGraphDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, cloudName string) (
*plugins.AppPluginRoute, string, error) {
pluginRouteName := "azureresourcegraph"
switch cloudName {
case "chinaazuremonitor":
pluginRouteName = "chinaazureresourcegraph"
case "govazuremonitor":
pluginRouteName = "govazureresourcegraph"
}
var argRoute *plugins.AppPluginRoute
for _, route := range plugin.Routes {
if route.Path == pluginRouteName {
argRoute = route
break
}
}
return argRoute, pluginRouteName, nil
}
func (e *AzureResourceGraphDatasource) unmarshalResponse(res *http.Response) (AzureResourceGraphResponse, error) {
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return AzureResourceGraphResponse{}, err
}
defer func() {
if err := res.Body.Close(); err != nil {
azlog.Warn("Failed to close response body", "err", err)
}
}()
if res.StatusCode/100 != 2 {
azlog.Debug("Request failed", "status", res.Status, "body", string(body))
return AzureResourceGraphResponse{}, fmt.Errorf("request failed, status: %s, body: %s", res.Status, string(body))
}
var data AzureResourceGraphResponse
d := json.NewDecoder(bytes.NewReader(body))
d.UseNumber()
err = d.Decode(&data)
if err != nil {
azlog.Debug("Failed to unmarshal azure resource graph response", "error", err, "status", res.Status, "body", string(body))
return AzureResourceGraphResponse{}, err
}
return data, nil
}

View File

@ -0,0 +1,75 @@
package azuremonitor
import (
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/stretchr/testify/require"
)
func TestBuildingAzureResourceGraphQueries(t *testing.T) {
datasource := &AzureResourceGraphDatasource{}
fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local)
tests := []struct {
name string
queryModel []plugins.DataSubQuery
timeRange plugins.DataTimeRange
azureResourceGraphQueries []*AzureResourceGraphQuery
Err require.ErrorAssertionFunc
}{
{
name: "Query with macros should be interpolated",
timeRange: plugins.DataTimeRange{
From: fmt.Sprintf("%v", fromStart.Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
},
queryModel: []plugins.DataSubQuery{
{
DataSource: &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{}),
},
Model: simplejson.NewFromAny(map[string]interface{}{
"queryType": "Azure Resource Graph",
"azureResourceGraph": map[string]interface{}{
"query": "resources | where $__contains(name,'res1','res2')",
"resultFormat": "table",
},
}),
RefID: "A",
},
},
azureResourceGraphQueries: []*AzureResourceGraphQuery{
{
RefID: "A",
ResultFormat: "table",
URL: "",
Model: simplejson.NewFromAny(map[string]interface{}{
"azureResourceGraph": map[string]interface{}{
"query": "resources | where $__contains(name,'res1','res2')",
"resultFormat": "table",
},
}),
InterpolatedQuery: "resources | where ['name'] in ('res1','res2')",
},
},
Err: require.NoError,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
queries, err := datasource.buildQueries(tt.queryModel, tt.timeRange)
tt.Err(t, err)
if diff := cmp.Diff(tt.azureResourceGraphQueries, queries, cmpopts.IgnoreUnexported(simplejson.Json{})); diff != "" {
t.Errorf("Result mismatch (-want +got):\n%s", diff)
}
})
}
}

View File

@ -10,8 +10,8 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
)
// LogTableToFrame converts an AzureLogAnalyticsTable to a data.Frame.
func LogTableToFrame(table *AzureLogAnalyticsTable) (*data.Frame, error) {
// ResponseTableToFrame converts an AzureResponseTable to a data.Frame.
func ResponseTableToFrame(table *AzureResponseTable) (*data.Frame, error) {
converterFrame, err := converterFrameForTable(table)
if err != nil {
return nil, err
@ -27,7 +27,7 @@ func LogTableToFrame(table *AzureLogAnalyticsTable) (*data.Frame, error) {
return converterFrame.Frame, nil
}
func converterFrameForTable(t *AzureLogAnalyticsTable) (*data.FrameInputConverter, error) {
func converterFrameForTable(t *AzureResponseTable) (*data.FrameInputConverter, error) {
converters := []data.FieldConverter{}
colNames := make([]string, len(t.Columns))
colTypes := make([]string, len(t.Columns)) // for metadata
@ -64,12 +64,14 @@ var converterMap = map[string]data.FieldConverter{
"guid": stringConverter,
"timespan": stringConverter,
"dynamic": stringConverter,
"object": objectToStringConverter,
"datetime": timeConverter,
"int": intConverter,
"long": longConverter,
"real": realConverter,
"bool": boolConverter,
"decimal": decimalConverter,
"integer": intConverter,
}
var stringConverter = data.FieldConverter{
@ -88,6 +90,26 @@ var stringConverter = data.FieldConverter{
},
}
var objectToStringConverter = data.FieldConverter{
OutputFieldType: data.FieldTypeNullableString,
Converter: func(kustoValue interface{}) (interface{}, error) {
var output *string
if kustoValue == nil {
return output, nil
}
data, err := json.Marshal(kustoValue)
if err != nil {
fmt.Printf("failed to marshal column value: %s", err)
}
asString := string(data)
output = &asString
return output, nil
},
}
var timeConverter = data.FieldConverter{
OutputFieldType: data.FieldTypeNullableTime,
Converter: func(v interface{}) (interface{}, error) {

View File

@ -114,10 +114,11 @@ func TestLogTableToFrame(t *testing.T) {
data.NewField("XReal", nil, []*float64{pointer.Float64(1.797693134862315708145274237317043567981e+308)}),
data.NewField("XTimeSpan", nil, []*string{pointer.String("00:00:00.0000001")}),
data.NewField("XDecimal", nil, []*float64{pointer.Float64(79228162514264337593543950335)}),
data.NewField("XObject", nil, []*string{pointer.String(`"{\"person\": \"Daniel\", \"cats\": 23, \"diagnosis\": \"cat problem\"}"`)}),
)
frame.Meta = &data.FrameMeta{
Custom: &LogAnalyticsMeta{ColumnTypes: []string{"bool", "string", "datetime",
"dynamic", "guid", "int", "long", "real", "timespan", "decimal"}},
"dynamic", "guid", "int", "long", "real", "timespan", "decimal", "object"}},
}
return frame
},
@ -142,7 +143,7 @@ func TestLogTableToFrame(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
res := loadLogAnalyticsTestFileWithNumber(t, tt.testFile)
frame, err := LogTableToFrame(&res.Tables[0])
frame, err := ResponseTableToFrame(&res.Tables[0])
require.NoError(t, err)
if diff := cmp.Diff(tt.expectedFrame(), frame, data.FrameTestCompareOptions()...); diff != "" {

View File

@ -13,6 +13,8 @@ import (
"github.com/grafana/grafana/pkg/setting"
)
const timeSeries = "time_series"
var (
azlog = log.New("tsdb.azuremonitor")
legendKeyFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
@ -72,6 +74,7 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat
var applicationInsightsQueries []plugins.DataSubQuery
var azureLogAnalyticsQueries []plugins.DataSubQuery
var insightsAnalyticsQueries []plugins.DataSubQuery
var azureResourceGraphQueries []plugins.DataSubQuery
for _, query := range tsdbQuery.Queries {
queryType := query.Model.Get("queryType").MustString("")
@ -85,6 +88,8 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat
azureLogAnalyticsQueries = append(azureLogAnalyticsQueries, query)
case "Insights Analytics":
insightsAnalyticsQueries = append(insightsAnalyticsQueries, query)
case "Azure Resource Graph":
azureResourceGraphQueries = append(azureResourceGraphQueries, query)
default:
return plugins.DataResponse{}, fmt.Errorf("alerting not supported for %q", queryType)
}
@ -118,6 +123,12 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat
cfg: e.cfg,
}
argDatasource := &AzureResourceGraphDatasource{
httpClient: e.httpClient,
dsInfo: e.dsInfo,
pluginManager: e.pluginManager,
}
azResult, err := azDatasource.executeTimeSeriesQuery(ctx, azureMonitorQueries, *tsdbQuery.TimeRange)
if err != nil {
return plugins.DataResponse{}, err
@ -138,6 +149,11 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat
return plugins.DataResponse{}, err
}
argResult, err := argDatasource.executeTimeSeriesQuery(ctx, azureResourceGraphQueries, *tsdbQuery.TimeRange)
if err != nil {
return plugins.DataResponse{}, err
}
for k, v := range aiResult.Results {
azResult.Results[k] = v
}
@ -150,5 +166,9 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat
azResult.Results[k] = v
}
for k, v := range argResult.Responses {
azResult.Results[k] = plugins.DataQueryResult{Error: v.Error, Dataframes: plugins.NewDecodedDataFrames(v.Frames)}
}
return azResult, nil
}

View File

@ -163,12 +163,12 @@ func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *I
return queryResultError(err)
}
frame, err := LogTableToFrame(t)
frame, err := ResponseTableToFrame(t)
if err != nil {
return queryResultError(err)
}
if query.ResultFormat == "time_series" {
if query.ResultFormat == timeSeries {
tsSchema := frame.TimeSeriesSchema()
if tsSchema.Type == data.TimeSeriesTypeLong {
wideFrame, err := data.LongToWide(frame, nil)

View File

@ -42,6 +42,10 @@
{
"name":"XDecimal",
"type":"decimal"
},
{
"name":"XObject",
"type":"object"
}
],
"rows": [
@ -55,10 +59,11 @@
9223372036854775807,
1.7976931348623157e+308,
"00:00:00.0000001",
"79228162514264337593543950335"
"79228162514264337593543950335",
"{\"person\": \"Daniel\", \"cats\": 23, \"diagnosis\": \"cat problem\"}"
]
]
}
]
}

View File

@ -56,11 +56,16 @@ type AzureMonitorResponse struct {
// AzureLogAnalyticsResponse is the json response object from the Azure Log Analytics API.
type AzureLogAnalyticsResponse struct {
Tables []AzureLogAnalyticsTable `json:"tables"`
Tables []AzureResponseTable `json:"tables"`
}
// AzureLogAnalyticsTable is the table format for Log Analytics responses
type AzureLogAnalyticsTable struct {
// AzureResourceGraphResponse is the json response object from the Azure Resource Graph Analytics API.
type AzureResourceGraphResponse struct {
Data AzureResponseTable `json:"data"`
}
// AzureResponseTable is the table format for Azure responses
type AzureResponseTable struct {
Name string `json:"name"`
Columns []struct {
Name string `json:"name"`
@ -137,6 +142,13 @@ type logJSONQuery struct {
} `json:"azureLogAnalytics"`
}
type argJSONQuery struct {
AzureResourceGraph struct {
Query string `json:"query"`
ResultFormat string `json:"resultFormat"`
} `json:"azureResourceGraph"`
}
// InsightsDimensions will unmarshal from a JSON string, or an array of strings,
// into a string array. This exists to support an older query format which is updated
// when a user saves the query or it is sent from the front end, but may not be when

View File

@ -11,6 +11,11 @@ export default function createMockQuery(): AzureMonitorQuery {
workspace: 'e3fe4fde-ad5e-4d60-9974-e2f3562ffdf2',
},
azureResourceGraph: {
query: 'Resources | summarize count()',
resultFormat: 'time_series',
},
azureMonitor: {
// aggOptions: [],
aggregation: 'Average',
@ -35,8 +40,8 @@ export default function createMockQuery(): AzureMonitorQuery {
queryType: AzureQueryType.AzureMonitor,
refId: 'A',
subscription: 'abc-123',
subscription: '99999999-cccc-bbbb-aaaa-9106972f9572',
subscriptions: ['99999999-cccc-bbbb-aaaa-9106972f9572'],
format: 'dunno lol', // unsure what this value should be. It's not there at runtime, but it's in the ts interface
};
}

View File

@ -0,0 +1,149 @@
import { TemplateSrv } from 'app/features/templating/template_srv';
import { backendSrv } from 'app/core/services/backend_srv';
import AzureResourceGraphDatasource from './azure_resource_graph_datasource';
import { CustomVariableModel, initialVariableModelState, VariableHide } from 'app/features/variables/types';
import { initialCustomVariableModelState } from 'app/features/variables/custom/reducer';
const templateSrv = new TemplateSrv();
const single: CustomVariableModel = {
...initialVariableModelState,
id: 'var1',
name: 'var1',
index: 0,
current: { value: 'var1-foo', text: 'var1-foo', selected: true },
options: [{ value: 'var1-foo', text: 'var1-foo', selected: true }],
multi: false,
includeAll: false,
query: '',
hide: VariableHide.dontHide,
type: 'custom',
};
const multi: CustomVariableModel = {
...initialVariableModelState,
id: 'var3',
name: 'var3',
index: 2,
current: { value: ['var3-foo', 'var3-baz'], text: 'var3-foo + var3-baz', selected: true },
options: [
{ selected: true, value: 'var3-foo', text: 'var3-foo' },
{ selected: false, value: 'var3-bar', text: 'var3-bar' },
{ selected: true, value: 'var3-baz', text: 'var3-baz' },
],
multi: true,
includeAll: false,
query: '',
hide: VariableHide.dontHide,
type: 'custom',
};
templateSrv.init([single, multi]);
jest.mock('app/core/services/backend_srv');
jest.mock('@grafana/runtime', () => ({
...((jest.requireActual('@grafana/runtime') as unknown) as object),
getBackendSrv: () => backendSrv,
getTemplateSrv: () => templateSrv,
}));
describe('AzureResourceGraphDatasource', () => {
const datasourceRequestMock = jest.spyOn(backendSrv, 'datasourceRequest');
beforeEach(() => {
jest.clearAllMocks();
datasourceRequestMock.mockImplementation(jest.fn());
});
const ctx: any = {};
beforeEach(() => {
ctx.instanceSettings = {
url: 'http://azureresourcegraphapi',
};
ctx.ds = new AzureResourceGraphDatasource(ctx.instanceSettings);
});
describe('When applying template variables', () => {
it('should expand single value template variable', () => {
const target = {
azureResourceGraph: {
query: 'Resources | $var1',
resultFormat: '',
},
};
expect(ctx.ds.applyTemplateVariables(target)).toStrictEqual({
azureResourceGraph: { query: 'Resources | var1-foo', resultFormat: 'table' },
format: undefined,
queryType: 'Azure Resource Graph',
refId: undefined,
subscriptions: undefined,
});
});
it('should expand multi value template variable', () => {
const target = {
azureResourceGraph: {
query: 'resources | where $__contains(name, $var3)',
resultFormat: '',
},
};
expect(ctx.ds.applyTemplateVariables(target)).toStrictEqual({
azureResourceGraph: {
query: `resources | where $__contains(name, 'var3-foo','var3-baz')`,
resultFormat: 'table',
},
format: undefined,
queryType: 'Azure Resource Graph',
refId: undefined,
subscriptions: undefined,
});
});
});
describe('When interpolating variables', () => {
beforeEach(() => {
ctx.variable = { ...initialCustomVariableModelState };
});
describe('and value is a string', () => {
it('should return an unquoted value', () => {
expect(ctx.ds.interpolateVariable('abc', ctx.variable)).toEqual('abc');
});
});
describe('and value is a number', () => {
it('should return an unquoted value', () => {
expect(ctx.ds.interpolateVariable(1000, ctx.variable)).toEqual(1000);
});
});
describe('and value is an array of strings', () => {
it('should return comma separated quoted values', () => {
expect(ctx.ds.interpolateVariable(['a', 'b', 'c'], ctx.variable)).toEqual("'a','b','c'");
});
});
describe('and variable allows multi-value and value is a string', () => {
it('should return a quoted value', () => {
ctx.variable.multi = true;
expect(ctx.ds.interpolateVariable('abc', ctx.variable)).toEqual("'abc'");
});
});
describe('and variable contains single quote', () => {
it('should return a quoted value', () => {
ctx.variable.multi = true;
expect(ctx.ds.interpolateVariable("a'bc", ctx.variable)).toEqual("'a'bc'");
});
});
describe('and variable allows all and value is a string', () => {
it('should return a quoted value', () => {
ctx.variable.includeAll = true;
expect(ctx.ds.interpolateVariable('abc', ctx.variable)).toEqual("'abc'");
});
});
});
});

View File

@ -0,0 +1,56 @@
// eslint-disable-next-line lodash/import-scope
import _ from 'lodash';
import { AzureMonitorQuery, AzureDataSourceJsonData, AzureQueryType } from '../types';
import { ScopedVars } from '@grafana/data';
import { getTemplateSrv, DataSourceWithBackend } from '@grafana/runtime';
export default class AzureResourceGraphDatasource extends DataSourceWithBackend<
AzureMonitorQuery,
AzureDataSourceJsonData
> {
filterQuery(item: AzureMonitorQuery): boolean {
return !!item.azureResourceGraph?.query;
}
applyTemplateVariables(target: AzureMonitorQuery, scopedVars: ScopedVars): Record<string, any> {
const item = target.azureResourceGraph;
const templateSrv = getTemplateSrv();
const query = templateSrv.replace(item.query, scopedVars, this.interpolateVariable);
return {
refId: target.refId,
format: target.format,
queryType: AzureQueryType.AzureResourceGraph,
subscriptions: target.subscriptions,
azureResourceGraph: {
resultFormat: 'table',
query,
},
};
}
interpolateVariable(value: string, variable: { multi: any; includeAll: any }) {
if (typeof value === 'string') {
if (variable.multi || variable.includeAll) {
return "'" + value + "'";
} else {
return value;
}
}
if (typeof value === 'number') {
return value;
}
const quotedValues = _.map(value, (val) => {
if (typeof value === 'number') {
return value;
}
return "'" + val + "'";
});
return quotedValues.join(',');
}
}

View File

@ -0,0 +1,51 @@
import React from 'react';
import { AzureMonitorErrorish, AzureMonitorOption, AzureMonitorQuery } from '../../types';
import Datasource from '../../datasource';
import { InlineFieldRow } from '@grafana/ui';
import SubscriptionField from '../SubscriptionField';
import QueryField from './QueryField';
interface LogsQueryEditorProps {
query: AzureMonitorQuery;
datasource: Datasource;
subscriptionId: string;
onChange: (newQuery: AzureMonitorQuery) => void;
variableOptionGroup: { label: string; options: AzureMonitorOption[] };
setError: (source: string, error: AzureMonitorErrorish | undefined) => void;
}
const ArgQueryEditor: React.FC<LogsQueryEditorProps> = ({
query,
datasource,
subscriptionId,
variableOptionGroup,
onChange,
setError,
}) => {
return (
<div data-testid="azure-monitor-logs-query-editor">
<InlineFieldRow>
<SubscriptionField
multiSelect
query={query}
datasource={datasource}
subscriptionId={subscriptionId}
variableOptionGroup={variableOptionGroup}
onQueryChange={onChange}
setError={setError}
/>
</InlineFieldRow>
<QueryField
query={query}
datasource={datasource}
subscriptionId={subscriptionId}
variableOptionGroup={variableOptionGroup}
onQueryChange={onChange}
setError={setError}
/>
</div>
);
};
export default ArgQueryEditor;

View File

@ -0,0 +1,32 @@
import { CodeEditor } from '@grafana/ui';
import React, { useCallback } from 'react';
import { AzureQueryEditorFieldProps } from '../../types';
const QueryField: React.FC<AzureQueryEditorFieldProps> = ({ query, onQueryChange }) => {
const onChange = useCallback(
(newQuery: string) => {
onQueryChange({
...query,
azureResourceGraph: {
...query.azureResourceGraph,
query: newQuery,
},
});
},
[onQueryChange, query]
);
return (
<CodeEditor
value={query.azureResourceGraph.query}
language="kusto"
height={200}
width={1000}
showMiniMap={false}
onBlur={onChange}
onSave={onChange}
/>
);
};
export default QueryField;

View File

@ -0,0 +1 @@
export { default } from './ArgQueryEditor';

View File

@ -1,6 +1,6 @@
import React, { useCallback, useMemo } from 'react';
import { SelectableValue } from '@grafana/data';
import { Select } from '@grafana/ui';
import React, { useCallback, useMemo } from 'react';
import { AzureMonitorOption, AzureQueryEditorFieldProps, AzureResultFormat } from '../../types';
import { findOption } from '../../utils/common';
import { Field } from '../Field';

View File

@ -6,6 +6,7 @@ import MetricsQueryEditor from '../MetricsQueryEditor';
import QueryTypeField from './QueryTypeField';
import useLastError from '../../utils/useLastError';
import LogsQueryEditor from '../LogsQueryEditor';
import ArgQueryEditor from '../ArgQueryEditor';
import ApplicationInsightsEditor from '../ApplicationInsightsEditor';
import InsightsAnalyticsEditor from '../InsightsAnalyticsEditor';
import { Space } from '../Space';
@ -93,6 +94,18 @@ const EditorForQueryType: React.FC<EditorForQueryTypeProps> = ({
case AzureQueryType.InsightsAnalytics:
return <InsightsAnalyticsEditor query={query} />;
case AzureQueryType.AzureResourceGraph:
return (
<ArgQueryEditor
subscriptionId={subscriptionId}
query={query}
datasource={datasource}
onChange={onChange}
variableOptionGroup={variableOptionGroup}
setError={setError}
/>
);
}
return null;

View File

@ -10,6 +10,7 @@ const QUERY_TYPES = [
{ value: AzureQueryType.LogAnalytics, label: 'Logs' },
{ value: AzureQueryType.ApplicationInsights, label: 'Application Insights' },
{ value: AzureQueryType.InsightsAnalytics, label: 'Insights Analytics' },
{ value: AzureQueryType.AzureResourceGraph, label: 'Azure Resource Graph' },
];
interface QueryTypeFieldProps {

View File

@ -1,13 +1,14 @@
import React, { useCallback, useEffect, useState, useMemo } from 'react';
import { SelectableValue } from '@grafana/data';
import { Select } from '@grafana/ui';
import { Select, MultiSelect } from '@grafana/ui';
import { AzureMonitorQuery, AzureQueryType, AzureQueryEditorFieldProps, AzureMonitorOption } from '../types';
import { findOption } from '../utils/common';
import { findOption, findOptions } from '../utils/common';
import { Field } from './Field';
interface SubscriptionFieldProps extends AzureQueryEditorFieldProps {
onQueryChange: (newQuery: AzureMonitorQuery) => void;
multiSelect?: boolean;
}
const ERROR_SOURCE = 'metrics-subscription';
@ -17,6 +18,7 @@ const SubscriptionField: React.FC<SubscriptionFieldProps> = ({
variableOptionGroup,
onQueryChange,
setError,
multiSelect = false,
}) => {
const [subscriptions, setSubscriptions] = useState<AzureMonitorOption[]>([]);
@ -92,9 +94,33 @@ const SubscriptionField: React.FC<SubscriptionFieldProps> = ({
[query, onQueryChange]
);
const onSubscriptionsChange = useCallback(
(change: Array<SelectableValue<string>>) => {
if (!change) {
return;
}
query.subscriptions = change.map((c) => c.value ?? '');
onQueryChange(query);
},
[query, onQueryChange]
);
const options = useMemo(() => [...subscriptions, variableOptionGroup], [subscriptions, variableOptionGroup]);
return (
return multiSelect ? (
<Field label="Subscriptions">
<MultiSelect
isClearable
value={findOptions(subscriptions, query.subscriptions)}
inputId="azure-monitor-subscriptions-field"
onChange={onSubscriptionsChange}
options={options}
width={38}
/>
</Field>
) : (
<Field label="Subscription">
<Select
value={findOption(subscriptions, query.subscription)}

View File

@ -17,12 +17,14 @@ import { DataSourceWithBackend, getTemplateSrv, TemplateSrv } from '@grafana/run
import InsightsAnalyticsDatasource from './insights_analytics/insights_analytics_datasource';
import { migrateMetricsDimensionFilters } from './query_ctrl';
import { map } from 'rxjs/operators';
import AzureResourceGraphDatasource from './azure_resource_graph/azure_resource_graph_datasource';
export default class Datasource extends DataSourceApi<AzureMonitorQuery, AzureDataSourceJsonData> {
azureMonitorDatasource: AzureMonitorDatasource;
appInsightsDatasource: AppInsightsDatasource;
azureLogAnalyticsDatasource: AzureLogAnalyticsDatasource;
insightsAnalyticsDatasource: InsightsAnalyticsDatasource;
azureResourceGraphDatasource: AzureResourceGraphDatasource;
pseudoDatasource: Record<AzureQueryType, DataSourceWithBackend>;
optionsKey: Record<AzureQueryType, string>;
@ -36,12 +38,14 @@ export default class Datasource extends DataSourceApi<AzureMonitorQuery, AzureDa
this.appInsightsDatasource = new AppInsightsDatasource(instanceSettings);
this.azureLogAnalyticsDatasource = new AzureLogAnalyticsDatasource(instanceSettings);
this.insightsAnalyticsDatasource = new InsightsAnalyticsDatasource(instanceSettings);
this.azureResourceGraphDatasource = new AzureResourceGraphDatasource(instanceSettings);
const pseudoDatasource: any = {};
pseudoDatasource[AzureQueryType.ApplicationInsights] = this.appInsightsDatasource;
pseudoDatasource[AzureQueryType.AzureMonitor] = this.azureMonitorDatasource;
pseudoDatasource[AzureQueryType.InsightsAnalytics] = this.insightsAnalyticsDatasource;
pseudoDatasource[AzureQueryType.LogAnalytics] = this.azureLogAnalyticsDatasource;
pseudoDatasource[AzureQueryType.AzureResourceGraph] = this.azureResourceGraphDatasource;
this.pseudoDatasource = pseudoDatasource;
const optionsKey: any = {};
@ -49,6 +53,7 @@ export default class Datasource extends DataSourceApi<AzureMonitorQuery, AzureDa
optionsKey[AzureQueryType.AzureMonitor] = 'azureMonitor';
optionsKey[AzureQueryType.InsightsAnalytics] = 'insightsAnalytics';
optionsKey[AzureQueryType.LogAnalytics] = 'azureLogAnalytics';
optionsKey[AzureQueryType.AzureResourceGraph] = 'azureResourceGraph';
this.optionsKey = optionsKey;
}

View File

@ -97,6 +97,74 @@
},
"headers": [{ "name": "x-ms-app", "content": "Grafana" }]
},
{
"path": "azureresourcegraph",
"method": "POST",
"url": "https://management.azure.com",
"authType": "azure",
"tokenAuth": {
"scopes": ["https://management.azure.com/.default"],
"params": {
"azure_auth_type": "{{.JsonData.azureAuthType}}",
"azure_cloud": "AzureCloud",
"tenant_id": "{{.JsonData.tenantId}}",
"client_id": "{{.JsonData.clientId}}",
"client_secret": "{{.SecureJsonData.clientSecret}}"
}
},
"headers": [{ "name": "x-ms-app", "content": "Grafana" }]
},
{
"path": "chinaazureresourcegraph",
"method": "POST",
"url": "https://management.azure.com",
"authType": "azure",
"tokenAuth": {
"scopes": ["https://management.chinacloudapi.cn/.default"],
"params": {
"azure_auth_type": "{{.JsonData.azureAuthType}}",
"azure_cloud": "AzureChinaCloud",
"tenant_id": "{{.JsonData.tenantId}}",
"client_id": "{{.JsonData.clientId}}",
"client_secret": "{{.SecureJsonData.clientSecret}}"
}
},
"headers": [{ "name": "x-ms-app", "content": "Grafana" }]
},
{
"path": "govazureresourcegraph",
"method": "POST",
"url": "https://management.usgovcloudapi.net",
"authType": "azure",
"tokenAuth": {
"scopes": ["https://management.usgovcloudapi.net/.default"],
"params": {
"azure_auth_type": "{{.JsonData.azureAuthType}}",
"azure_cloud": "AzureUSGovernment",
"tenant_id": "{{.JsonData.tenantId}}",
"client_id": "{{.JsonData.clientId}}",
"client_secret": "{{.SecureJsonData.clientSecret}}"
}
},
"headers": [{ "name": "x-ms-app", "content": "Grafana" }]
},
{
"path": "germanyazureresourcegraph",
"method": "POST",
"url": "https://management.microsoftazure.de",
"authType": "azure",
"tokenAuth": {
"scopes": ["https://management.microsoftazure.de/.default"],
"params": {
"azure_auth_type": "{{.JsonData.azureAuthType}}",
"azure_cloud": "AzureGermanCloud",
"tenant_id": "{{.JsonData.tenantId}}",
"client_id": "{{.JsonData.clientId}}",
"client_secret": "{{.SecureJsonData.clientSecret}}"
}
},
"headers": [{ "name": "x-ms-app", "content": "Grafana" }]
},
{
"path": "appinsights",
"method": "GET",

View File

@ -27,12 +27,14 @@ export class AzureMonitorQueryCtrl extends QueryCtrl {
{ id: AzureQueryType.LogAnalytics, label: 'Logs' },
{ id: AzureQueryType.ApplicationInsights, label: 'Application Insights' },
{ id: AzureQueryType.InsightsAnalytics, label: 'Insights Analytics' },
{ id: AzureQueryType.AzureResourceGraph, label: 'Azure Resource Graph' },
];
// Query types that have been migrated to React
reactQueryEditors = [
AzureQueryType.AzureMonitor,
AzureQueryType.LogAnalytics,
AzureQueryType.AzureResourceGraph,
// AzureQueryType.ApplicationInsights,
// AzureQueryType.InsightsAnalytics,
];
@ -44,12 +46,17 @@ export class AzureMonitorQueryCtrl extends QueryCtrl {
refId: string;
queryType: AzureQueryType;
subscription: string;
subscriptions: string[];
azureMonitor: AzureMetricQuery;
azureLogAnalytics: {
query: string;
resultFormat: string;
workspace: string;
};
azureResourceGraph: {
query: string;
resultFormat: string;
};
appInsights: {
// metric style query when rawQuery == false
metricName: string;
@ -105,6 +112,9 @@ export class AzureMonitorQueryCtrl extends QueryCtrl {
? this.datasource.azureLogAnalyticsDatasource.defaultOrFirstWorkspace
: '',
},
azureResourceGraph: {
resultFormat: 'table',
},
appInsights: {
metricName: this.defaultDropdownValue,
// dimension: [],
@ -327,6 +337,10 @@ export class AzureMonitorQueryCtrl extends QueryCtrl {
this.target.subscription = this.subscriptions[0].value;
}
if (!this.target.subscriptions) {
this.target.subscriptions = subscriptions.map((sub) => sub.value);
}
return this.subscriptions;
});
}

View File

@ -17,17 +17,20 @@ export enum AzureQueryType {
ApplicationInsights = 'Application Insights',
InsightsAnalytics = 'Insights Analytics',
LogAnalytics = 'Azure Log Analytics',
AzureResourceGraph = 'Azure Resource Graph',
}
export interface AzureMonitorQuery extends DataQuery {
queryType: AzureQueryType;
format: string;
subscription: string;
subscriptions: string[];
azureMonitor: AzureMetricQuery;
azureLogAnalytics: AzureLogsQuery;
appInsights?: ApplicationInsightsQuery;
insightsAnalytics: InsightsAnalyticsQuery;
azureResourceGraph: AzureResourceGraphQuery;
}
export type ConcealedSecret = symbol;
@ -91,6 +94,11 @@ export interface AzureLogsQuery {
workspace: string;
}
export interface AzureResourceGraphQuery {
query: string;
resultFormat: string;
}
export interface ApplicationInsightsQuery {
metricName: string;
timeGrain: string;

View File

@ -6,6 +6,17 @@ import { AzureMonitorOption } from '../types';
export const findOption = (options: AzureMonitorOption[], value: string | undefined) =>
value ? options.find((v) => v.value === value) ?? { value, label: value } : null;
export const findOptions = (options: AzureMonitorOption[], values: string[] = []) => {
if (values.length === 0) {
return [];
}
const set = values.reduce((accum, item) => {
accum.add(item);
return accum;
}, new Set());
return options.filter((option) => set.has(option.value));
};
export const toOption = (v: { text: string; value: string }) => ({ value: v.value, label: v.text });
export function convertTimeGrainsToMs<T extends { value: string }>(timeGrains: T[]) {