IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

azuremonitor: port azure log analytics query function to the backend (#23839)

Browse Source 
* azuremonitor: add support for log analytics macros

Also adds tests for the kql macros

* azuremonitor: backend implementation for Log Analytics

* azuremonitor: remove gzip header from plugin route

The Go net/http library adds an accept encoding header
for gzip automatically.

https://golang.org/src/net/http/transport.go\#L2454

So no need to specify it manually

* azuremonitor: parses log analytics time series

* azuremonitor: support for table data for Log Analytics

* azuremonitor: for log analytics switch to calling the API...

...from the backend for time series and table queries.

* azuremonitor: fix missing err check

* azuremonitor: support Azure China, Azure Gov...

for log analytics on the backend.

* azuremonitor: review fixes

* azuremonitor: rename test files folder to testdata

To follow Go conventions for test data in tests

* azuremonitor: review fixes

* azuremonitor: better error message for http requests

* azuremonitor: fix for load workspaces on config page

* azuremonitor: strict null check fixes

Co-authored-by: bergquist <carl.bergquist@gmail.com>
This commit is contained in:
Daniel Lee
2020-04-27 17:43:02 +02:00
committed by GitHub
parent 458f6bdb87
commit c05049f395
32 changed files with 1460 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
public/app/plugins/datasource/grafana-azure-monitor-datasource/app_insights/app_insights_datasource.ts
View File

@@ -23,9 +23,9 @@ export default class AppInsightsDatasource {
/** @ngInject */
constructor(instanceSettings: DataSourceInstanceSettings<AzureDataSourceJsonData>, private templateSrv: TemplateSrv) {
this.id = instanceSettings.id;
this.applicationId = instanceSettings.jsonData.appInsightsAppId;
this.applicationId = instanceSettings.jsonData.appInsightsAppId || '';
switch (instanceSettings.jsonData.cloudName) {
switch (instanceSettings.jsonData?.cloudName) {
// Azure US Government
case 'govazuremonitor':
break;
@@ -41,7 +41,7 @@ export default class AppInsightsDatasource {
this.baseUrl = `/appinsights/${this.version}/apps/${this.applicationId}`;
}
this.url = instanceSettings.url;
this.url = instanceSettings.url || '';
}
isConfigured(): boolean {

120
public/app/plugins/datasource/grafana-azure-monitor-datasource/azure_log_analytics/azure_log_analytics_datasource.test.ts
View File

@@ -1,6 +1,5 @@
import AzureMonitorDatasource from '../datasource';
import FakeSchemaData from './__mocks__/schema';
import { TemplateSrv } from 'app/features/templating/template_srv';
import { KustoSchema, AzureLogsVariable } from '../types';
import { toUtc } from '@grafana/data';
@@ -147,114 +146,55 @@ describe('AzureLogAnalyticsDatasource', () => {
};
const response = {
tables: [
{
name: 'PrimaryResult',
columns: [
results: {
A: {
refId: 'A',
meta: {
columns: ['TimeGenerated', 'Computer', 'avg_CounterValue'],
query:
'Perf\r\n| where ObjectName == "Memory" and CounterName == "Available MBytes Memory"\n| where TimeGenerated >= datetime(\'2020-04-23T09:15:20Z\') and TimeGenerated <= datetime(\'2020-04-23T09:20:20Z\')\n| where 1 == 1\n| summarize avg(CounterValue) by bin(TimeGenerated, 1m), Computer \n| order by TimeGenerated asc',
},
series: [
{
name: 'TimeGenerated',
type: 'datetime',
name: 'grafana-vm',
points: [
[2017.25, 1587633300000],
[2048, 1587633360000],
[2048.3333333333335, 1587633420000],
[2049, 1587633480000],
[2049, 1587633540000],
[2049, 1587633600000],
],
},
{
name: 'Category',
type: 'string',
},
{
name: 'count_',
type: 'long',
},
],
rows: [
['2018-06-02T20:20:00Z', 'Administrative', 2],
['2018-06-02T20:25:00Z', 'Administrative', 22],
['2018-06-02T20:30:00Z', 'Policy', 20],
],
},
],
},
};
describe('in time series format', () => {
describe('and the data is valid (has time, metric and value columns)', () => {
beforeEach(() => {
datasourceRequestMock.mockImplementation((options: { url: string }) => {
expect(options.url).toContain('query=AzureActivity');
expect(options.url).toContain('/api/tsdb/query');
return Promise.resolve({ data: response, status: 200 });
});
});
it('should return a list of datapoints', () => {
return ctx.ds.query(options).then((results: any) => {
expect(results.data.length).toBe(2);
expect(results.data[0].datapoints.length).toBe(2);
expect(results.data[0].target).toEqual('Administrative');
expect(results.data[0].datapoints[0][1]).toEqual(1527970800000);
expect(results.data[0].datapoints[0][0]).toEqual(2);
expect(results.data[0].datapoints[1][1]).toEqual(1527971100000);
expect(results.data[0].datapoints[1][0]).toEqual(22);
expect(results.data.length).toBe(1);
expect(results.data[0].name).toEqual('grafana-vm');
expect(results.data[0].fields.length).toBe(2);
expect(results.data[0].fields[0].name).toBe('Time');
expect(results.data[0].fields[1].name).toBe('grafana-vm');
expect(results.data[0].fields[0].values.toArray().length).toBe(6);
expect(results.data[0].fields[0].values.get(0)).toEqual(1587633300000);
expect(results.data[0].fields[1].values.get(0)).toEqual(2017.25);
expect(results.data[0].fields[0].values.get(1)).toEqual(1587633360000);
expect(results.data[0].fields[1].values.get(1)).toEqual(2048);
});
});
});
describe('and the data has no time column)', () => {
beforeEach(() => {
const invalidResponse = {
tables: [
{
name: 'PrimaryResult',
columns: [
{
name: 'Category',
type: 'string',
},
{
name: 'count_',
type: 'long',
},
],
rows: [['Administrative', 2]],
},
],
};
datasourceRequestMock.mockImplementation((options: { url: string }) => {
expect(options.url).toContain('query=AzureActivity');
return Promise.resolve({ data: invalidResponse, status: 200 });
});
});
it('should throw an exception', () => {
ctx.ds.query(options).catch((err: any) => {
expect(err.message).toContain('The Time Series format requires a time column.');
});
});
});
});
describe('in tableformat', () => {
beforeEach(() => {
options.targets[0].azureLogAnalytics.resultFormat = 'table';
datasourceRequestMock.mockImplementation((options: { url: string }) => {
expect(options.url).toContain('query=AzureActivity');
return Promise.resolve({ data: response, status: 200 });
});
});
it('should return a list of columns and rows', () => {
return ctx.ds.query(options).then((results: any) => {
expect(results.data[0].type).toBe('table');
expect(results.data[0].columns.length).toBe(3);
expect(results.data[0].rows.length).toBe(3);
expect(results.data[0].columns[0].text).toBe('TimeGenerated');
expect(results.data[0].columns[0].type).toBe('datetime');
expect(results.data[0].columns[1].text).toBe('Category');
expect(results.data[0].columns[1].type).toBe('string');
expect(results.data[0].columns[2].text).toBe('count_');
expect(results.data[0].columns[2].type).toBe('long');
expect(results.data[0].rows[0][0]).toEqual('2018-06-02T20:20:00Z');
expect(results.data[0].rows[0][1]).toEqual('Administrative');
expect(results.data[0].rows[0][2]).toEqual(2);
});
});
});
});

72
public/app/plugins/datasource/grafana-azure-monitor-datasource/azure_log_analytics/azure_log_analytics_datasource.ts
View File

@@ -2,7 +2,8 @@ import _ from 'lodash';
import LogAnalyticsQuerystringBuilder from '../log_analytics/querystring_builder';
import ResponseParser from './response_parser';
import { AzureMonitorQuery, AzureDataSourceJsonData, AzureLogsVariable } from '../types';
import { DataQueryRequest, DataSourceInstanceSettings } from '@grafana/data';
import { TimeSeries, toDataFrame } from '@grafana/data';
import { DataQueryRequest, DataQueryResponseData, DataSourceInstanceSettings } from '@grafana/data';
import { getBackendSrv } from '@grafana/runtime';
import { TemplateSrv } from 'app/features/templating/template_srv';
@@ -24,10 +25,11 @@ export default class AzureLogAnalyticsDatasource {
switch (this.instanceSettings.jsonData.cloudName) {
case 'govazuremonitor': // Azure US Government
this.baseUrl = '/govloganalyticsazure';
break;
case 'germanyazuremonitor': // Azure Germany
break;
case 'chinaazuremonitor': // Azue China
case 'chinaazuremonitor': // Azure China
this.baseUrl = '/chinaloganalyticsazure';
break;
default:
@@ -35,8 +37,8 @@ export default class AzureLogAnalyticsDatasource {
this.baseUrl = '/loganalyticsazure';
}
this.url = instanceSettings.url;
this.defaultOrFirstWorkspace = this.instanceSettings.jsonData.logAnalyticsDefaultWorkspace;
this.url = instanceSettings.url || '';
this.defaultOrFirstWorkspace = this.instanceSettings.jsonData.logAnalyticsDefaultWorkspace || '';
this.setWorkspaceUrl();
}
@@ -59,10 +61,11 @@ export default class AzureLogAnalyticsDatasource {
switch (this.instanceSettings.jsonData.cloudName) {
case 'govazuremonitor': // Azure US Government
this.azureMonitorUrl = `/govworkspacesloganalytics/subscriptions`;
break;
case 'germanyazuremonitor': // Azure Germany
break;
case 'chinaazuremonitor': // Azue China
case 'chinaazuremonitor': // Azure China
this.azureMonitorUrl = `/chinaworkspacesloganalytics/subscriptions`;
break;
default:
@@ -91,7 +94,7 @@ export default class AzureLogAnalyticsDatasource {
if (!workspace) {
return Promise.resolve();
}
const url = `${this.baseUrl}/${workspace}/metadata`;
const url = `${this.baseUrl}/${this.templateSrv.replace(workspace, {})}/metadata`;
return this.doRequest(url).then((response: any) => {
return new ResponseParser(response.data).parseSchemaResult();
@@ -104,42 +107,65 @@ export default class AzureLogAnalyticsDatasource {
}).map(target => {
const item = target.azureLogAnalytics;
const querystringBuilder = new LogAnalyticsQuerystringBuilder(
this.templateSrv.replace(item.query, options.scopedVars, this.interpolateVariable),
options,
'TimeGenerated'
);
const generated = querystringBuilder.generate();
let workspace = this.templateSrv.replace(item.workspace, options.scopedVars);
if (!workspace && this.defaultOrFirstWorkspace) {
workspace = this.defaultOrFirstWorkspace;
}
const url = `${this.baseUrl}/${workspace}/query?${generated.uriString}`;
const subscriptionId = this.templateSrv.replace(target.subscription || this.subscriptionId, options.scopedVars);
const query = this.templateSrv.replace(item.query, options.scopedVars, this.interpolateVariable);
return {
refId: target.refId,
intervalMs: options.intervalMs,
maxDataPoints: options.maxDataPoints,
datasourceId: this.id,
url: url,
query: generated.rawQuery,
format: target.format,
resultFormat: item.resultFormat,
queryType: 'Azure Log Analytics',
subscriptionId: subscriptionId,
azureLogAnalytics: {
resultFormat: item.resultFormat,
query: query,
workspace: workspace,
},
};
});
if (!queries || queries.length === 0) {
return;
return [];
}
const promises = this.doQueries(queries);
return Promise.all(promises).then(results => {
return new ResponseParser(results).parseQueryResult();
const { data } = await getBackendSrv().datasourceRequest({
url: '/api/tsdb/query',
method: 'POST',
data: {
from: options.range.from.valueOf().toString(),
to: options.range.to.valueOf().toString(),
queries,
},
});
const result: DataQueryResponseData[] = [];
if (data.results) {
Object.values(data.results).forEach((queryRes: any) => {
queryRes.series?.forEach((series: any) => {
const timeSeries: TimeSeries = {
target: series.name,
datapoints: series.points,
refId: queryRes.refId,
meta: queryRes.meta,
};
result.push(toDataFrame(timeSeries));
});
queryRes.tables?.forEach((table: any) => {
result.push(toDataFrame(table));
});
});
}
return result;
}
metricFindQuery(query: string) {
@@ -363,7 +389,7 @@ export default class AzureLogAnalyticsDatasource {
return undefined;
}
isValidConfigField(field: string) {
isValidConfigField(field: string | undefined) {
return field && field.length > 0;
}
}

2
public/app/plugins/datasource/grafana-azure-monitor-datasource/components/ConfigEditor.tsx
View File

@@ -170,7 +170,7 @@ export class ConfigEditor extends PureComponent<Props, State> {
let azureMonitorUrl = '',
subscriptionId = this.templateSrv.replace(subscription || this.props.options.jsonData.subscriptionId);
if (!!subscriptionId || !!azureLogAnalyticsSameAs) {
if (azureLogAnalyticsSameAs) {
const azureCloud = cloudName || 'azuremonitor';
azureMonitorUrl = `/${azureCloud}/subscriptions`;
} else {

39
public/app/plugins/datasource/grafana-azure-monitor-datasource/plugin.json
View File

@@ -137,6 +137,21 @@
},
"headers": [{ "name": "x-ms-app", "content": "Grafana" }]
},
{
"path": "govworkspacesloganalytics",
"method": "GET",
"url": "https://management.usgovcloudapi.net",
"tokenAuth": {
"url": "https://login.microsoftonline.us/{{.JsonData.logAnalyticsTenantId}}/oauth2/token",
"params": {
"grant_type": "client_credentials",
"client_id": "{{.JsonData.logAnalyticsClientId}}",
"client_secret": "{{.SecureJsonData.logAnalyticsClientSecret}}",
"resource": "https://management.usgovcloudapi.net/"
}
},
"headers": [{ "name": "x-ms-app", "content": "Grafana" }]
},
{
"path": "loganalyticsazure",
"method": "GET",
@@ -152,8 +167,7 @@
},
"headers": [
{ "name": "x-ms-app", "content": "Grafana" },
{ "name": "Cache-Control", "content": "public, max-age=60" },
{ "name": "Accept-Encoding", "content": "gzip" }
{ "name": "Cache-Control", "content": "public, max-age=60" }
]
},
{
@@ -171,8 +185,25 @@
},
"headers": [
{ "name": "x-ms-app", "content": "Grafana" },
{ "name": "Cache-Control", "content": "public, max-age=60" },
{ "name": "Accept-Encoding", "content": "gzip" }
{ "name": "Cache-Control", "content": "public, max-age=60" }
]
},
{
"path": "govloganalyticsazure",
"method": "GET",
"url": "https://api.loganalytics.us/v1/workspaces",
"tokenAuth": {
"url": "https://login.microsoftonline.us/{{.JsonData.logAnalyticsTenantId}}/oauth2/token",
"params": {
"grant_type": "client_credentials",
"client_id": "{{.JsonData.logAnalyticsClientId}}",
"client_secret": "{{.SecureJsonData.logAnalyticsClientSecret}}",
"resource": "https://api.loganalytics.us"
}
},
"headers": [
{ "name": "x-ms-app", "content": "Grafana" },
{ "name": "Cache-Control", "content": "public, max-age=60" }
]
}
],