IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Elasticsearch: Create Raw Doc metric to render raw JSON docs in columns in the new table panel (#26233)

Browse Source 
* test

* WIP: Create v2 version

* Update tests, remove conosole logs, refactor

* Remove incorrect types

* Update type

* Rename legacy and new metrics

* Update

* Run request when Raw Data tto Raw Document switch

* Fix size updating

* Remove _source field from table results as we are showing each source field as column

* Remove _source just for metrics, not logs

* Revert "Remove _source just for metrics, not logs"

This reverts commit 611b6922f7.

* Revert "Remove _source field from table results as we are showing each source field as column"

This reverts commit 31a9d5f81b.

* Add vis preference for logs

* Update visualisation to logs

* Revert "Revert "Remove _source just for metrics""

This reverts commit a102ab2894.

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>
This commit is contained in:
Ivana Huckova
2020-07-15 15:20:39 +02:00
committed by GitHub
parent 5b2ff4498e
commit 3fd810417f
7 changed files with 110 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
public/app/plugins/datasource/elasticsearch/elastic_response.ts
View File

@@ -391,6 +391,88 @@ export class ElasticResponse {
}
getTimeSeries() {
if (this.targets.some((target: any) => target.metrics.some((metric: any) => metric.type === 'raw_data'))) {
return this.processResponseToDataFrames(false);
}
return this.processResponseToSeries();
}
getLogs(logMessageField?: string, logLevelField?: string): DataQueryResponse {
return this.processResponseToDataFrames(true, logMessageField, logLevelField);
}
processResponseToDataFrames(
isLogsRequest: boolean,
logMessageField?: string,
logLevelField?: string
): DataQueryResponse {
const dataFrame: DataFrame[] = [];
for (let n = 0; n < this.response.responses.length; n++) {
const response = this.response.responses[n];
if (response.error) {
throw this.getErrorFromElasticResponse(this.response, response.error);
}
if (response.hits && response.hits.hits.length > 0) {
const { propNames, docs } = flattenHits(response.hits.hits);
if (docs.length > 0) {
let series = createEmptyDataFrame(
propNames,
this.targets[0].timeField,
isLogsRequest,
logMessageField,
logLevelField
);
// Add a row for each document
for (const doc of docs) {
if (logLevelField) {
// Remap level field based on the datasource config. This field is then used in explore to figure out the
// log level. We may rewrite some actual data in the level field if they are different.
doc['level'] = doc[logLevelField];
}
series.add(doc);
}
if (isLogsRequest) {
series = addPreferredVisualisationType(series, 'logs');
}
dataFrame.push(series);
}
}
if (response.aggregations) {
const aggregations = response.aggregations;
const target = this.targets[n];
const tmpSeriesList: any[] = [];
const table = new TableModel();
this.processBuckets(aggregations, target, tmpSeriesList, table, {}, 0);
this.trimDatapoints(tmpSeriesList, target);
this.nameSeries(tmpSeriesList, target);
if (table.rows.length > 0) {
dataFrame.push(toDataFrame(table));
}
for (let y = 0; y < tmpSeriesList.length; y++) {
let series = toDataFrame(tmpSeriesList[y]);
// When log results, show aggregations only in graph. Log fields are then going to be shown in table.
if (isLogsRequest) {
series = addPreferredVisualisationType(series, 'graph');
}
dataFrame.push(series);
}
}
}
return { data: dataFrame };
}
processResponseToSeries = () => {
const seriesList = [];
for (let i = 0; i < this.response.responses.length; i++) {
@@ -424,59 +506,7 @@ export class ElasticResponse {
}
return { data: seriesList };
}
getLogs(logMessageField?: string, logLevelField?: string): DataQueryResponse {
const dataFrame: DataFrame[] = [];
for (let n = 0; n < this.response.responses.length; n++) {
const response = this.response.responses[n];
if (response.error) {
throw this.getErrorFromElasticResponse(this.response, response.error);
}
const { propNames, docs } = flattenHits(response.hits.hits);
if (docs.length > 0) {
let series = createEmptyDataFrame(propNames, this.targets[0].timeField, logMessageField, logLevelField);
// Add a row for each document
for (const doc of docs) {
if (logLevelField) {
// Remap level field based on the datasource config. This field is then used in explore to figure out the
// log level. We may rewrite some actual data in the level field if they are different.
doc['level'] = doc[logLevelField];
}
series.add(doc);
}
series = addPreferredVisualisationType(series, 'logs');
dataFrame.push(series);
}
if (response.aggregations) {
const aggregations = response.aggregations;
const target = this.targets[n];
const tmpSeriesList: any[] = [];
const table = new TableModel();
this.processBuckets(aggregations, target, tmpSeriesList, table, {}, 0);
this.trimDatapoints(tmpSeriesList, target);
this.nameSeries(tmpSeriesList, target);
for (let y = 0; y < tmpSeriesList.length; y++) {
let series = toDataFrame(tmpSeriesList[y]);
// When log results, show aggregations only in graph. Log fields are then going to be shown in table.
series = addPreferredVisualisationType(series, 'graph');
dataFrame.push(series);
}
}
}
return { data: dataFrame };
}
};
}
type Doc = {
@@ -532,6 +562,7 @@ const flattenHits = (hits: Doc[]): { docs: Array<Record<string, any>>; propNames
const createEmptyDataFrame = (
propNames: string[],
timeField: string,
isLogsRequest: boolean,
logMessageField?: string,
logLevelField?: string
): MutableDataFrame => {
@@ -549,13 +580,6 @@ const createEmptyDataFrame = (
}).parse = (v: any) => {
return v || '';
};
} else {
series.addField({
name: '_source',
type: FieldType.string,
}).parse = (v: any) => {
return JSON.stringify(v, null, 2);
};
}
if (logLevelField) {
@@ -574,6 +598,10 @@ const createEmptyDataFrame = (
if (fieldNames.includes(propName)) {
continue;
}
// Do not add _source field (besides logs) as we are showing each _source field in table instead.
if (!isLogsRequest && propName === '_source') {
continue;
}
series.addField({
name: propName,

8
public/app/plugins/datasource/elasticsearch/metric_agg.ts
View File

@@ -101,7 +101,8 @@ export class ElasticMetricAggCtrl {
$scope.updateMovingAvgModelSettings();
break;
}
case 'raw_document': {
case 'raw_document':
case 'raw_data': {
$scope.agg.settings.size = $scope.agg.settings.size || 500;
$scope.settingsLinkText = 'Size: ' + $scope.agg.settings.size;
$scope.target.metrics.splice(0, $scope.target.metrics.length, $scope.agg);
@@ -164,7 +165,10 @@ export class ElasticMetricAggCtrl {
$scope.showOptions = false;
// reset back to metric/group by query
if ($scope.target.bucketAggs.length === 0 && $scope.agg.type !== 'raw_document') {
if (
$scope.target.bucketAggs.length === 0 &&
($scope.agg.type !== 'raw_document' || $scope.agg.type !== 'raw_data')
) {
$scope.target.bucketAggs = [queryDef.defaultBucketAgg()];
}

2
public/app/plugins/datasource/elasticsearch/partials/metric_agg.html
View File

@@ -101,7 +101,7 @@
<label class="gf-form-label width-10">Percentiles</label>
<input type="text" class="gf-form-input max-width-12" ng-model="agg.settings.percents" array-join ng-blur="onChange()"></input>
</div>
<div class="gf-form offset-width-7" ng-if="agg.type === 'raw_document'">
<div class="gf-form offset-width-7" ng-if="agg.type === 'raw_document' || agg.type === 'raw_data'">
<label class="gf-form-label width-10">Size</label>
<input type="number" class="gf-form-input max-width-12" ng-model="agg.settings.size" ng-blur="onChange()"></input>
</div>

5
public/app/plugins/datasource/elasticsearch/query_builder.ts
View File

@@ -212,7 +212,8 @@ export class ElasticQueryBuilder {
// If target doesn't have bucketAggs and type is not raw_document, it is invalid query.
if (target.bucketAggs.length === 0) {
metric = target.metrics[0];
if (!metric || metric.type !== 'raw_document') {
if (!metric || !(metric.type === 'raw_document' || metric.type === 'raw_data')) {
throw { message: 'Invalid query' };
}
}
@@ -221,7 +222,7 @@ export class ElasticQueryBuilder {
* Check if metric type is raw_document. If metric doesn't have size (or size is 0), update size to 500.
* Otherwise it will not be a valid query and error will be thrown.
*/
if (target.metrics?.[0]?.type === 'raw_document') {
if (target.metrics?.[0]?.type === 'raw_document' || target.metrics?.[0]?.type === 'raw_data') {
metric = target.metrics[0];
const size = (metric.settings && metric.settings.size !== 0 && metric.settings.size) || 500;
return this.documentQuery(query, size);

6
public/app/plugins/datasource/elasticsearch/query_ctrl.ts
View File

@@ -51,9 +51,15 @@ export class ElasticQueryCtrl extends QueryCtrl {
}
queryUpdated() {
// As Raw Data and Raw Document have the same request, we need to run refresh if they are updated
const isPossiblyRawDataSwitch = this.target.metrics.some(
(metric: any) => metric.type === 'raw_data' || metric.type === 'raw_document'
);
const newJson = angular.toJson(this.datasource.queryBuilder.build(this.target), true);
if (this.rawQueryOld && newJson !== this.rawQueryOld) {
this.refresh();
} else if (isPossiblyRawDataSwitch) {
this.refresh();
}
this.rawQueryOld = newJson;

3
public/app/plugins/datasource/elasticsearch/query_def.ts
View File

@@ -79,7 +79,8 @@ export const metricAggTypes = [
supportsMultipleBucketPaths: true,
minVersion: 2,
},
{ text: 'Raw Document', value: 'raw_document', requiresField: false },
{ text: 'Raw Document (legacy)', value: 'raw_document', requiresField: false },
{ text: 'Raw Data', value: 'raw_data', requiresField: false },
{ text: 'Logs', value: 'logs', requiresField: false },
];

8
public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts
View File

@@ -92,25 +92,25 @@ describe('ElasticQueryDef', () => {
describe('pipeline aggs depending on esverison', () => {
describe('using esversion undefined', () => {
test('should not get pipeline aggs', () => {
expect(queryDef.getMetricAggTypes(undefined).length).toBe(10);
expect(queryDef.getMetricAggTypes(undefined).length).toBe(11);
});
});
describe('using esversion 1', () => {
test('should not get pipeline aggs', () => {
expect(queryDef.getMetricAggTypes(1).length).toBe(10);
expect(queryDef.getMetricAggTypes(1).length).toBe(11);
});
});
describe('using esversion 2', () => {
test('should get pipeline aggs', () => {
expect(queryDef.getMetricAggTypes(2).length).toBe(14);
expect(queryDef.getMetricAggTypes(2).length).toBe(15);
});
});
describe('using esversion 5', () => {
test('should get pipeline aggs', () => {
expect(queryDef.getMetricAggTypes(5).length).toBe(14);
expect(queryDef.getMetricAggTypes(5).length).toBe(15);
});
});
});