diff --git a/public/app/plugins/datasource/loki/backendResultTransformer.test.ts b/public/app/plugins/datasource/loki/backendResultTransformer.test.ts index eb222f1b94f..4364286ccff 100644 --- a/public/app/plugins/datasource/loki/backendResultTransformer.test.ts +++ b/public/app/plugins/datasource/loki/backendResultTransformer.test.ts @@ -37,7 +37,7 @@ const frame: DataFrame = { values: new ArrayVector(['1645029699311000500', '1645029699312000500', '1645029699313000500']), }, ], - length: 1, + length: 3, }; function makeRequest(expr: string): DataQueryRequest { @@ -78,6 +78,16 @@ describe('loki backendResultTransformer', () => { searchWords: ['thing1'], }; expectedFrame.fields[2].type = FieldType.time; + expectedFrame.fields.push({ + name: 'id', + type: FieldType.string, + config: {}, + values: new ArrayVector([ + '6b099923-25a6-5336-96fa-c84a14b7c351_A', + '0e1b7c47-a956-5cf2-a803-d487679745bd_A', + '6f9a840c-6a00-525b-9ed4-cceea29e62af_A', + ]), + }); const expected: DataQueryResponse = { data: [expectedFrame] }; diff --git a/public/app/plugins/datasource/loki/backendResultTransformer.ts b/public/app/plugins/datasource/loki/backendResultTransformer.ts index 2639d95fda2..6d6e73b5644 100644 --- a/public/app/plugins/datasource/loki/backendResultTransformer.ts +++ b/public/app/plugins/datasource/loki/backendResultTransformer.ts @@ -2,6 +2,7 @@ import { DataQueryRequest, DataQueryResponse, DataFrame, isDataFrame, FieldType, import { LokiQuery, LokiQueryType } from './types'; import { makeTableFrames } from './makeTableFrames'; import { formatQuery, getHighlighterExpressionsFromQuery } from './query_utils'; +import { makeIdField } from './makeIdField'; function isMetricFrame(frame: DataFrame): boolean { return frame.fields.every((field) => field.type === FieldType.time || field.type === FieldType.number); @@ -36,6 +37,9 @@ function processStreamFrame(frame: DataFrame, query: LokiQuery | undefined): Dat } }); + // we add a calculated id-field + newFields.push(makeIdField(frame)); + return { ...newFrame, fields: newFields, diff --git a/public/app/plugins/datasource/loki/makeIdField.test.ts b/public/app/plugins/datasource/loki/makeIdField.test.ts new file mode 100644 index 00000000000..04ab5c10e8d --- /dev/null +++ b/public/app/plugins/datasource/loki/makeIdField.test.ts @@ -0,0 +1,87 @@ +import { ArrayVector, DataFrame, FieldType } from '@grafana/data'; +import { makeIdField } from './makeIdField'; + +function makeFrame(timestamps: number[], values: string[], timestampNss: string[], refId?: string): DataFrame { + return { + name: 'frame', + refId, + meta: { + executedQueryString: 'something1', + }, + fields: [ + { + name: 'Time', + type: FieldType.time, + config: {}, + values: new ArrayVector(timestamps), + }, + { + name: 'Value', + type: FieldType.string, + config: {}, + labels: { + foo: 'bar', + }, + values: new ArrayVector(values), + }, + { + name: 'tsNs', + type: FieldType.time, + config: {}, + values: new ArrayVector(timestampNss), + }, + ], + length: timestamps.length, + }; +} + +describe('loki makeIdField', () => { + it('should always generate unique ids for logs', () => { + const frame = makeFrame( + [1579857562021, 1579857562021, 1579857562021, 1579857562021], + [ + 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"', + 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"', + 't=2020-02-12T15:04:51+0000 lvl=info msg="Non-Duplicated"', + 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"', + ], + ['1579857562021616000', '1579857562021616000', '1579857562021616000', '1579857562021616000'] + ); + expect(makeIdField(frame)).toEqual({ + config: {}, + name: 'id', + type: 'string', + values: new ArrayVector([ + '75fceace-9f98-5134-b222-643fdcde2877', + '75fceace-9f98-5134-b222-643fdcde2877_1', + '4a081a89-040d-5f64-9477-a4d846ce9f6b', + '75fceace-9f98-5134-b222-643fdcde2877_2', + ]), + }); + }); + + it('should append refId to the unique ids if refId is provided', () => { + const frame = makeFrame( + [1579857562021, 1579857562021, 1579857562021, 1579857562021], + [ + 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"', + 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"', + 't=2020-02-12T15:04:51+0000 lvl=info msg="Non-Duplicated"', + 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"', + ], + ['1579857562021616000', '1579857562021616000', '1579857562021616000', '1579857562021616000'], + 'X' + ); + expect(makeIdField(frame)).toEqual({ + config: {}, + name: 'id', + type: 'string', + values: new ArrayVector([ + '75fceace-9f98-5134-b222-643fdcde2877_X', + '75fceace-9f98-5134-b222-643fdcde2877_1_X', + '4a081a89-040d-5f64-9477-a4d846ce9f6b_X', + '75fceace-9f98-5134-b222-643fdcde2877_2_X', + ]), + }); + }); +}); diff --git a/public/app/plugins/datasource/loki/makeIdField.ts b/public/app/plugins/datasource/loki/makeIdField.ts new file mode 100644 index 00000000000..02550b07a62 --- /dev/null +++ b/public/app/plugins/datasource/loki/makeIdField.ts @@ -0,0 +1,54 @@ +import { v5 as uuidv5 } from 'uuid'; + +import { ArrayVector, DataFrame, Field, FieldType, Labels } from '@grafana/data'; + +const UUID_NAMESPACE = '6ec946da-0f49-47a8-983a-1d76d17e7c92'; + +function createUid(text: string, usedUids: Map, refId?: string): string { + const id = uuidv5(text, UUID_NAMESPACE); + + // check how many times have we seen this id before, + // set the count to zero, if never. + const count = usedUids.get(id) ?? 0; + + // if we have seen this id before, we need to make + // it unique by appending the seen-count + // (starts with 1, and goes up) + const uniqueId = count > 0 ? `${id}_${count}` : id; + + // we increment the counter for this id, to be used when we are called the next time + usedUids.set(id, count + 1); + + // we add refId to the end, if it is available + return refId !== undefined ? `${uniqueId}_${refId}` : uniqueId; +} + +export function makeIdField(frame: DataFrame): Field { + const allLabels: Labels = {}; + + // collect labels from every field + frame.fields.forEach((field) => { + Object.assign(allLabels, field.labels); + }); + + const labelsString = Object.entries(allLabels) + .map(([key, val]) => `${key}="${val}"`) + .sort() + .join(''); + + const usedUids = new Map(); + + const { length } = frame; + + const uids: string[] = new Array(length); + + // we need to go through the dataframe "row by row" + for (let i = 0; i < length; i++) { + const row = frame.fields.map((f) => String(f.values.get(i))); + const text = `${labelsString}_${row.join('_')}`; + const uid = createUid(text, usedUids, frame.refId); + uids[i] = uid; + } + + return { name: 'id', type: FieldType.string, config: {}, values: new ArrayVector(uids) }; +}