IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2026-07-29 15:59:50 -05:00
Code Issues Packages Projects Releases Wiki Activity

Transform: improve the "outer join" performance/behavior (#30407)

Browse Source
This commit is contained in:
Ryan McKinley
2021-01-28 14:00:28 -08:00
committed by GitHub
parent 3390c6a852
commit db9a8bf04a
9 changed files with 1038 additions and 919 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/grafana-data/src/transformations/index.ts
+1 -1
View File
@@ -11,4 +11,4 @@ export {
} from './standardTransformersRegistry';
export { RegexpOrNamesMatcherOptions, ByNamesMatcherOptions, ByNamesMatcherMode } from './matchers/nameMatcher';
export { RenameByRegexTransformerOptions } from './transformers/renameByRegex';
export { outerJoinDataFrames } from './transformers/seriesToColumns';
export { outerJoinDataFrames } from './transformers/joinDataFrames';
packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts
+69 -39
View File
@@ -49,52 +49,82 @@ describe('ensureColumns transformer', () => {
const frame = filtered[0];
expect(frame.fields.length).toEqual(5);
expect(filtered[0]).toEqual(
toDataFrame({
fields: [
{
name: 'TheTime',
type: 'time',
config: {},
values: [1000, 2000],
labels: undefined,
expect(filtered[0]).toMatchInlineSnapshot(`
Object {
"fields": Array [
Object {
"config": Object {},
"name": "TheTime",
"state": Object {
"displayName": "TheTime",
},
"type": "time",
"values": Array [
1000,
2000,
],
},
{
name: 'A',
type: 'number',
config: {},
values: [1, 100],
labels: {},
Object {
"config": Object {},
"labels": Object {},
"name": "A",
"state": Object {
"displayName": "A",
},
"type": "number",
"values": Array [
1,
100,
],
},
{
name: 'B',
type: 'number',
config: {},
values: [2, 200],
labels: {},
Object {
"config": Object {},
"labels": Object {},
"name": "B",
"state": Object {
"displayName": "B",
},
"type": "number",
"values": Array [
2,
200,
],
},
{
name: 'C',
type: 'number',
config: {},
values: [3, 300],
labels: {},
Object {
"config": Object {},
"labels": Object {},
"name": "C",
"state": Object {
"displayName": "C",
},
"type": "number",
"values": Array [
3,
300,
],
},
{
name: 'D',
type: 'string',
config: {},
values: ['first', 'second'],
labels: {},
Object {
"config": Object {},
"labels": Object {},
"name": "D",
"state": Object {
"displayName": "D",
},
"type": "string",
"values": Array [
"first",
"second",
],
},
],
meta: {
transformations: ['ensureColumns'],
"length": 2,
"meta": Object {
"transformations": Array [
"ensureColumns",
],
},
name: undefined,
refId: undefined,
})
);
}
`);
});
});
packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts
+302
View File
@@ -0,0 +1,302 @@
import { toDataFrame } from '../../dataframe/processDataFrame';
import { FieldType } from '../../types/dataFrame';
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { ArrayVector } from '../../vector';
import { calculateFieldTransformer } from './calculateField';
import { isLikelyAscendingVector, outerJoinDataFrames } from './joinDataFrames';
describe('align frames', () => {
beforeAll(() => {
mockTransformationsRegistry([calculateFieldTransformer]);
});
it('by first time field', () => {
const series1 = toDataFrame({
fields: [
{ name: 'TheTime', type: FieldType.time, values: [1000, 2000] },
{ name: 'A', type: FieldType.number, values: [1, 100] },
],
});
const series2 = toDataFrame({
fields: [
{ name: '_time', type: FieldType.time, values: [1000, 1500, 2000] },
{ name: 'A', type: FieldType.number, values: [2, 20, 200] },
{ name: 'B', type: FieldType.number, values: [3, 30, 300] },
{ name: 'C', type: FieldType.string, values: ['first', 'second', 'third'] },
],
});
const out = outerJoinDataFrames({ frames: [series1, series2] })!;
expect(
out.fields.map((f) => ({
name: f.name,
values: f.values.toArray(),
}))
).toMatchInlineSnapshot(`
Array [
Object {
"name": "TheTime",
"values": Array [
1000,
1500,
2000,
],
},
Object {
"name": "A",
"values": Array [
1,
undefined,
100,
],
},
Object {
"name": "A",
"values": Array [
2,
20,
200,
],
},
Object {
"name": "B",
"values": Array [
3,
30,
300,
],
},
Object {
"name": "C",
"values": Array [
"first",
"second",
"third",
],
},
]
`);
});
it('unsorted input keep indexes', () => {
//----------
const series1 = toDataFrame({
fields: [
{ name: 'TheTime', type: FieldType.time, values: [1000, 2000, 1500] },
{ name: 'A1', type: FieldType.number, values: [1, 2, 15] },
],
});
const series3 = toDataFrame({
fields: [
{ name: 'Time', type: FieldType.time, values: [2000, 1000] },
{ name: 'A2', type: FieldType.number, values: [2, 1] },
],
});
let out = outerJoinDataFrames({ frames: [series1, series3], keepOriginIndices: true })!;
expect(
out.fields.map((f) => ({
name: f.name,
values: f.values.toArray(),
state: f.state,
}))
).toMatchInlineSnapshot(`
Array [
Object {
"name": "TheTime",
"state": Object {
"displayName": "TheTime",
"origin": Object {
"fieldIndex": 0,
"frameIndex": 0,
},
},
"values": Array [
1000,
1500,
2000,
],
},
Object {
"name": "A1",
"state": Object {
"displayName": "A1",
"origin": Object {
"fieldIndex": 1,
"frameIndex": 0,
},
},
"values": Array [
1,
15,
2,
],
},
Object {
"name": "A2",
"state": Object {
"displayName": "A2",
"origin": Object {
"fieldIndex": 1,
"frameIndex": 1,
},
},
"values": Array [
1,
undefined,
2,
],
},
]
`);
// Fast path still adds origin indecies
out = outerJoinDataFrames({ frames: [series1], keepOriginIndices: true })!;
expect(
out.fields.map((f) => ({
name: f.name,
state: f.state,
}))
).toMatchInlineSnapshot(`
Array [
Object {
"name": "TheTime",
"state": Object {
"displayName": "TheTime",
"origin": Object {
"fieldIndex": 0,
"frameIndex": 0,
},
},
},
Object {
"name": "A1",
"state": Object {
"displayName": "A1",
"origin": Object {
"fieldIndex": 1,
"frameIndex": 0,
},
},
},
]
`);
});
it('sort single frame', () => {
const series1 = toDataFrame({
fields: [
{ name: 'TheTime', type: FieldType.time, values: [6000, 2000, 1500] },
{ name: 'A1', type: FieldType.number, values: [1, 22, 15] },
],
});
const out = outerJoinDataFrames({ frames: [series1], enforceSort: true, keepOriginIndices: true })!;
expect(
out.fields.map((f) => ({
name: f.name,
values: f.values.toArray(),
}))
).toMatchInlineSnapshot(`
Array [
Object {
"name": "TheTime",
"values": Array [
1500,
2000,
6000,
],
},
Object {
"name": "A1",
"values": Array [
15,
22,
1,
],
},
]
`);
});
it('supports duplicate times', () => {
//----------
// NOTE!!!
// * ideally we would *keep* dupicate fields
//----------
const series1 = toDataFrame({
fields: [
{ name: 'TheTime', type: FieldType.time, values: [1000, 2000] },
{ name: 'A', type: FieldType.number, values: [1, 100] },
],
});
const series3 = toDataFrame({
fields: [
{ name: 'Time', type: FieldType.time, values: [1000, 1000, 1000] },
{ name: 'A', type: FieldType.number, values: [2, 20, 200] },
],
});
const out = outerJoinDataFrames({ frames: [series1, series3] })!;
expect(
out.fields.map((f) => ({
name: f.name,
values: f.values.toArray(),
}))
).toMatchInlineSnapshot(`
Array [
Object {
"name": "TheTime",
"values": Array [
1000,
2000,
],
},
Object {
"name": "A",
"values": Array [
1,
100,
],
},
Object {
"name": "A",
"values": Array [
200,
undefined,
],
},
]
`);
});
describe('check ascending data', () => {
it('simple ascending', () => {
const v = new ArrayVector([1, 2, 3, 4, 5]);
expect(isLikelyAscendingVector(v)).toBeTruthy();
});
it('simple ascending with null', () => {
const v = new ArrayVector([null, 2, 3, 4, null]);
expect(isLikelyAscendingVector(v)).toBeTruthy();
});
it('single value', () => {
const v = new ArrayVector([null, null, null, 4, null]);
expect(isLikelyAscendingVector(v)).toBeTruthy();
expect(isLikelyAscendingVector(new ArrayVector([4]))).toBeTruthy();
expect(isLikelyAscendingVector(new ArrayVector([]))).toBeTruthy();
});
it('middle values', () => {
const v = new ArrayVector([null, null, 5, 4, null]);
expect(isLikelyAscendingVector(v)).toBeFalsy();
});
it('decending', () => {
expect(isLikelyAscendingVector(new ArrayVector([7, 6, null]))).toBeFalsy();
expect(isLikelyAscendingVector(new ArrayVector([7, 8, 6]))).toBeFalsy();
});
});
});
packages/grafana-data/src/transformations/transformers/joinDataFrames.ts
+313
View File
@@ -0,0 +1,313 @@
import { DataFrame, Field, FieldMatcher, FieldType, Vector } from '../../types';
import { ArrayVector } from '../../vector';
import { fieldMatchers } from '../matchers';
import { FieldMatcherID } from '../matchers/ids';
import { getTimeField, sortDataFrame } from '../../dataframe';
import { getFieldDisplayName } from '../../field';
export function pickBestJoinField(data: DataFrame[]): FieldMatcher {
const { timeField } = getTimeField(data[0]);
if (timeField) {
return fieldMatchers.get(FieldMatcherID.firstTimeField).get({});
}
let common: string[] = [];
for (const f of data[0].fields) {
if (f.type === FieldType.number) {
common.push(f.name);
}
}
for (let i = 1; i < data.length; i++) {
const names: string[] = [];
for (const f of data[0].fields) {
if (f.type === FieldType.number) {
names.push(f.name);
}
}
common = common.filter((v) => !names.includes(v));
}
return fieldMatchers.get(FieldMatcherID.byName).get(common[0]);
}
/**
* @alpha
*/
export interface JoinOptions {
/**
* The input fields
*/
frames: DataFrame[];
/**
* The field to join -- frames that do not have this field will be droppped
*/
joinBy?: FieldMatcher;
/**
* Optionally filter the non-join fields
*/
keep?: FieldMatcher;
/**
* When the result is a single frame, this will to a quick check to see if the values are sorted,
* and sort if necessary. If the first/last values are in order the whole vector is assumed to be
* sorted
*/
enforceSort?: boolean;
/**
* @internal -- used when we need to keep a reference to the original frame/field index
*/
keepOriginIndices?: boolean;
}
function getJoinMatcher(options: JoinOptions): FieldMatcher {
return options.joinBy ?? pickBestJoinField(options.frames);
}
/**
* This will return a single frame joined by the first matching field. When a join field is not specified,
* the default will use the first time field
*/
export function outerJoinDataFrames(options: JoinOptions): DataFrame | undefined {
if (!options.frames?.length) {
return undefined;
}
if (options.frames.length === 1) {
let frame = options.frames[0];
if (options.keepOriginIndices) {
frame = {
...frame,
fields: frame.fields.map((f, fieldIndex) => {
const copy = { ...f };
const origin = {
frameIndex: 0,
fieldIndex,
};
if (copy.state) {
copy.state.origin = origin;
} else {
copy.state = { origin };
}
return copy;
}),
};
}
if (options.enforceSort) {
const joinFieldMatcher = getJoinMatcher(options);
const joinIndex = frame.fields.findIndex((f) => joinFieldMatcher(f, frame, options.frames));
if (joinIndex >= 0) {
if (!isLikelyAscendingVector(frame.fields[joinIndex].values)) {
return sortDataFrame(frame, joinIndex);
}
}
}
return frame;
}
const nullModes: JoinNullMode[][] = [];
const allData: AlignedData[] = [];
const originalFields: Field[] = [];
const joinFieldMatcher = getJoinMatcher(options);
for (let frameIndex = 0; frameIndex < options.frames.length; frameIndex++) {
const frame = options.frames[frameIndex];
if (!frame || !frame.fields?.length) {
continue; // skip the frame
}
const nullModesFrame: JoinNullMode[] = [NULL_REMOVE];
let join: Field | undefined = undefined;
let fields: Field[] = [];
for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
const field = frame.fields[fieldIndex];
getFieldDisplayName(field, frame, options.frames); // cache displayName in state
if (!join && joinFieldMatcher(field, frame, options.frames)) {
join = field;
} else {
if (options.keep && !options.keep(field, frame, options.frames)) {
continue; // skip field
}
// Support the standard graph span nulls field config
nullModesFrame.push(field.config.custom?.spanNulls ? NULL_REMOVE : NULL_EXPAND);
let labels = field.labels ?? {};
if (frame.name) {
labels = { ...labels, name: frame.name };
}
fields.push({
...field,
labels, // add the name label from frame
});
}
if (options.keepOriginIndices) {
field.state!.origin = {
frameIndex,
fieldIndex,
};
}
}
if (!join) {
continue; // skip the frame
}
if (originalFields.length === 0) {
originalFields.push(join); // first join field
}
nullModes.push(nullModesFrame);
const a: AlignedData = [join.values.toArray()]; //
for (const field of fields) {
a.push(field.values.toArray());
originalFields.push(field);
}
allData.push(a);
}
const joined = join(allData, nullModes);
return {
// ...options.data[0], // keep name, meta?
length: joined[0].length,
fields: originalFields.map((f, index) => ({
...f,
values: new ArrayVector(joined[index]),
})),
};
}
//--------------------------------------------------------------------------------
// Below here is copied from uplot (MIT License)
// https://github.com/leeoniya/uPlot/blob/master/src/utils.js#L325
// This avoids needing to import uplot into the data package
//--------------------------------------------------------------------------------
// Copied from uplot
type AlignedData = [number[], ...Array<Array<number | null>>];
// nullModes
const NULL_REMOVE = 0; // nulls are converted to undefined (e.g. for spanGaps: true)
const NULL_RETAIN = 1; // nulls are retained, with alignment artifacts set to undefined (default)
const NULL_EXPAND = 2; // nulls are expanded to include any adjacent alignment artifacts
type JoinNullMode = number; // NULL_IGNORE | NULL_RETAIN | NULL_EXPAND;
// sets undefined values to nulls when adjacent to existing nulls (minesweeper)
function nullExpand(yVals: Array<number | null>, nullIdxs: number[], alignedLen: number) {
for (let i = 0, xi, lastNullIdx = -1; i < nullIdxs.length; i++) {
let nullIdx = nullIdxs[i];
if (nullIdx > lastNullIdx) {
xi = nullIdx - 1;
while (xi >= 0 && yVals[xi] == null) {
yVals[xi--] = null;
}
xi = nullIdx + 1;
while (xi < alignedLen && yVals[xi] == null) {
yVals[(lastNullIdx = xi++)] = null;
}
}
}
}
// nullModes is a tables-matched array indicating how to treat nulls in each series
function join(tables: AlignedData[], nullModes: number[][]) {
const xVals = new Set<number>();
for (let ti = 0; ti < tables.length; ti++) {
let t = tables[ti];
let xs = t[0];
let len = xs.length;
for (let i = 0; i < len; i++) {
xVals.add(xs[i]);
}
}
let data = [Array.from(xVals).sort((a, b) => a - b)];
let alignedLen = data[0].length;
let xIdxs = new Map();
for (let i = 0; i < alignedLen; i++) {
xIdxs.set(data[0][i], i);
}
for (let ti = 0; ti < tables.length; ti++) {
let t = tables[ti];
let xs = t[0];
for (let si = 1; si < t.length; si++) {
let ys = t[si];
let yVals = Array(alignedLen).fill(undefined);
let nullMode = nullModes ? nullModes[ti][si] : NULL_RETAIN;
let nullIdxs = [];
for (let i = 0; i < ys.length; i++) {
let yVal = ys[i];
let alignedIdx = xIdxs.get(xs[i]);
if (yVal == null) {
if (nullMode !== NULL_REMOVE) {
yVals[alignedIdx] = yVal;
if (nullMode === NULL_EXPAND) {
nullIdxs.push(alignedIdx);
}
}
} else {
yVals[alignedIdx] = yVal;
}
}
nullExpand(yVals, nullIdxs, alignedLen);
data.push(yVals);
}
}
return data;
}
// Quick test if the first and last points look to be ascending
// Only exported for tests
export function isLikelyAscendingVector(data: Vector): boolean {
let first: any = undefined;
for (let idx = 0; idx < data.length; idx++) {
const v = data.get(idx);
if (v != null) {
if (first != null) {
if (first > v) {
return false; // descending
}
break;
}
first = v;
}
}
let idx = data.length - 1;
while (idx >= 0) {
const v = data.get(idx--);
if (v != null) {
if (first > v) {
return false;
}
return true;
}
}
return true; // only one non-null point
}
packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts
+320 -228
View File
@@ -2,7 +2,6 @@ import {
ArrayVector,
DataTransformerConfig,
DataTransformerID,
Field,
FieldType,
toDataFrame,
transformDataFrame,
@@ -45,58 +44,102 @@ describe('SeriesToColumns Transformer', () => {
(received) => {
const data = received[0];
const filtered = data[0];
expect(filtered.fields).toEqual([
{
name: 'time',
state: {
displayName: 'time',
expect(filtered.fields).toMatchInlineSnapshot(`
Array [
Object {
"config": Object {},
"name": "time",
"state": Object {
"displayName": "time",
},
"type": "time",
"values": Array [
1000,
3000,
4000,
5000,
6000,
7000,
],
},
type: FieldType.time,
values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]),
config: {},
labels: undefined,
},
{
name: 'temperature',
state: {
displayName: 'temperature even',
Object {
"config": Object {},
"labels": Object {
"name": "even",
},
"name": "temperature",
"state": Object {
"displayName": "even temperature",
},
"type": "number",
"values": Array [
undefined,
10.3,
10.4,
10.5,
10.6,
undefined,
],
},
type: FieldType.number,
values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'humidity',
state: {
displayName: 'humidity even',
Object {
"config": Object {},
"labels": Object {
"name": "even",
},
"name": "humidity",
"state": Object {
"displayName": "even humidity",
},
"type": "number",
"values": Array [
undefined,
10000.3,
10000.4,
10000.5,
10000.6,
undefined,
],
},
type: FieldType.number,
values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'temperature',
state: {
displayName: 'temperature odd',
Object {
"config": Object {},
"labels": Object {
"name": "odd",
},
"name": "temperature",
"state": Object {
"displayName": "odd temperature",
},
"type": "number",
"values": Array [
11.1,
11.3,
undefined,
11.5,
undefined,
11.7,
],
},
type: FieldType.number,
values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]),
config: {},
labels: { name: 'odd' },
},
{
name: 'humidity',
state: {
displayName: 'humidity odd',
Object {
"config": Object {},
"labels": Object {
"name": "odd",
},
"name": "humidity",
"state": Object {
"displayName": "odd humidity",
},
"type": "number",
"values": Array [
11000.1,
11000.3,
undefined,
11000.5,
undefined,
11000.7,
],
},
type: FieldType.number,
values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]),
config: {},
labels: { name: 'odd' },
},
]);
]
`);
}
);
});
@@ -113,58 +156,7 @@ describe('SeriesToColumns Transformer', () => {
(received) => {
const data = received[0];
const filtered = data[0];
expect(filtered.fields).toEqual([
{
name: 'temperature',
state: {
displayName: 'temperature',
},
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6, 11.1, 11.3, 11.5, 11.7]),
config: {},
labels: undefined,
},
{
name: 'time',
state: {
displayName: 'time even',
},
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000, null, null, null, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'humidity',
state: {
displayName: 'humidity even',
},
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6, null, null, null, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'time',
state: {
displayName: 'time odd',
},
type: FieldType.time,
values: new ArrayVector([null, null, null, null, 1000, 3000, 5000, 7000]),
config: {},
labels: { name: 'odd' },
},
{
name: 'humidity',
state: {
displayName: 'humidity odd',
},
type: FieldType.number,
values: new ArrayVector([null, null, null, null, 11000.1, 11000.3, 11000.5, 11000.7]),
config: {},
labels: { name: 'odd' },
},
]);
expect(filtered.fields).toMatchInlineSnapshot(`Array []`);
}
);
});
@@ -185,58 +177,102 @@ describe('SeriesToColumns Transformer', () => {
(received) => {
const data = received[0];
const filtered = data[0];
expect(filtered.fields).toEqual([
{
name: 'time',
state: {
displayName: 'time',
expect(filtered.fields).toMatchInlineSnapshot(`
Array [
Object {
"config": Object {},
"name": "time",
"state": Object {
"displayName": "time",
},
"type": "time",
"values": Array [
1000,
3000,
4000,
5000,
6000,
7000,
],
},
type: FieldType.time,
values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]),
config: {},
labels: undefined,
},
{
name: 'temperature',
state: {
displayName: 'temperature even',
Object {
"config": Object {},
"labels": Object {
"name": "even",
},
"name": "temperature",
"state": Object {
"displayName": "even temperature",
},
"type": "number",
"values": Array [
undefined,
10.3,
10.4,
10.5,
10.6,
undefined,
],
},
type: FieldType.number,
values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'humidity',
state: {
displayName: 'humidity even',
Object {
"config": Object {},
"labels": Object {
"name": "even",
},
"name": "humidity",
"state": Object {
"displayName": "even humidity",
},
"type": "number",
"values": Array [
undefined,
10000.3,
10000.4,
10000.5,
10000.6,
undefined,
],
},
type: FieldType.number,
values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'temperature',
state: {
displayName: 'temperature odd',
Object {
"config": Object {},
"labels": Object {
"name": "odd",
},
"name": "temperature",
"state": Object {
"displayName": "odd temperature",
},
"type": "number",
"values": Array [
11.1,
11.3,
undefined,
11.5,
undefined,
11.7,
],
},
type: FieldType.number,
values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]),
config: {},
labels: { name: 'odd' },
},
{
name: 'humidity',
state: {
displayName: 'humidity odd',
Object {
"config": Object {},
"labels": Object {
"name": "odd",
},
"name": "humidity",
"state": Object {
"displayName": "odd humidity",
},
"type": "number",
"values": Array [
11000.1,
11000.3,
undefined,
11000.5,
undefined,
11000.7,
],
},
type: FieldType.number,
values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]),
config: {},
labels: { name: 'odd' },
},
]);
]
`);
}
);
});
@@ -270,40 +306,58 @@ describe('SeriesToColumns Transformer', () => {
(received) => {
const data = received[0];
const filtered = data[0];
const expected: Field[] = [
{
name: 'time',
state: {
displayName: 'time',
expect(filtered.fields).toMatchInlineSnapshot(`
Array [
Object {
"config": Object {},
"name": "time",
"state": Object {
"displayName": "time",
},
"type": "time",
"values": Array [
1000,
2000,
3000,
4000,
],
},
type: FieldType.time,
values: new ArrayVector([1000, 2000, 3000, 4000]),
config: {},
labels: undefined,
},
{
name: 'temperature',
type: FieldType.number,
values: new ArrayVector([1, 3, 5, 7]),
config: {},
state: {
displayName: 'temperature temperature',
Object {
"config": Object {},
"labels": Object {
"name": "temperature",
},
"name": "temperature",
"state": Object {
"displayName": "temperature temperature",
},
"type": "number",
"values": Array [
1,
3,
5,
7,
],
},
labels: { name: 'temperature' },
},
{
name: 'temperature',
state: {
displayName: 'temperature B',
Object {
"config": Object {},
"labels": Object {
"name": "B",
},
"name": "temperature",
"state": Object {
"displayName": "B temperature",
},
"type": "number",
"values": Array [
2,
4,
6,
8,
],
},
type: FieldType.number,
values: new ArrayVector([2, 4, 6, 8]),
config: {},
labels: { name: 'B' },
},
];
expect(filtered.fields).toEqual(expected);
]
`);
}
);
});
@@ -341,31 +395,55 @@ describe('SeriesToColumns Transformer', () => {
await expect(transformDataFrame([cfg], [frame1, frame2, frame3])).toEmitValuesWith((received) => {
const data = received[0];
const filtered = data[0];
expect(filtered.fields).toEqual([
{
name: 'time',
state: { displayName: 'time' },
type: FieldType.time,
values: new ArrayVector([1, 2, 3]),
config: {},
},
{
name: 'temperature',
state: { displayName: 'temperature A' },
type: FieldType.number,
values: new ArrayVector([10, 11, 12]),
config: {},
labels: { name: 'A' },
},
{
name: 'temperature',
state: { displayName: 'temperature C' },
type: FieldType.number,
values: new ArrayVector([20, 22, 24]),
config: {},
labels: { name: 'C' },
},
]);
expect(filtered.fields).toMatchInlineSnapshot(`
Array [
Object {
"config": Object {},
"name": "time",
"state": Object {
"displayName": "time",
},
"type": "time",
"values": Array [
1,
2,
3,
],
},
Object {
"config": Object {},
"labels": Object {
"name": "A",
},
"name": "temperature",
"state": Object {
"displayName": "A temperature",
},
"type": "number",
"values": Array [
10,
11,
12,
],
},
Object {
"config": Object {},
"labels": Object {
"name": "C",
},
"name": "temperature",
"state": Object {
"displayName": "C temperature",
},
"type": "number",
"values": Array [
20,
22,
24,
],
},
]
`);
});
});
@@ -394,31 +472,45 @@ describe('SeriesToColumns Transformer', () => {
await expect(transformDataFrame([cfg], [frame1, frame2])).toEmitValuesWith((received) => {
const data = received[0];
const filtered = data[0];
expect(filtered.fields).toEqual([
{
name: 'time',
state: { displayName: 'time' },
type: FieldType.time,
values: new ArrayVector([1]),
config: {},
},
{
name: 'temperature',
state: { displayName: 'temperature 1' },
type: FieldType.number,
values: new ArrayVector([10]),
config: {},
labels: {},
},
{
name: 'temperature',
state: { displayName: 'temperature 2' },
type: FieldType.number,
values: new ArrayVector([20]),
config: {},
labels: {},
},
]);
expect(filtered.fields).toMatchInlineSnapshot(`
Array [
Object {
"config": Object {},
"name": "time",
"state": Object {
"displayName": "time",
},
"type": "time",
"values": Array [
1,
],
},
Object {
"config": Object {},
"labels": Object {},
"name": "temperature",
"state": Object {
"displayName": "temperature",
},
"type": "number",
"values": Array [
10,
],
},
Object {
"config": Object {},
"labels": Object {},
"name": "temperature",
"state": Object {
"displayName": "temperature",
},
"type": "number",
"values": Array [
20,
],
},
]
`);
});
});
});
packages/grafana-data/src/transformations/transformers/seriesToColumns.ts
+18 -135
View File
@@ -1,153 +1,36 @@
import { map } from 'rxjs/operators';
import { DataFrame, DataTransformerInfo, Field } from '../../types';
import { DataTransformerInfo, FieldMatcher } from '../../types';
import { DataTransformerID } from './ids';
import { MutableDataFrame } from '../../dataframe';
import { ArrayVector } from '../../vector';
import { getFieldDisplayName } from '../../field/fieldState';
import { outerJoinDataFrames } from './joinDataFrames';
import { fieldMatchers } from '../matchers';
import { FieldMatcherID } from '../matchers/ids';
export interface SeriesToColumnsOptions {
byField?: string;
byField?: string; // empty will pick the field automatically
}
const DEFAULT_KEY_FIELD = 'Time';
export const seriesToColumnsTransformer: DataTransformerInfo<SeriesToColumnsOptions> = {
id: DataTransformerID.seriesToColumns,
name: 'Series as columns',
name: 'Series as columns', // Called 'Outer join' in the UI!
description: 'Groups series by field and returns values as columns',
defaultOptions: {
byField: DEFAULT_KEY_FIELD,
byField: undefined, // DEFAULT_KEY_FIELD,
},
operator: (options) => (source) =>
source.pipe(
map((data) => {
return outerJoinDataFrames(data, options);
if (data.length > 1) {
let joinBy: FieldMatcher | undefined = undefined;
if (options.byField) {
joinBy = fieldMatchers.get(FieldMatcherID.byName).get(options.byField);
}
const joined = outerJoinDataFrames({ frames: data, joinBy });
if (joined) {
return [joined];
}
}
return data;
})
),
};
/**
* @internal
*/
export function outerJoinDataFrames(data: DataFrame[], options: SeriesToColumnsOptions) {
const keyFieldMatch = options.byField || DEFAULT_KEY_FIELD;
const allFields: FieldsToProcess[] = [];
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
const keyField = findKeyField(frame, keyFieldMatch);
if (!keyField) {
continue;
}
for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
const sourceField = frame.fields[fieldIndex];
if (sourceField === keyField) {
continue;
}
let labels = sourceField.labels ?? {};
if (frame.name) {
labels = { ...labels, name: frame.name };
}
allFields.push({
keyField,
sourceField,
newField: {
...sourceField,
state: null,
values: new ArrayVector([]),
labels,
},
});
}
}
// if no key fields or more than one value field
if (allFields.length <= 1) {
return data;
}
const resultFrame = new MutableDataFrame();
resultFrame.addField({
...allFields[0].keyField,
values: new ArrayVector([]),
});
for (const item of allFields) {
item.newField = resultFrame.addField(item.newField);
}
const keyFieldTitle = getFieldDisplayName(resultFrame.fields[0], resultFrame);
const byKeyField: { [key: string]: { [key: string]: any } } = {};
/*
this loop creates a dictionary object that groups the key fields values
{
"key field first value as string" : {
"key field name": key field first value,
"other series name": other series value
"other series n name": other series n value
},
"key field n value as string" : {
"key field name": key field n value,
"other series name": other series value
"other series n name": other series n value
}
}
*/
for (let fieldIndex = 0; fieldIndex < allFields.length; fieldIndex++) {
const { sourceField, keyField, newField } = allFields[fieldIndex];
const newFieldTitle = getFieldDisplayName(newField, resultFrame);
for (let valueIndex = 0; valueIndex < sourceField.values.length; valueIndex++) {
const value = sourceField.values.get(valueIndex);
const keyValue = keyField.values.get(valueIndex);
if (!byKeyField[keyValue]) {
byKeyField[keyValue] = { [newFieldTitle]: value, [keyFieldTitle]: keyValue };
} else {
byKeyField[keyValue][newFieldTitle] = value;
}
}
}
const keyValueStrings = Object.keys(byKeyField);
for (let rowIndex = 0; rowIndex < keyValueStrings.length; rowIndex++) {
const keyValueAsString = keyValueStrings[rowIndex];
for (let fieldIndex = 0; fieldIndex < resultFrame.fields.length; fieldIndex++) {
const field = resultFrame.fields[fieldIndex];
const otherColumnName = getFieldDisplayName(field, resultFrame);
const value = byKeyField[keyValueAsString][otherColumnName] ?? null;
field.values.add(value);
}
}
return [resultFrame];
}
function findKeyField(frame: DataFrame, matchTitle: string): Field | null {
for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
const field = frame.fields[fieldIndex];
if (matchTitle === getFieldDisplayName(field)) {
return field;
}
}
return null;
}
interface FieldsToProcess {
newField: Field;
sourceField: Field;
keyField: Field;
}
packages/grafana-ui/src/components/GraphNG/GraphNG.tsx
+15 -2
View File
@@ -5,14 +5,16 @@ import {
DisplayValue,
FieldConfig,
FieldMatcher,
FieldMatcherID,
fieldMatchers,
fieldReducers,
FieldType,
formattedValueToString,
getFieldDisplayName,
outerJoinDataFrames,
reduceField,
TimeRange,
} from '@grafana/data';
import { joinDataFrames } from './utils';
import { useTheme } from '../../themes';
import { UPlotChart } from '../uPlot/Plot';
import { PlotProps } from '../uPlot/types';
@@ -64,7 +66,16 @@ export const GraphNG: React.FC<GraphNGProps> = ({
const theme = useTheme();
const hasLegend = useRef(legend && legend.displayMode !== LegendDisplayMode.Hidden);
const frame = useMemo(() => joinDataFrames(data, fields), [data, fields]);
const frame = useMemo(() => {
// Default to timeseries config
if (!fields) {
fields = {
x: fieldMatchers.get(FieldMatcherID.firstTimeField).get({}),
y: fieldMatchers.get(FieldMatcherID.numeric).get({}),
};
}
return outerJoinDataFrames({ frames: data, joinBy: fields.x, keep: fields.y, keepOriginIndices: true });
}, [data, fields]);
const compareFrames = useCallback((a?: DataFrame | null, b?: DataFrame | null) => {
if (a && b) {
@@ -107,6 +118,7 @@ export const GraphNG: React.FC<GraphNGProps> = ({
// X is the first field in the aligned frame
const xField = frame.fields[0];
let seriesIndex = 0;
if (xField.type === FieldType.time) {
builder.addScale({
@@ -150,6 +162,7 @@ export const GraphNG: React.FC<GraphNGProps> = ({
if (field === xField || field.type !== FieldType.number) {
continue;
}
field.state!.seriesIndex = seriesIndex++;
const fmt = field.display ?? defaultFormatter;
const scaleKey = config.unit || FIXED_UNIT;
packages/grafana-ui/src/components/GraphNG/utils.test.ts
-334
View File
@@ -1,334 +0,0 @@
import { ArrayVector, DataFrame, FieldType, toDataFrame } from '@grafana/data';
import { joinDataFrames, isLikelyAscendingVector } from './utils';
describe('joinDataFrames', () => {
describe('joined frame', () => {
it('should align multiple data frames into one data frame', () => {
const data: DataFrame[] = [
toDataFrame({
fields: [
{ name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
{ name: 'temperature A', type: FieldType.number, values: [1, 3, 5, 7] },
],
}),
toDataFrame({
fields: [
{ name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
{ name: 'temperature B', type: FieldType.number, values: [0, 2, 6, 7] },
],
}),
];
const joined = joinDataFrames(data);
expect(joined?.fields).toMatchInlineSnapshot(`
Array [
Object {
"config": Object {},
"name": "time",
"state": Object {
"origin": undefined,
},
"type": "time",
"values": Array [
1000,
2000,
3000,
4000,
],
},
Object {
"config": Object {},
"name": "temperature A",
"state": Object {
"displayName": "temperature A",
"origin": Object {
"fieldIndex": 1,
"frameIndex": 0,
},
"seriesIndex": 0,
},
"type": "number",
"values": Array [
1,
3,
5,
7,
],
},
Object {
"config": Object {},
"name": "temperature B",
"state": Object {
"displayName": "temperature B",
"origin": Object {
"fieldIndex": 1,
"frameIndex": 1,
},
"seriesIndex": 1,
},
"type": "number",
"values": Array [
0,
2,
6,
7,
],
},
]
`);
});
it('should align multiple data frames into one data frame but only keep first time field', () => {
const data: DataFrame[] = [
toDataFrame({
fields: [
{ name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
{ name: 'temperature', type: FieldType.number, values: [1, 3, 5, 7] },
],
}),
toDataFrame({
fields: [
{ name: 'time2', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
{ name: 'temperature B', type: FieldType.number, values: [0, 2, 6, 7] },
],
}),
];
const aligned = joinDataFrames(data);
expect(aligned?.fields).toMatchInlineSnapshot(`
Array [
Object {
"config": Object {},
"name": "time",
"state": Object {
"origin": undefined,
},
"type": "time",
"values": Array [
1000,
2000,
3000,
4000,
],
},
Object {
"config": Object {},
"name": "temperature",
"state": Object {
"displayName": "temperature",
"origin": Object {
"fieldIndex": 1,
"frameIndex": 0,
},
"seriesIndex": 0,
},
"type": "number",
"values": Array [
1,
3,
5,
7,
],
},
Object {
"config": Object {},
"name": "temperature B",
"state": Object {
"displayName": "temperature B",
"origin": Object {
"fieldIndex": 1,
"frameIndex": 1,
},
"seriesIndex": 1,
},
"type": "number",
"values": Array [
0,
2,
6,
7,
],
},
]
`);
});
it('should align multiple data frames into one data frame and skip non-numeric fields', () => {
const data: DataFrame[] = [
toDataFrame({
fields: [
{ name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
{ name: 'temperature', type: FieldType.number, values: [1, 3, 5, 7] },
{ name: 'state', type: FieldType.string, values: ['on', 'off', 'off', 'on'] },
],
}),
];
const aligned = joinDataFrames(data);
expect(aligned?.fields).toMatchInlineSnapshot(`
Array [
Object {
"config": Object {},
"name": "time",
"state": Object {
"origin": undefined,
},
"type": "time",
"values": Array [
1000,
2000,
3000,
4000,
],
},
Object {
"config": Object {},
"name": "temperature",
"state": Object {
"displayName": "temperature",
"origin": Object {
"fieldIndex": 1,
"frameIndex": 0,
},
"seriesIndex": 0,
},
"type": "number",
"values": Array [
1,
3,
5,
7,
],
},
]
`);
});
it('should align multiple data frames into one data frame and skip non-numeric fields', () => {
const data: DataFrame[] = [
toDataFrame({
fields: [
{ name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
{ name: 'temperature', type: FieldType.number, values: [1, 3, 5, 7] },
{ name: 'state', type: FieldType.string, values: ['on', 'off', 'off', 'on'] },
],
}),
];
const aligned = joinDataFrames(data);
expect(aligned?.fields).toMatchInlineSnapshot(`
Array [
Object {
"config": Object {},
"name": "time",
"state": Object {
"origin": undefined,
},
"type": "time",
"values": Array [
1000,
2000,
3000,
4000,
],
},
Object {
"config": Object {},
"name": "temperature",
"state": Object {
"displayName": "temperature",
"origin": Object {
"fieldIndex": 1,
"frameIndex": 0,
},
"seriesIndex": 0,
},
"type": "number",
"values": Array [
1,
3,
5,
7,
],
},
]
`);
});
});
describe('getDataFrameFieldIndex', () => {
let aligned: DataFrame | null;
beforeAll(() => {
const data: DataFrame[] = [
toDataFrame({
fields: [
{ name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
{ name: 'temperature A', type: FieldType.number, values: [1, 3, 5, 7] },
],
}),
toDataFrame({
fields: [
{ name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
{ name: 'temperature B', type: FieldType.number, values: [0, 2, 6, 7] },
{ name: 'humidity', type: FieldType.number, values: [0, 2, 6, 7] },
],
}),
toDataFrame({
fields: [
{ name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] },
{ name: 'temperature C', type: FieldType.number, values: [0, 2, 6, 7] },
],
}),
];
aligned = joinDataFrames(data);
});
it.each`
yDim | index
${1} | ${[0, 1]}
${2} | ${[1, 1]}
${3} | ${[1, 2]}
${4} | ${[2, 1]}
`('should return correct index for yDim', ({ yDim, index }) => {
const [frameIndex, fieldIndex] = index;
expect(aligned?.fields[yDim].state?.origin).toEqual({
frameIndex,
fieldIndex,
});
});
});
describe('check ascending data', () => {
it('simple ascending', () => {
const v = new ArrayVector([1, 2, 3, 4, 5]);
expect(isLikelyAscendingVector(v)).toBeTruthy();
});
it('simple ascending with null', () => {
const v = new ArrayVector([null, 2, 3, 4, null]);
expect(isLikelyAscendingVector(v)).toBeTruthy();
});
it('single value', () => {
const v = new ArrayVector([null, null, null, 4, null]);
expect(isLikelyAscendingVector(v)).toBeTruthy();
expect(isLikelyAscendingVector(new ArrayVector([4]))).toBeTruthy();
expect(isLikelyAscendingVector(new ArrayVector([]))).toBeTruthy();
});
it('middle values', () => {
const v = new ArrayVector([null, null, 5, 4, null]);
expect(isLikelyAscendingVector(v)).toBeFalsy();
});
it('decending', () => {
expect(isLikelyAscendingVector(new ArrayVector([7, 6, null]))).toBeFalsy();
expect(isLikelyAscendingVector(new ArrayVector([7, 8, 6]))).toBeFalsy();
});
});
});
packages/grafana-ui/src/components/GraphNG/utils.ts
-180
View File
@@ -1,180 +0,0 @@
import {
DataFrame,
ArrayVector,
NullValueMode,
getFieldDisplayName,
Field,
fieldMatchers,
FieldMatcherID,
FieldType,
FieldState,
DataFrameFieldIndex,
sortDataFrame,
Vector,
} from '@grafana/data';
import uPlot, { AlignedData, JoinNullMode } from 'uplot';
import { XYFieldMatchers } from './GraphNG';
// the results ofter passing though data
export interface XYDimensionFields {
x: Field; // independent axis (cause)
y: Field[]; // dependent axis (effect)
}
export function mapDimesions(match: XYFieldMatchers, frame: DataFrame, frames?: DataFrame[]): XYDimensionFields {
let x: Field | undefined;
const y: Field[] = [];
for (const field of frame.fields) {
if (!x && match.x(field, frame, frames ?? [])) {
x = field;
}
if (match.y(field, frame, frames ?? [])) {
y.push(field);
}
}
return { x: x as Field, y };
}
/**
* Returns a single DataFrame with:
* - A shared time column
* - only numeric fields
*
* @alpha
*/
export function joinDataFrames(frames: DataFrame[], fields?: XYFieldMatchers): DataFrame | null {
const valuesFromFrames: AlignedData[] = [];
const sourceFields: Field[] = [];
const sourceFieldsRefs: Record<number, DataFrameFieldIndex> = {};
const nullModes: JoinNullMode[][] = [];
// Default to timeseries config
if (!fields) {
fields = {
x: fieldMatchers.get(FieldMatcherID.firstTimeField).get({}),
y: fieldMatchers.get(FieldMatcherID.numeric).get({}),
};
}
for (let frameIndex = 0; frameIndex < frames.length; frameIndex++) {
let frame = frames[frameIndex];
let dims = mapDimesions(fields, frame, frames);
if (!(dims.x && dims.y.length)) {
continue; // no numeric and no time fields
}
// Quick check that x is ascending order
if (!isLikelyAscendingVector(dims.x.values)) {
const xIndex = frame.fields.indexOf(dims.x);
frame = sortDataFrame(frame, xIndex);
dims = mapDimesions(fields, frame, frames);
}
let nullModesFrame: JoinNullMode[] = [0];
// Add the first X axis
if (!sourceFields.length) {
sourceFields.push(dims.x);
}
const alignedData: AlignedData = [
dims.x.values.toArray(), // The x axis (time)
];
for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
const field = frame.fields[fieldIndex];
if (!fields.y(field, frame, frames)) {
continue;
}
let values = field.values.toArray();
let joinNullMode = field.config.custom?.spanNulls ? 0 : 2;
if (field.config.nullValueMode === NullValueMode.AsZero) {
values = values.map((v) => (v === null ? 0 : v));
joinNullMode = 0;
}
sourceFieldsRefs[sourceFields.length] = { frameIndex, fieldIndex };
alignedData.push(values);
nullModesFrame.push(joinNullMode);
// This will cache an appropriate field name in the field state
getFieldDisplayName(field, frame, frames);
sourceFields.push(field);
}
valuesFromFrames.push(alignedData);
nullModes.push(nullModesFrame);
}
if (valuesFromFrames.length === 0) {
return null;
}
// do the actual alignment (outerJoin on the first arrays)
let joinedData = uPlot.join(valuesFromFrames, nullModes);
if (joinedData!.length !== sourceFields.length) {
throw new Error('outerJoinValues lost a field?');
}
let seriesIdx = 0;
// Replace the values from the outer-join field
return {
...frames[0],
length: joinedData![0].length,
fields: joinedData!.map((vals, idx) => {
let state: FieldState = {
...sourceFields[idx].state,
origin: sourceFieldsRefs[idx],
};
if (sourceFields[idx].type !== FieldType.time) {
state.seriesIndex = seriesIdx;
seriesIdx++;
}
return {
...sourceFields[idx],
state,
values: new ArrayVector(vals),
};
}),
};
}
// Quick test if the first and last points look to be ascending
export function isLikelyAscendingVector(data: Vector): boolean {
let first: any = undefined;
for (let idx = 0; idx < data.length; idx++) {
const v = data.get(idx);
if (v != null) {
if (first != null) {
if (first > v) {
return false; // descending
}
break;
}
first = v;
}
}
let idx = data.length - 1;
while (idx >= 0) {
const v = data.get(idx--);
if (v != null) {
if (first > v) {
return false;
}
return true;
}
}
return true; // only one non-null point
}