Transformers: Support inner vs outer join (#53913)

This commit is contained in:
Ryan McKinley 2022-08-23 10:14:03 -07:00 committed by GitHub
parent 1766ea9fdf
commit 1d4e01f8ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 1183 additions and 125 deletions

View File

@ -0,0 +1,638 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 1351,
"links": [],
"liveNow": false,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 9,
"panels": [],
"title": "Join by time",
"type": "row"
},
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 1
},
"id": 11,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"refId": "A",
"scenarioId": "random_walk",
"seriesCount": 4
}
],
"title": "Timeseries data",
"type": "timeseries"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 1
},
"id": 13,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 11,
"refId": "A"
}
],
"title": "Same data (as a table)",
"type": "table"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 24,
"x": 0,
"y": 9
},
"id": 16,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 11,
"refId": "A"
}
],
"title": "OUTER join on time (default)",
"transformations": [
{
"id": "joinByField",
"options": {}
}
],
"type": "table"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 5,
"panels": [],
"title": "Join by string field",
"type": "row"
},
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 15
},
"id": 2,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 0,
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"csvContent": "OrderID,CustomerID,Time\n100,A,10000\n101,B,20000\n102,C,30000",
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"refId": "Orders",
"scenarioId": "csv_content"
},
{
"csvContent": "CustomerID,Name,Country\nA,Customer A,USA\nB,Customer B,Germany\nC,Customer C,Spain\nD,Customer D,Canada",
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"hide": false,
"refId": "Customers",
"scenarioId": "csv_content"
}
],
"title": "Orders",
"transformations": [],
"type": "table"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 15
},
"id": 3,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 1,
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 2,
"refId": "A"
}
],
"title": "Customers",
"transformations": [],
"type": "table"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "CustomerID"
},
"properties": [
{
"id": "custom.width",
"value": 101
}
]
},
{
"matcher": {
"id": "byName",
"options": "OrderID"
},
"properties": [
{
"id": "custom.width",
"value": 89
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 23
},
"id": 6,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 0,
"showHeader": true,
"sortBy": []
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 2,
"refId": "A"
}
],
"title": "OUTER join on CustomerID (keeps missing values)",
"transformations": [
{
"id": "joinByField",
"options": {
"byField": "CustomerID",
"mode": "outer"
}
}
],
"type": "table"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "CustomerID"
},
"properties": [
{
"id": "custom.width",
"value": 101
}
]
},
{
"matcher": {
"id": "byName",
"options": "OrderID"
},
"properties": [
{
"id": "custom.width",
"value": 89
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 23
},
"id": 7,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 0,
"showHeader": true,
"sortBy": []
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 2,
"refId": "A"
}
],
"title": "INNER join on CustomerID ",
"transformations": [
{
"id": "joinByField",
"options": {
"byField": "CustomerID",
"mode": "inner"
}
}
],
"type": "table"
}
],
"schemaVersion": 37,
"style": "dark",
"tags": [
"gdev",
"transform"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Join by field",
"uid": "gw0K4rmVz",
"version": 6,
"weekStart": ""
}

View File

@ -0,0 +1,358 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 1342,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"maxDataPoints": 1,
"options": {
"colorMode": "none",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"labels": "site=A,measure=speed,state=CA",
"refId": "A",
"scenarioId": "random_walk"
},
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"hide": false,
"labels": "site=B,measure=speed,state=OR",
"refId": "B",
"scenarioId": "random_walk"
},
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"hide": false,
"labels": "site=B,measure=temp",
"refId": "C",
"scenarioId": "random_walk"
},
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"hide": false,
"labels": "site=A,measure=temp",
"refId": "D",
"scenarioId": "random_walk"
}
],
"title": "Labeled values",
"type": "stat"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 0
},
"id": 5,
"maxDataPoints": 1,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 2,
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 2,
"refId": "A"
}
],
"title": "Same values... in a table",
"transformations": [],
"type": "table"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 10
},
"id": 4,
"maxDataPoints": 1,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 2,
"refId": "A"
}
],
"title": "Join by site",
"transformations": [
{
"id": "joinByLabels",
"options": {
"join": [
"site"
],
"value": "measure"
}
}
],
"type": "table"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 10
},
"id": 6,
"maxDataPoints": 1,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 2,
"refId": "A"
}
],
"title": "Join on all labels",
"transformations": [
{
"id": "joinByLabels",
"options": {
"value": "measure"
}
}
],
"type": "table"
}
],
"schemaVersion": 37,
"style": "dark",
"tags": [
"gdev",
"transform"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Join by labels",
"uid": "FVl-9CR4z",
"version": 10,
"weekStart": ""
}

View File

@ -322,15 +322,77 @@ We would then get :
| server 2 | 88.6 | 90 | 2020-07-07 10:32:20 | Overload |
| server 3 | 59.6 | 62 | 2020-07-07 11:34:20 | OK |
This transformation allows you to extract some key information out of your time series and display them in a convenient way.
This transformation enables you to extract key information from your time series and display it in a convenient way.
### Join by field (outer join)
### Join by field
Use this transformation to join multiple time series from a result set by field.
Use this transformation to join multiple results into a single table. This is especially useful for converting multiple
time series results into a single wide table with a shared time field.
This transformation is especially useful if you want to combine queries so that you can calculate results from the fields.
#### Inner join
In the example below, I have a template query displaying time series data from multiple servers in a table visualization. I can only view the results of one query at a time.
An inner join merges data from multiple tables where all tables share the same value from the selected field. This type of join excludes
data where values do not match in every result.
Use this transformation to combine the results from multiple queries (combining on a passed join field or the first time column) into one result, and drop rows where a successful join cannot occur.
In the following example, two queries return table data. It is visualized as two separate tables before applying the inner join transformation.
Query A:
| Time | Job | Uptime |
| ------------------- | ------- | --------- |
| 2020-07-07 11:34:20 | node | 25260122 |
| 2020-07-07 11:24:20 | postgre | 123001233 |
| 2020-07-07 11:14:20 | postgre | 345001233 |
Query B:
| Time | Server | Errors |
| ------------------- | -------- | ------ |
| 2020-07-07 11:34:20 | server 1 | 15 |
| 2020-07-07 11:24:20 | server 2 | 5 |
| 2020-07-07 11:04:20 | server 3 | 10 |
The result after applying the inner join transformation looks like the following:
| Time | Job | Uptime | Server | Errors |
| ------------------- | ------- | --------- | -------- | ------ |
| 2020-07-07 11:34:20 | node | 25260122 | server 1 | 15 |
| 2020-07-07 11:24:20 | postgre | 123001233 | server 2 | 5 |
#### Outer join
An outer join includes all data from an inner join and rows where values do not match in every input.
Use this transformation to combine the results from multiple queries (combining on a passed join field or the first time column) into one result, and drop rows where a successful join cannot occur - performing an inner join.
In the following example, two queries return table data. It is visualized as two tables before applying the inner join transformation.
Query A:
| Time | Job | Uptime |
| ------------------- | ------- | --------- |
| 2020-07-07 11:34:20 | node | 25260122 |
| 2020-07-07 11:24:20 | postgre | 123001233 |
| 2020-07-07 11:14:20 | postgre | 345001233 |
Query B:
| Time | Server | Errors |
| ------------------- | -------- | ------ |
| 2020-07-07 11:34:20 | server 1 | 15 |
| 2020-07-07 11:24:20 | server 2 | 5 |
| 2020-07-07 11:04:20 | server 3 | 10 |
The result after applying the inner join transformation looks like the following:
| Time | Job | Uptime | Server | Errors |
| ------------------- | ------- | --------- | -------- | ------ |
| 2020-07-07 11:34:20 | node | 25260122 | server 1 | 15 |
| 2020-07-07 11:24:20 | postgre | 123001233 | server 2 | 5 |
In the following example, a template query displays time series data from multiple servers in a table visualization. The results of only one query can be viewed at a time.
{{< figure src="/static/img/docs/transformations/join-fields-before-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}}
@ -643,32 +705,3 @@ Here is the result after adding a Limit transformation with a value of '3':
| 2020-07-07 11:34:20 | Temperature | 25 |
| 2020-07-07 11:34:20 | Humidity | 22 |
| 2020-07-07 10:32:20 | Humidity | 29 |
### Join by field (Inner join)
Use this transformation to combine the results from multiple queries (combining on a passed join field or the first time column) into one single result and drop rows where a successful join isn't able to occur - performing an inner join.
In the example below, we have two queries returning table data. It is visualized as two separate tables before applying the inner join transformation.
Query A:
| Time | Job | Uptime |
| ------------------- | ------- | --------- |
| 2020-07-07 11:34:20 | node | 25260122 |
| 2020-07-07 11:24:20 | postgre | 123001233 |
| 2020-07-07 11:14:20 | postgre | 345001233 |
Query B:
| Time | Server | Errors |
| ------------------- | -------- | ------ |
| 2020-07-07 11:34:20 | server 1 | 15 |
| 2020-07-07 11:24:20 | server 2 | 5 |
| 2020-07-07 11:04:20 | server 3 | 10 |
Result after applying the inner join transformation:
| Time | Job | Uptime | Server | Errors |
| ------------------- | ------- | --------- | -------- | ------ |
| 2020-07-07 11:34:20 | node | 25260122 | server 1 | 15 |
| 2020-07-07 11:24:20 | postgre | 123001233 | server 2 | 5 |

View File

@ -9,6 +9,7 @@ import { filterByValueTransformer } from './transformers/filterByValue';
import { groupByTransformer } from './transformers/groupBy';
import { groupingToMatrixTransformer } from './transformers/groupingToMatrix';
import { histogramTransformer } from './transformers/histogram';
import { joinByFieldTransformer } from './transformers/joinByField';
import { labelsToFieldsTransformer } from './transformers/labelsToFields';
import { limitTransformer } from './transformers/limit';
import { mergeTransformer } from './transformers/merge';
@ -18,7 +19,6 @@ import { organizeFieldsTransformer } from './transformers/organize';
import { reduceTransformer } from './transformers/reduce';
import { renameFieldsTransformer } from './transformers/rename';
import { renameByRegexTransformer } from './transformers/renameByRegex';
import { seriesToColumnsTransformer } from './transformers/seriesToColumns';
import { seriesToRowsTransformer } from './transformers/seriesToRows';
import { sortByTransformer } from './transformers/sortBy';
@ -34,7 +34,9 @@ export const standardTransformers = {
reduceTransformer,
concatenateTransformer,
calculateFieldTransformer,
seriesToColumnsTransformer,
joinByFieldTransformer,
/** @deprecated */
seriesToColumnsTransformer: joinByFieldTransformer,
seriesToRowsTransformer,
renameFieldsTransformer,
labelsToFieldsTransformer,

View File

@ -5,7 +5,7 @@ import { transformDataFrame } from '../transformDataFrame';
import { ensureColumnsTransformer } from './ensureColumns';
import { DataTransformerID } from './ids';
import { seriesToColumnsTransformer } from './seriesToColumns';
import { joinByFieldTransformer } from './joinByField';
const seriesA = toDataFrame({
fields: [
@ -33,7 +33,7 @@ const seriesNoTime = toDataFrame({
describe('ensureColumns transformer', () => {
beforeAll(() => {
mockTransformationsRegistry([ensureColumnsTransformer, seriesToColumnsTransformer]);
mockTransformationsRegistry([ensureColumnsTransformer, joinByFieldTransformer]);
});
it('will transform to columns if time field exists and multiple frames', async () => {

View File

@ -5,7 +5,7 @@ import { DataFrame } from '../../types/dataFrame';
import { SynchronousDataTransformerInfo } from '../../types/transformations';
import { DataTransformerID } from './ids';
import { seriesToColumnsTransformer } from './seriesToColumns';
import { joinByFieldTransformer } from './joinByField';
export const ensureColumnsTransformer: SynchronousDataTransformerInfo = {
id: DataTransformerID.ensureColumns,
@ -19,7 +19,7 @@ export const ensureColumnsTransformer: SynchronousDataTransformerInfo = {
const timeFieldName = findConsistentTimeFieldName(frames);
if (frames.length > 1 && timeFieldName) {
return seriesToColumnsTransformer.transformer({
return joinByFieldTransformer.transformer({
byField: timeFieldName,
})(frames);
}

View File

@ -1,5 +1,4 @@
export enum DataTransformerID {
// join = 'join', // Pick a field and merge all series based on that field
append = 'append',
// rotate = 'rotate', // Columns to rows
reduce = 'reduce',
@ -7,6 +6,7 @@ export enum DataTransformerID {
organize = 'organize',
rename = 'rename',
calculateField = 'calculateField',
/** @deprecated use joinByField */
seriesToColumns = 'seriesToColumns',
seriesToRows = 'seriesToRows',
merge = 'merge',
@ -30,6 +30,7 @@ export enum DataTransformerID {
fieldLookup = 'fieldLookup',
heatmap = 'heatmap',
spatial = 'spatial',
joinByField = 'joinByField',
joinByLabels = 'joinByLabels',
extractFields = 'extractFields',
groupingToMatrix = 'groupingToMatrix',

View File

@ -5,11 +5,11 @@ import { ArrayVector } from '../../vector';
import { transformDataFrame } from '../transformDataFrame';
import { DataTransformerID } from './ids';
import { JoinMode, SeriesToColumnsOptions, seriesToColumnsTransformer } from './seriesToColumns';
import { JoinMode, JoinByFieldOptions, joinByFieldTransformer } from './joinByField';
describe('SeriesToColumns Transformer', () => {
describe('JOIN Transformer', () => {
beforeAll(() => {
mockTransformationsRegistry([seriesToColumnsTransformer]);
mockTransformationsRegistry([joinByFieldTransformer]);
});
describe('outer join', () => {
@ -32,7 +32,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('joins by time field', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -134,7 +134,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('joins by temperature field', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'temperature',
@ -250,7 +250,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('joins by time field in reverse order', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -375,7 +375,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('when dataframe and field share the same name then use the field name', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -438,7 +438,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('joins if fields are missing', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -516,7 +516,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('handles duplicate field name', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -597,7 +597,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('inner joins by time field', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -678,7 +678,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('inner joins by temperature field', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'temperature',
@ -763,7 +763,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('inner joins by time field in reverse order', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -867,7 +867,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('when dataframe and field share the same name then use the field name', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -931,7 +931,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('joins if fields are missing', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -1010,7 +1010,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('handles duplicate field name', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',

View File

@ -12,23 +12,25 @@ export enum JoinMode {
inner = 'inner',
}
export interface SeriesToColumnsOptions {
export interface JoinByFieldOptions {
byField?: string; // empty will pick the field automatically
mode?: JoinMode;
}
export const seriesToColumnsTransformer: SynchronousDataTransformerInfo<SeriesToColumnsOptions> = {
id: DataTransformerID.seriesToColumns,
name: 'Series as columns', // Called 'Outer join' in the UI!
description: 'Groups series by field and returns values as columns',
export const joinByFieldTransformer: SynchronousDataTransformerInfo<JoinByFieldOptions> = {
id: DataTransformerID.joinByField,
aliasIds: [DataTransformerID.seriesToColumns],
name: 'Join by field',
description:
'Combine rows from two or more tables, based on a related field between them. This can be used to outer join multiple time series on the _time_ field to show many time series in one table.',
defaultOptions: {
byField: undefined, // DEFAULT_KEY_FIELD,
mode: JoinMode.outer,
},
operator: (options) => (source) => source.pipe(map((data) => seriesToColumnsTransformer.transformer(options)(data))),
operator: (options) => (source) => source.pipe(map((data) => joinByFieldTransformer.transformer(options)(data))),
transformer: (options: SeriesToColumnsOptions) => {
transformer: (options: JoinByFieldOptions) => {
let joinBy: FieldMatcher | undefined = undefined;
return (data: DataFrame[]) => {
if (data.length > 1) {

View File

@ -4,8 +4,8 @@ import { mockTransformationsRegistry } from '../../utils/tests/mockTransformatio
import { ArrayVector } from '../../vector';
import { calculateFieldTransformer } from './calculateField';
import { JoinMode } from './joinByField';
import { isLikelyAscendingVector, joinDataFrames } from './joinDataFrames';
import { JoinMode } from './seriesToColumns';
describe('align frames', () => {
beforeAll(() => {

View File

@ -6,7 +6,7 @@ import { ArrayVector } from '../../vector';
import { fieldMatchers } from '../matchers';
import { FieldMatcherID } from '../matchers/ids';
import { JoinMode } from './seriesToColumns';
import { JoinMode } from './joinByField';
export function pickBestJoinField(data: DataFrame[]): FieldMatcher {
const { timeField } = getTimeField(data[0]);
@ -34,7 +34,7 @@ export function pickBestJoinField(data: DataFrame[]): FieldMatcher {
}
/**
* @alpha
* @internal
*/
export interface JoinOptions {
/**

View File

@ -6,6 +6,7 @@ export const mockTransformationsRegistry = (transformers: Array<DataTransformerI
return transformers.map((t) => {
return {
id: t.id,
aliasIds: t.aliasIds,
name: t.name,
transformation: t,
description: t.description,

View File

@ -109,7 +109,7 @@ export const decorateWithTableResult = (data: ExplorePanelData): Observable<Expl
// non timeseries or some mix of data we are not trying to join on anything and just try to merge them in
// single table, which may not make sense in most cases, but it's up to the user to query something sensible.
const transformer = hasOnlyTimeseries
? of(data.tableFrames).pipe(standardTransformers.seriesToColumnsTransformer.operator({}))
? of(data.tableFrames).pipe(standardTransformers.joinByFieldTransformer.operator({}))
: of(data.tableFrames).pipe(standardTransformers.mergeTransformer.operator({}));
return transformer.pipe(

View File

@ -45,7 +45,7 @@ export const InspectDataOptions: FC<Props> = ({
const showFieldConfigsOption = panel && !panel.plugin?.fieldConfigRegistry.isEmpty();
let dataSelect = dataFrames;
if (selectedDataFrame === DataTransformerID.seriesToColumns) {
if (selectedDataFrame === DataTransformerID.joinByField) {
dataSelect = data!;
}
@ -67,7 +67,7 @@ export const InspectDataOptions: FC<Props> = ({
const parts: string[] = [];
if (selectedDataFrame === DataTransformerID.seriesToColumns) {
if (selectedDataFrame === DataTransformerID.joinByField) {
parts.push(t({ id: 'dashboard.inspect-data.series-to-columns', message: 'Series joined by time' }));
} else if (data.length > 1) {
parts.push(getFrameDisplayName(data[selectedDataFrame as number]));

View File

@ -44,7 +44,7 @@ interface Props {
}
interface State {
/** The string is seriesToColumns transformation. Otherwise it is a dataframe index */
/** The string is joinByField transformation. Otherwise it is a dataframe index */
selectedDataFrame: number | DataTransformerID;
transformId: DataTransformerID;
dataFrameIndex: number;
@ -197,7 +197,7 @@ export class InspectDataTab extends PureComponent<Props, State> {
onDataFrameChange = (item: SelectableValue<DataTransformerID | number>) => {
this.setState({
transformId:
item.value === DataTransformerID.seriesToColumns ? DataTransformerID.seriesToColumns : DataTransformerID.noop,
item.value === DataTransformerID.joinByField ? DataTransformerID.joinByField : DataTransformerID.noop,
dataFrameIndex: typeof item.value === 'number' ? item.value : 0,
selectedDataFrame: item.value!,
});
@ -349,14 +349,14 @@ export class InspectDataTab extends PureComponent<Props, State> {
function buildTransformationOptions() {
const transformations: Array<SelectableValue<DataTransformerID>> = [
{
value: DataTransformerID.seriesToColumns,
value: DataTransformerID.joinByField,
label: t({
id: 'dashboard.inspect-data.transformation',
message: 'Series joined by time',
}),
transformer: {
id: DataTransformerID.seriesToColumns,
options: { byField: 'Time' },
id: DataTransformerID.joinByField,
options: { byField: undefined }, // defaults to time field
},
},
];

View File

@ -0,0 +1,72 @@
import React, { useCallback } from 'react';
import {
DataTransformerID,
SelectableValue,
standardTransformers,
TransformerRegistryItem,
TransformerUIProps,
} from '@grafana/data';
import { JoinByFieldOptions, JoinMode } from '@grafana/data/src/transformations/transformers/joinByField';
import { Select, InlineFieldRow, InlineField } from '@grafana/ui';
import { useAllFieldNamesFromDataFrames } from '../utils';
const modes = [
{ value: JoinMode.outer, label: 'OUTER', description: 'Keep all rows from any table with a value' },
{ value: JoinMode.inner, label: 'INNER', description: 'Drop rows that do not match a value in all tables' },
];
export function SeriesToFieldsTransformerEditor({ input, options, onChange }: TransformerUIProps<JoinByFieldOptions>) {
const fieldNames = useAllFieldNamesFromDataFrames(input).map((item: string) => ({ label: item, value: item }));
const onSelectField = useCallback(
(value: SelectableValue<string>) => {
onChange({
...options,
byField: value?.value,
});
},
[onChange, options]
);
const onSetMode = useCallback(
(value: SelectableValue<JoinMode>) => {
onChange({
...options,
mode: value?.value,
});
},
[onChange, options]
);
return (
<>
<InlineFieldRow>
<InlineField label="Mode" labelWidth={8} grow>
<Select options={modes} value={options.mode ?? JoinMode.outer} onChange={onSetMode} />
</InlineField>
</InlineFieldRow>
<InlineFieldRow>
<InlineField label="Field" labelWidth={8} grow>
<Select
options={fieldNames}
value={options.byField}
onChange={onSelectField}
placeholder="time"
isClearable
/>
</InlineField>
</InlineFieldRow>
</>
);
}
export const joinByFieldTransformerRegistryItem: TransformerRegistryItem<JoinByFieldOptions> = {
id: DataTransformerID.joinByField,
aliasIds: [DataTransformerID.seriesToColumns],
editor: SeriesToFieldsTransformerEditor,
transformation: standardTransformers.joinByFieldTransformer,
name: standardTransformers.joinByFieldTransformer.name,
description: standardTransformers.joinByFieldTransformer.description,
};

View File

@ -1,49 +0,0 @@
import React, { useCallback } from 'react';
import {
DataTransformerID,
SelectableValue,
standardTransformers,
TransformerRegistryItem,
TransformerUIProps,
} from '@grafana/data';
import { SeriesToColumnsOptions } from '@grafana/data/src/transformations/transformers/seriesToColumns';
import { Select } from '@grafana/ui';
import { useAllFieldNamesFromDataFrames } from '../utils';
export const SeriesToFieldsTransformerEditor: React.FC<TransformerUIProps<SeriesToColumnsOptions>> = ({
input,
options,
onChange,
}) => {
const fieldNames = useAllFieldNamesFromDataFrames(input).map((item: string) => ({ label: item, value: item }));
const onSelectField = useCallback(
(value: SelectableValue<string>) => {
onChange({
...options,
byField: value?.value,
});
},
[onChange, options]
);
return (
<div className="gf-form-inline">
<div className="gf-form gf-form--grow">
<div className="gf-form-label width-8">Field name</div>
<Select options={fieldNames} value={options.byField} onChange={onSelectField} isClearable />
</div>
</div>
);
};
export const seriesToFieldsTransformerRegistryItem: TransformerRegistryItem<SeriesToColumnsOptions> = {
id: DataTransformerID.seriesToColumns,
editor: SeriesToFieldsTransformerEditor,
transformation: standardTransformers.seriesToColumnsTransformer,
name: 'Outer join',
description:
'Joins many time series/tables by a field. This can be used to outer join multiple time series on the _time_ field to show many time series in one table.',
};

View File

@ -11,13 +11,13 @@ import { filterFramesByRefIdTransformRegistryItem } from './editors/FilterByRefI
import { groupByTransformRegistryItem } from './editors/GroupByTransformerEditor';
import { groupingToMatrixTransformRegistryItem } from './editors/GroupingToMatrixTransformerEditor';
import { histogramTransformRegistryItem } from './editors/HistogramTransformerEditor';
import { joinByFieldTransformerRegistryItem } from './editors/JoinByFieldTransformerEditor';
import { labelsToFieldsTransformerRegistryItem } from './editors/LabelsToFieldsTransformerEditor';
import { limitTransformRegistryItem } from './editors/LimitTransformerEditor';
import { mergeTransformerRegistryItem } from './editors/MergeTransformerEditor';
import { organizeFieldsTransformRegistryItem } from './editors/OrganizeFieldsTransformerEditor';
import { reduceTransformRegistryItem } from './editors/ReduceTransformerEditor';
import { renameByRegexTransformRegistryItem } from './editors/RenameByRegexTransformer';
import { seriesToFieldsTransformerRegistryItem } from './editors/SeriesToFieldsTransformerEditor';
import { seriesToRowsTransformerRegistryItem } from './editors/SeriesToRowsTransformerEditor';
import { sortByTransformRegistryItem } from './editors/SortByTransformerEditor';
import { extractFieldsTransformRegistryItem } from './extractFields/ExtractFieldsTransformerEditor';
@ -35,7 +35,7 @@ export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> =
filterFramesByRefIdTransformRegistryItem,
filterByValueTransformRegistryItem,
organizeFieldsTransformRegistryItem,
seriesToFieldsTransformerRegistryItem,
joinByFieldTransformerRegistryItem,
seriesToRowsTransformerRegistryItem,
concatenateTransformRegistryItem,
calculateFieldTransformRegistryItem,