From 1d4e01f8ba7ce7a008f29aebfac826ae166d13f7 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 23 Aug 2022 10:14:03 -0700 Subject: [PATCH] Transformers: Support inner vs outer join (#53913) --- .../transforms/join-by-field.json | 638 ++++++++++++++++++ .../transforms/join-by-labels.json | 358 ++++++++++ docs/sources/panels/transform-data/index.md | 101 ++- .../src/transformations/transformers.ts | 6 +- .../transformers/ensureColumns.test.ts | 4 +- .../transformers/ensureColumns.ts | 4 +- .../src/transformations/transformers/ids.ts | 3 +- ...sToColumns.test.ts => joinByField.test.ts} | 30 +- .../{seriesToColumns.ts => joinByField.ts} | 16 +- .../transformers/joinDataFrames.test.ts | 2 +- .../transformers/joinDataFrames.ts | 4 +- .../tests/mockTransformationsRegistry.ts | 1 + .../app/features/explore/utils/decorators.ts | 2 +- .../features/inspector/InspectDataOptions.tsx | 4 +- .../app/features/inspector/InspectDataTab.tsx | 10 +- .../editors/JoinByFieldTransformerEditor.tsx | 72 ++ .../SeriesToFieldsTransformerEditor.tsx | 49 -- .../transformers/standardTransformers.ts | 4 +- 18 files changed, 1183 insertions(+), 125 deletions(-) create mode 100644 devenv/dev-dashboards/transforms/join-by-field.json create mode 100644 devenv/dev-dashboards/transforms/join-by-labels.json rename packages/grafana-data/src/transformations/transformers/{seriesToColumns.test.ts => joinByField.test.ts} (96%) rename packages/grafana-data/src/transformations/transformers/{seriesToColumns.ts => joinByField.ts} (61%) create mode 100644 public/app/features/transformers/editors/JoinByFieldTransformerEditor.tsx delete mode 100644 public/app/features/transformers/editors/SeriesToFieldsTransformerEditor.tsx diff --git a/devenv/dev-dashboards/transforms/join-by-field.json b/devenv/dev-dashboards/transforms/join-by-field.json new file mode 100644 index 00000000000..d4c8e018042 --- /dev/null +++ b/devenv/dev-dashboards/transforms/join-by-field.json @@ -0,0 +1,638 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1351, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "Join by time", + "type": "row" + }, + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "refId": "A", + "scenarioId": "random_walk", + "seriesCount": 4 + } + ], + "title": "Timeseries data", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 13, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 11, + "refId": "A" + } + ], + "title": "Same data (as a table)", + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 16, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 11, + "refId": "A" + } + ], + "title": "OUTER join on time (default)", + "transformations": [ + { + "id": "joinByField", + "options": {} + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 5, + "panels": [], + "title": "Join by string field", + "type": "row" + }, + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 2, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "csvContent": "OrderID,CustomerID,Time\n100,A,10000\n101,B,20000\n102,C,30000", + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "refId": "Orders", + "scenarioId": "csv_content" + }, + { + "csvContent": "CustomerID,Name,Country\nA,Customer A,USA\nB,Customer B,Germany\nC,Customer C,Spain\nD,Customer D,Canada", + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "hide": false, + "refId": "Customers", + "scenarioId": "csv_content" + } + ], + "title": "Orders", + "transformations": [], + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 3, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 2, + "refId": "A" + } + ], + "title": "Customers", + "transformations": [], + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "CustomerID" + }, + "properties": [ + { + "id": "custom.width", + "value": 101 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OrderID" + }, + "properties": [ + { + "id": "custom.width", + "value": 89 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 6, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 2, + "refId": "A" + } + ], + "title": "OUTER join on CustomerID (keeps missing values)", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "CustomerID", + "mode": "outer" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "CustomerID" + }, + "properties": [ + { + "id": "custom.width", + "value": 101 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OrderID" + }, + "properties": [ + { + "id": "custom.width", + "value": 89 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 7, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 2, + "refId": "A" + } + ], + "title": "INNER join on CustomerID ", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "CustomerID", + "mode": "inner" + } + } + ], + "type": "table" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [ + "gdev", + "transform" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Join by field", + "uid": "gw0K4rmVz", + "version": 6, + "weekStart": "" +} \ No newline at end of file diff --git a/devenv/dev-dashboards/transforms/join-by-labels.json b/devenv/dev-dashboards/transforms/join-by-labels.json new file mode 100644 index 00000000000..bf2e8fa8259 --- /dev/null +++ b/devenv/dev-dashboards/transforms/join-by-labels.json @@ -0,0 +1,358 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1342, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "maxDataPoints": 1, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "labels": "site=A,measure=speed,state=CA", + "refId": "A", + "scenarioId": "random_walk" + }, + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "hide": false, + "labels": "site=B,measure=speed,state=OR", + "refId": "B", + "scenarioId": "random_walk" + }, + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "hide": false, + "labels": "site=B,measure=temp", + "refId": "C", + "scenarioId": "random_walk" + }, + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "hide": false, + "labels": "site=A,measure=temp", + "refId": "D", + "scenarioId": "random_walk" + } + ], + "title": "Labeled values", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 5, + "maxDataPoints": 1, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 2, + "showHeader": true + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 2, + "refId": "A" + } + ], + "title": "Same values... in a table", + "transformations": [], + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 4, + "maxDataPoints": 1, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 2, + "refId": "A" + } + ], + "title": "Join by site", + "transformations": [ + { + "id": "joinByLabels", + "options": { + "join": [ + "site" + ], + "value": "measure" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 6, + "maxDataPoints": 1, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.2.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 2, + "refId": "A" + } + ], + "title": "Join on all labels", + "transformations": [ + { + "id": "joinByLabels", + "options": { + "value": "measure" + } + } + ], + "type": "table" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [ + "gdev", + "transform" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Join by labels", + "uid": "FVl-9CR4z", + "version": 10, + "weekStart": "" +} \ No newline at end of file diff --git a/docs/sources/panels/transform-data/index.md b/docs/sources/panels/transform-data/index.md index 8f7ea477508..72e6a9ae261 100644 --- a/docs/sources/panels/transform-data/index.md +++ b/docs/sources/panels/transform-data/index.md @@ -322,15 +322,77 @@ We would then get : | server 2 | 88.6 | 90 | 2020-07-07 10:32:20 | Overload | | server 3 | 59.6 | 62 | 2020-07-07 11:34:20 | OK | -This transformation allows you to extract some key information out of your time series and display them in a convenient way. +This transformation enables you to extract key information from your time series and display it in a convenient way. -### Join by field (outer join) +### Join by field -Use this transformation to join multiple time series from a result set by field. +Use this transformation to join multiple results into a single table. This is especially useful for converting multiple +time series results into a single wide table with a shared time field. -This transformation is especially useful if you want to combine queries so that you can calculate results from the fields. +#### Inner join -In the example below, I have a template query displaying time series data from multiple servers in a table visualization. I can only view the results of one query at a time. +An inner join merges data from multiple tables where all tables share the same value from the selected field. This type of join excludes +data where values do not match in every result. + +Use this transformation to combine the results from multiple queries (combining on a passed join field or the first time column) into one result, and drop rows where a successful join cannot occur. + +In the following example, two queries return table data. It is visualized as two separate tables before applying the inner join transformation. + +Query A: + +| Time | Job | Uptime | +| ------------------- | ------- | --------- | +| 2020-07-07 11:34:20 | node | 25260122 | +| 2020-07-07 11:24:20 | postgre | 123001233 | +| 2020-07-07 11:14:20 | postgre | 345001233 | + +Query B: + +| Time | Server | Errors | +| ------------------- | -------- | ------ | +| 2020-07-07 11:34:20 | server 1 | 15 | +| 2020-07-07 11:24:20 | server 2 | 5 | +| 2020-07-07 11:04:20 | server 3 | 10 | + +The result after applying the inner join transformation looks like the following: + +| Time | Job | Uptime | Server | Errors | +| ------------------- | ------- | --------- | -------- | ------ | +| 2020-07-07 11:34:20 | node | 25260122 | server 1 | 15 | +| 2020-07-07 11:24:20 | postgre | 123001233 | server 2 | 5 | + +#### Outer join + +An outer join includes all data from an inner join and rows where values do not match in every input. + +Use this transformation to combine the results from multiple queries (combining on a passed join field or the first time column) into one result, and drop rows where a successful join cannot occur - performing an inner join. + +In the following example, two queries return table data. It is visualized as two tables before applying the inner join transformation. + +Query A: + +| Time | Job | Uptime | +| ------------------- | ------- | --------- | +| 2020-07-07 11:34:20 | node | 25260122 | +| 2020-07-07 11:24:20 | postgre | 123001233 | +| 2020-07-07 11:14:20 | postgre | 345001233 | + +Query B: + +| Time | Server | Errors | +| ------------------- | -------- | ------ | +| 2020-07-07 11:34:20 | server 1 | 15 | +| 2020-07-07 11:24:20 | server 2 | 5 | +| 2020-07-07 11:04:20 | server 3 | 10 | + +The result after applying the inner join transformation looks like the following: + +| Time | Job | Uptime | Server | Errors | +| ------------------- | ------- | --------- | -------- | ------ | +| 2020-07-07 11:34:20 | node | 25260122 | server 1 | 15 | +| 2020-07-07 11:24:20 | postgre | 123001233 | server 2 | 5 | + +In the following example, a template query displays time series data from multiple servers in a table visualization. The results of only one query can be viewed at a time. {{< figure src="/static/img/docs/transformations/join-fields-before-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} @@ -643,32 +705,3 @@ Here is the result after adding a Limit transformation with a value of '3': | 2020-07-07 11:34:20 | Temperature | 25 | | 2020-07-07 11:34:20 | Humidity | 22 | | 2020-07-07 10:32:20 | Humidity | 29 | - -### Join by field (Inner join) - -Use this transformation to combine the results from multiple queries (combining on a passed join field or the first time column) into one single result and drop rows where a successful join isn't able to occur - performing an inner join. - -In the example below, we have two queries returning table data. It is visualized as two separate tables before applying the inner join transformation. - -Query A: - -| Time | Job | Uptime | -| ------------------- | ------- | --------- | -| 2020-07-07 11:34:20 | node | 25260122 | -| 2020-07-07 11:24:20 | postgre | 123001233 | -| 2020-07-07 11:14:20 | postgre | 345001233 | - -Query B: - -| Time | Server | Errors | -| ------------------- | -------- | ------ | -| 2020-07-07 11:34:20 | server 1 | 15 | -| 2020-07-07 11:24:20 | server 2 | 5 | -| 2020-07-07 11:04:20 | server 3 | 10 | - -Result after applying the inner join transformation: - -| Time | Job | Uptime | Server | Errors | -| ------------------- | ------- | --------- | -------- | ------ | -| 2020-07-07 11:34:20 | node | 25260122 | server 1 | 15 | -| 2020-07-07 11:24:20 | postgre | 123001233 | server 2 | 5 | diff --git a/packages/grafana-data/src/transformations/transformers.ts b/packages/grafana-data/src/transformations/transformers.ts index 49f13475292..218e5f8e9c0 100644 --- a/packages/grafana-data/src/transformations/transformers.ts +++ b/packages/grafana-data/src/transformations/transformers.ts @@ -9,6 +9,7 @@ import { filterByValueTransformer } from './transformers/filterByValue'; import { groupByTransformer } from './transformers/groupBy'; import { groupingToMatrixTransformer } from './transformers/groupingToMatrix'; import { histogramTransformer } from './transformers/histogram'; +import { joinByFieldTransformer } from './transformers/joinByField'; import { labelsToFieldsTransformer } from './transformers/labelsToFields'; import { limitTransformer } from './transformers/limit'; import { mergeTransformer } from './transformers/merge'; @@ -18,7 +19,6 @@ import { organizeFieldsTransformer } from './transformers/organize'; import { reduceTransformer } from './transformers/reduce'; import { renameFieldsTransformer } from './transformers/rename'; import { renameByRegexTransformer } from './transformers/renameByRegex'; -import { seriesToColumnsTransformer } from './transformers/seriesToColumns'; import { seriesToRowsTransformer } from './transformers/seriesToRows'; import { sortByTransformer } from './transformers/sortBy'; @@ -34,7 +34,9 @@ export const standardTransformers = { reduceTransformer, concatenateTransformer, calculateFieldTransformer, - seriesToColumnsTransformer, + joinByFieldTransformer, + /** @deprecated */ + seriesToColumnsTransformer: joinByFieldTransformer, seriesToRowsTransformer, renameFieldsTransformer, labelsToFieldsTransformer, diff --git a/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts b/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts index 75290127225..b67bb998870 100644 --- a/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts +++ b/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts @@ -5,7 +5,7 @@ import { transformDataFrame } from '../transformDataFrame'; import { ensureColumnsTransformer } from './ensureColumns'; import { DataTransformerID } from './ids'; -import { seriesToColumnsTransformer } from './seriesToColumns'; +import { joinByFieldTransformer } from './joinByField'; const seriesA = toDataFrame({ fields: [ @@ -33,7 +33,7 @@ const seriesNoTime = toDataFrame({ describe('ensureColumns transformer', () => { beforeAll(() => { - mockTransformationsRegistry([ensureColumnsTransformer, seriesToColumnsTransformer]); + mockTransformationsRegistry([ensureColumnsTransformer, joinByFieldTransformer]); }); it('will transform to columns if time field exists and multiple frames', async () => { diff --git a/packages/grafana-data/src/transformations/transformers/ensureColumns.ts b/packages/grafana-data/src/transformations/transformers/ensureColumns.ts index 6f331e15f5f..6ec89a218f1 100644 --- a/packages/grafana-data/src/transformations/transformers/ensureColumns.ts +++ b/packages/grafana-data/src/transformations/transformers/ensureColumns.ts @@ -5,7 +5,7 @@ import { DataFrame } from '../../types/dataFrame'; import { SynchronousDataTransformerInfo } from '../../types/transformations'; import { DataTransformerID } from './ids'; -import { seriesToColumnsTransformer } from './seriesToColumns'; +import { joinByFieldTransformer } from './joinByField'; export const ensureColumnsTransformer: SynchronousDataTransformerInfo = { id: DataTransformerID.ensureColumns, @@ -19,7 +19,7 @@ export const ensureColumnsTransformer: SynchronousDataTransformerInfo = { const timeFieldName = findConsistentTimeFieldName(frames); if (frames.length > 1 && timeFieldName) { - return seriesToColumnsTransformer.transformer({ + return joinByFieldTransformer.transformer({ byField: timeFieldName, })(frames); } diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts index a4805824abe..881add081a8 100644 --- a/packages/grafana-data/src/transformations/transformers/ids.ts +++ b/packages/grafana-data/src/transformations/transformers/ids.ts @@ -1,5 +1,4 @@ export enum DataTransformerID { - // join = 'join', // Pick a field and merge all series based on that field append = 'append', // rotate = 'rotate', // Columns to rows reduce = 'reduce', @@ -7,6 +6,7 @@ export enum DataTransformerID { organize = 'organize', rename = 'rename', calculateField = 'calculateField', + /** @deprecated use joinByField */ seriesToColumns = 'seriesToColumns', seriesToRows = 'seriesToRows', merge = 'merge', @@ -30,6 +30,7 @@ export enum DataTransformerID { fieldLookup = 'fieldLookup', heatmap = 'heatmap', spatial = 'spatial', + joinByField = 'joinByField', joinByLabels = 'joinByLabels', extractFields = 'extractFields', groupingToMatrix = 'groupingToMatrix', diff --git a/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts b/packages/grafana-data/src/transformations/transformers/joinByField.test.ts similarity index 96% rename from packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts rename to packages/grafana-data/src/transformations/transformers/joinByField.test.ts index 9f04b9daef4..95f330b4331 100644 --- a/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts +++ b/packages/grafana-data/src/transformations/transformers/joinByField.test.ts @@ -5,11 +5,11 @@ import { ArrayVector } from '../../vector'; import { transformDataFrame } from '../transformDataFrame'; import { DataTransformerID } from './ids'; -import { JoinMode, SeriesToColumnsOptions, seriesToColumnsTransformer } from './seriesToColumns'; +import { JoinMode, JoinByFieldOptions, joinByFieldTransformer } from './joinByField'; -describe('SeriesToColumns Transformer', () => { +describe('JOIN Transformer', () => { beforeAll(() => { - mockTransformationsRegistry([seriesToColumnsTransformer]); + mockTransformationsRegistry([joinByFieldTransformer]); }); describe('outer join', () => { @@ -32,7 +32,7 @@ describe('SeriesToColumns Transformer', () => { }); it('joins by time field', async () => { - const cfg: DataTransformerConfig = { + const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'time', @@ -134,7 +134,7 @@ describe('SeriesToColumns Transformer', () => { }); it('joins by temperature field', async () => { - const cfg: DataTransformerConfig = { + const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'temperature', @@ -250,7 +250,7 @@ describe('SeriesToColumns Transformer', () => { }); it('joins by time field in reverse order', async () => { - const cfg: DataTransformerConfig = { + const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'time', @@ -375,7 +375,7 @@ describe('SeriesToColumns Transformer', () => { }); it('when dataframe and field share the same name then use the field name', async () => { - const cfg: DataTransformerConfig = { + const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'time', @@ -438,7 +438,7 @@ describe('SeriesToColumns Transformer', () => { }); it('joins if fields are missing', async () => { - const cfg: DataTransformerConfig = { + const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'time', @@ -516,7 +516,7 @@ describe('SeriesToColumns Transformer', () => { }); it('handles duplicate field name', async () => { - const cfg: DataTransformerConfig = { + const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'time', @@ -597,7 +597,7 @@ describe('SeriesToColumns Transformer', () => { }); it('inner joins by time field', async () => { - const cfg: DataTransformerConfig = { + const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'time', @@ -678,7 +678,7 @@ describe('SeriesToColumns Transformer', () => { }); it('inner joins by temperature field', async () => { - const cfg: DataTransformerConfig = { + const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'temperature', @@ -763,7 +763,7 @@ describe('SeriesToColumns Transformer', () => { }); it('inner joins by time field in reverse order', async () => { - const cfg: DataTransformerConfig = { + const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'time', @@ -867,7 +867,7 @@ describe('SeriesToColumns Transformer', () => { }); it('when dataframe and field share the same name then use the field name', async () => { - const cfg: DataTransformerConfig = { + const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'time', @@ -931,7 +931,7 @@ describe('SeriesToColumns Transformer', () => { }); it('joins if fields are missing', async () => { - const cfg: DataTransformerConfig = { + const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'time', @@ -1010,7 +1010,7 @@ describe('SeriesToColumns Transformer', () => { }); it('handles duplicate field name', async () => { - const cfg: DataTransformerConfig = { + const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'time', diff --git a/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts b/packages/grafana-data/src/transformations/transformers/joinByField.ts similarity index 61% rename from packages/grafana-data/src/transformations/transformers/seriesToColumns.ts rename to packages/grafana-data/src/transformations/transformers/joinByField.ts index cf034c8d623..3b229fc7c66 100644 --- a/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts +++ b/packages/grafana-data/src/transformations/transformers/joinByField.ts @@ -12,23 +12,25 @@ export enum JoinMode { inner = 'inner', } -export interface SeriesToColumnsOptions { +export interface JoinByFieldOptions { byField?: string; // empty will pick the field automatically mode?: JoinMode; } -export const seriesToColumnsTransformer: SynchronousDataTransformerInfo = { - id: DataTransformerID.seriesToColumns, - name: 'Series as columns', // Called 'Outer join' in the UI! - description: 'Groups series by field and returns values as columns', +export const joinByFieldTransformer: SynchronousDataTransformerInfo = { + id: DataTransformerID.joinByField, + aliasIds: [DataTransformerID.seriesToColumns], + name: 'Join by field', + description: + 'Combine rows from two or more tables, based on a related field between them. This can be used to outer join multiple time series on the _time_ field to show many time series in one table.', defaultOptions: { byField: undefined, // DEFAULT_KEY_FIELD, mode: JoinMode.outer, }, - operator: (options) => (source) => source.pipe(map((data) => seriesToColumnsTransformer.transformer(options)(data))), + operator: (options) => (source) => source.pipe(map((data) => joinByFieldTransformer.transformer(options)(data))), - transformer: (options: SeriesToColumnsOptions) => { + transformer: (options: JoinByFieldOptions) => { let joinBy: FieldMatcher | undefined = undefined; return (data: DataFrame[]) => { if (data.length > 1) { diff --git a/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts b/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts index c97cfe27d42..d0d3332dce5 100644 --- a/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts +++ b/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts @@ -4,8 +4,8 @@ import { mockTransformationsRegistry } from '../../utils/tests/mockTransformatio import { ArrayVector } from '../../vector'; import { calculateFieldTransformer } from './calculateField'; +import { JoinMode } from './joinByField'; import { isLikelyAscendingVector, joinDataFrames } from './joinDataFrames'; -import { JoinMode } from './seriesToColumns'; describe('align frames', () => { beforeAll(() => { diff --git a/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts b/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts index ee994921111..767161d2f9a 100644 --- a/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts +++ b/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts @@ -6,7 +6,7 @@ import { ArrayVector } from '../../vector'; import { fieldMatchers } from '../matchers'; import { FieldMatcherID } from '../matchers/ids'; -import { JoinMode } from './seriesToColumns'; +import { JoinMode } from './joinByField'; export function pickBestJoinField(data: DataFrame[]): FieldMatcher { const { timeField } = getTimeField(data[0]); @@ -34,7 +34,7 @@ export function pickBestJoinField(data: DataFrame[]): FieldMatcher { } /** - * @alpha + * @internal */ export interface JoinOptions { /** diff --git a/packages/grafana-data/src/utils/tests/mockTransformationsRegistry.ts b/packages/grafana-data/src/utils/tests/mockTransformationsRegistry.ts index ac1dba07d50..52ddc964cc9 100644 --- a/packages/grafana-data/src/utils/tests/mockTransformationsRegistry.ts +++ b/packages/grafana-data/src/utils/tests/mockTransformationsRegistry.ts @@ -6,6 +6,7 @@ export const mockTransformationsRegistry = (transformers: Array { return { id: t.id, + aliasIds: t.aliasIds, name: t.name, transformation: t, description: t.description, diff --git a/public/app/features/explore/utils/decorators.ts b/public/app/features/explore/utils/decorators.ts index f8f4b46ebef..32462a8a135 100644 --- a/public/app/features/explore/utils/decorators.ts +++ b/public/app/features/explore/utils/decorators.ts @@ -109,7 +109,7 @@ export const decorateWithTableResult = (data: ExplorePanelData): Observable = ({ const showFieldConfigsOption = panel && !panel.plugin?.fieldConfigRegistry.isEmpty(); let dataSelect = dataFrames; - if (selectedDataFrame === DataTransformerID.seriesToColumns) { + if (selectedDataFrame === DataTransformerID.joinByField) { dataSelect = data!; } @@ -67,7 +67,7 @@ export const InspectDataOptions: FC = ({ const parts: string[] = []; - if (selectedDataFrame === DataTransformerID.seriesToColumns) { + if (selectedDataFrame === DataTransformerID.joinByField) { parts.push(t({ id: 'dashboard.inspect-data.series-to-columns', message: 'Series joined by time' })); } else if (data.length > 1) { parts.push(getFrameDisplayName(data[selectedDataFrame as number])); diff --git a/public/app/features/inspector/InspectDataTab.tsx b/public/app/features/inspector/InspectDataTab.tsx index 850e4234116..8ba325e63a1 100644 --- a/public/app/features/inspector/InspectDataTab.tsx +++ b/public/app/features/inspector/InspectDataTab.tsx @@ -44,7 +44,7 @@ interface Props { } interface State { - /** The string is seriesToColumns transformation. Otherwise it is a dataframe index */ + /** The string is joinByField transformation. Otherwise it is a dataframe index */ selectedDataFrame: number | DataTransformerID; transformId: DataTransformerID; dataFrameIndex: number; @@ -197,7 +197,7 @@ export class InspectDataTab extends PureComponent { onDataFrameChange = (item: SelectableValue) => { this.setState({ transformId: - item.value === DataTransformerID.seriesToColumns ? DataTransformerID.seriesToColumns : DataTransformerID.noop, + item.value === DataTransformerID.joinByField ? DataTransformerID.joinByField : DataTransformerID.noop, dataFrameIndex: typeof item.value === 'number' ? item.value : 0, selectedDataFrame: item.value!, }); @@ -349,14 +349,14 @@ export class InspectDataTab extends PureComponent { function buildTransformationOptions() { const transformations: Array> = [ { - value: DataTransformerID.seriesToColumns, + value: DataTransformerID.joinByField, label: t({ id: 'dashboard.inspect-data.transformation', message: 'Series joined by time', }), transformer: { - id: DataTransformerID.seriesToColumns, - options: { byField: 'Time' }, + id: DataTransformerID.joinByField, + options: { byField: undefined }, // defaults to time field }, }, ]; diff --git a/public/app/features/transformers/editors/JoinByFieldTransformerEditor.tsx b/public/app/features/transformers/editors/JoinByFieldTransformerEditor.tsx new file mode 100644 index 00000000000..cffa84015e0 --- /dev/null +++ b/public/app/features/transformers/editors/JoinByFieldTransformerEditor.tsx @@ -0,0 +1,72 @@ +import React, { useCallback } from 'react'; + +import { + DataTransformerID, + SelectableValue, + standardTransformers, + TransformerRegistryItem, + TransformerUIProps, +} from '@grafana/data'; +import { JoinByFieldOptions, JoinMode } from '@grafana/data/src/transformations/transformers/joinByField'; +import { Select, InlineFieldRow, InlineField } from '@grafana/ui'; + +import { useAllFieldNamesFromDataFrames } from '../utils'; + +const modes = [ + { value: JoinMode.outer, label: 'OUTER', description: 'Keep all rows from any table with a value' }, + { value: JoinMode.inner, label: 'INNER', description: 'Drop rows that do not match a value in all tables' }, +]; + +export function SeriesToFieldsTransformerEditor({ input, options, onChange }: TransformerUIProps) { + const fieldNames = useAllFieldNamesFromDataFrames(input).map((item: string) => ({ label: item, value: item })); + + const onSelectField = useCallback( + (value: SelectableValue) => { + onChange({ + ...options, + byField: value?.value, + }); + }, + [onChange, options] + ); + + const onSetMode = useCallback( + (value: SelectableValue) => { + onChange({ + ...options, + mode: value?.value, + }); + }, + [onChange, options] + ); + + return ( + <> + + + + + + + ); +} + +export const joinByFieldTransformerRegistryItem: TransformerRegistryItem = { + id: DataTransformerID.joinByField, + aliasIds: [DataTransformerID.seriesToColumns], + editor: SeriesToFieldsTransformerEditor, + transformation: standardTransformers.joinByFieldTransformer, + name: standardTransformers.joinByFieldTransformer.name, + description: standardTransformers.joinByFieldTransformer.description, +}; diff --git a/public/app/features/transformers/editors/SeriesToFieldsTransformerEditor.tsx b/public/app/features/transformers/editors/SeriesToFieldsTransformerEditor.tsx deleted file mode 100644 index 79bdf28a027..00000000000 --- a/public/app/features/transformers/editors/SeriesToFieldsTransformerEditor.tsx +++ /dev/null @@ -1,49 +0,0 @@ -import React, { useCallback } from 'react'; - -import { - DataTransformerID, - SelectableValue, - standardTransformers, - TransformerRegistryItem, - TransformerUIProps, -} from '@grafana/data'; -import { SeriesToColumnsOptions } from '@grafana/data/src/transformations/transformers/seriesToColumns'; -import { Select } from '@grafana/ui'; - -import { useAllFieldNamesFromDataFrames } from '../utils'; - -export const SeriesToFieldsTransformerEditor: React.FC> = ({ - input, - options, - onChange, -}) => { - const fieldNames = useAllFieldNamesFromDataFrames(input).map((item: string) => ({ label: item, value: item })); - - const onSelectField = useCallback( - (value: SelectableValue) => { - onChange({ - ...options, - byField: value?.value, - }); - }, - [onChange, options] - ); - - return ( -
-
-
Field name
-