diff --git a/devenv/dev-dashboards/datasource-loki/loki_query_splitting.json b/devenv/dev-dashboards/datasource-loki/loki_query_splitting.json index 35285857acc..804b354ea9d 100644 --- a/devenv/dev-dashboards/datasource-loki/loki_query_splitting.json +++ b/devenv/dev-dashboards/datasource-loki/loki_query_splitting.json @@ -1,900 +1,1139 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 7004, - "links": [], - "liveNow": false, - "panels": [ + "annotations": { + "list": [ { + "builtIn": 1, "datasource": { - "type": "loki", - "uid": "gdev-loki" + "type": "grafana", + "uid": "-- Grafana --" }, - "description": "Transformations:\n- Count\n- Sort by\n- Limit 10", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "editorMode": "code", - "expr": "{place=\"luna\"} | logfmt | label=\"val2\" | float > 60", - "maxLines": 5000, - "queryType": "range", - "refId": "A" - } - ], - "title": "Split logs", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "id_count", - "mode": "reduceRow", - "reduce": { - "include": [ - "id" - ], - "reducer": "count" - }, - "replaceFields": false - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "tsNs" - } - ] - } - }, - { - "id": "limit", - "options": { - "limitField": 10 - } - } - ], - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "description": "Transformations:\n- Count\n- Sort by\n- Limit 10", - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "editorMode": "code", - "expr": "{place=\"luna\"} | logfmt | label=\"val2\" | float > 60", - "maxLines": 5000, - "queryType": "range", - "refId": "do-not-chunk" - } - ], - "title": "Logs without splitting", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "id_count", - "mode": "reduceRow", - "reduce": { - "include": [ - "id" - ], - "reducer": "count" - }, - "replaceFields": false - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "tsNs" - } - ] - } - }, - { - "id": "limit", - "options": {} - } - ], - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 7 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "editorMode": "code", - "expr": "count_over_time({place=\"luna\"} | logfmt | label=\"val2\" | float > 60 | drop wave, _entry, level, float, counter [$__auto])", - "queryType": "range", - "refId": "A" - } - ], - "title": "Split TS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "editorMode": "code", - "expr": "count_over_time({place=\"luna\"} | logfmt | label=\"val2\" | float > 60 | drop wave, _entry, level, float, counter [$__auto])", - "queryType": "range", - "refId": "do-not-chunk" - } - ], - "title": "TS without splitting", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 5, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "editorMode": "code", - "expr": "{place=\"luna\"} | logfmt", - "maxLines": 5000, - "queryType": "range", - "refId": "A" - } - ], - "title": "Logs with filter transformation", - "transformations": [ - { - "id": "filterByValue", - "options": { - "filters": [ - { - "config": { - "id": "isNull", - "options": {} - }, - "fieldName": "TraceID" - } - ], - "match": "any", - "type": "exclude" - } - } - ], - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 15 - }, - "id": 6, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "editorMode": "code", - "expr": "{place=\"luna\"} | logfmt", - "maxLines": 5000, - "queryType": "range", - "refId": "do-not-chunk" - } - ], - "title": "Logs with filter transformation, no splitting", - "transformations": [ - { - "id": "filterByValue", - "options": { - "filters": [ - { - "config": { - "id": "isNull", - "options": {} - }, - "fieldName": "TraceID" - } - ], - "match": "any", - "type": "exclude" - } - } - ], - "type": "logs" - }, - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "id": 7, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.2.0-61469", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "editorMode": "code", - "expr": "{place=\"luna\", age=\"new\"}", - "maxLines": 5000, - "queryType": "range", - "refId": "A" - } - ], - "title": "Logs with extract fields transformation", - "transformations": [ - { - "id": "extractFields", - "options": { - "format": "json", - "keepTime": true, - "replace": true, - "source": "labels" - } - }, - { - "id": "calculateField", - "options": { - "alias": "Row", - "mode": "index", - "reduce": { - "reducer": "sum" - }, - "replaceFields": false - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 20 - }, - "id": 8, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.2.0-61469", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "editorMode": "code", - "expr": "{place=\"luna\", age=\"new\"}", - "maxLines": 5000, - "queryType": "range", - "refId": "do-not-chunk" - } - ], - "title": "Logs with extract fields transformation, no splitting", - "transformations": [ - { - "id": "extractFields", - "options": { - "format": "json", - "keepTime": true, - "replace": true, - "source": "labels" - } - }, - { - "id": "calculateField", - "options": { - "alias": "Row", - "mode": "index", - "reduce": { - "reducer": "sum" - }, - "replaceFields": false - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 9, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.2.0-61469", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "editorMode": "code", - "expr": "{place=\"moon\"}", - "maxLines": 5, - "queryType": "range", - "refId": "A" - } - ], - "title": "Logs with extract key=value and organize", - "transformations": [ - { - "id": "extractFields", - "options": { - "format": "auto", - "keepTime": true, - "replace": true, - "source": "Line" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "31": true, - "32": true, - "33": true, - "34": true, - "35": true, - "36": true, - "37": true, - "38": true, - "39": true, - "40": true, - "41": true, - "42": true, - "43": true, - "44": true, - "45": true, - "46": true, - "2023-09-20T14": true, - "33+00": true, - "caller": true, - "main.go": true, - "t": true, - "ts": true - }, - "indexByName": {}, - "renameByName": { - "level": "nivel", - "ts": "" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 28 - }, - "id": 10, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.2.0-61469", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "gdev-loki" - }, - "editorMode": "code", - "expr": "{place=\"moon\"}", - "maxLines": 5, - "queryType": "range", - "refId": "do-not-chunk" - } - ], - "title": "Logs with extract key=value and organize, no splitting", - "transformations": [ - { - "id": "extractFields", - "options": { - "format": "auto", - "keepTime": true, - "replace": true, - "source": "Line" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "31": true, - "32": true, - "33": true, - "34": true, - "35": true, - "36": true, - "37": true, - "38": true, - "39": true, - "40": true, - "41": true, - "42": true, - "43": true, - "44": true, - "45": true, - "46": true, - "2023-09-20T14": true, - "33+00": true, - "caller": true, - "main.go": true, - "t": true, - "ts": true - }, - "indexByName": {}, - "renameByName": { - "level": "nivel", - "ts": "" - } - } - } - ], - "type": "table" + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" } - ], - "refresh": "", - "schemaVersion": 38, - "tags": [], - "templating": { - "list": [] + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "description": "Transformations:\n- Count\n- Sort by\n- Limit 10", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "editorMode": "code", + "expr": "{place=\"luna\"} | logfmt | label=\"val2\" | float > 60", + "maxLines": 5000, + "queryType": "range", + "refId": "A" + } + ], + "title": "Split logs", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "id_count", + "mode": "reduceRow", + "reduce": { + "include": [ + "id" + ], + "reducer": "count" + }, + "replaceFields": false + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "tsNs" + } + ] + } + }, + { + "id": "limit", + "options": { + "limitField": 10 + } + } + ], + "type": "logs" }, - "time": { - "from": "now-6h", - "to": "now" + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "description": "Transformations:\n- Count\n- Sort by\n- Limit 10", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "editorMode": "code", + "expr": "{place=\"luna\"} | logfmt | label=\"val2\" | float > 60", + "maxLines": 5000, + "queryType": "range", + "refId": "do-not-chunk" + } + ], + "title": "Logs without splitting", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "id_count", + "mode": "reduceRow", + "reduce": { + "include": [ + "id" + ], + "reducer": "count" + }, + "replaceFields": false + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "tsNs" + } + ] + } + }, + { + "id": "limit", + "options": {} + } + ], + "type": "logs" }, - "timepicker": {}, - "timezone": "", - "title": "Datasource tests - Loki query splitting", - "uid": "dc4ec947-7e6b-4c3b-be8f-0abec7b0652a", - "version": 13, - "weekStart": "" - } \ No newline at end of file + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "editorMode": "code", + "expr": "count_over_time({place=\"luna\"} | logfmt | label=\"val2\" | float > 60 | drop wave, _entry, level, float, counter [$__auto])", + "queryType": "range", + "refId": "A" + } + ], + "title": "Split TS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "editorMode": "code", + "expr": "count_over_time({place=\"luna\"} | logfmt | label=\"val2\" | float > 60 | drop wave, _entry, level, float, counter [$__auto])", + "queryType": "range", + "refId": "do-not-chunk" + } + ], + "title": "TS without splitting", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 5, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "editorMode": "code", + "expr": "{place=\"luna\"} | logfmt", + "maxLines": 5000, + "queryType": "range", + "refId": "A" + } + ], + "title": "Logs with filter transformation", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "isNull", + "options": {} + }, + "fieldName": "TraceID" + } + ], + "match": "any", + "type": "exclude" + } + } + ], + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 6, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "editorMode": "code", + "expr": "{place=\"luna\"} | logfmt", + "maxLines": 5000, + "queryType": "range", + "refId": "do-not-chunk" + } + ], + "title": "Logs with filter transformation, no splitting", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "isNull", + "options": {} + }, + "fieldName": "TraceID" + } + ], + "match": "any", + "type": "exclude" + } + } + ], + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "editorMode": "code", + "expr": "{place=\"luna\", age=\"new\"}", + "maxLines": 5000, + "queryType": "range", + "refId": "A" + } + ], + "title": "Logs with extract fields transformation", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "json", + "keepTime": true, + "replace": true, + "source": "labels" + } + }, + { + "id": "calculateField", + "options": { + "alias": "Row", + "mode": "index", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "editorMode": "code", + "expr": "{place=\"luna\", age=\"new\"}", + "maxLines": 5000, + "queryType": "range", + "refId": "do-not-chunk" + } + ], + "title": "Logs with extract fields transformation, no splitting", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "json", + "keepTime": true, + "replace": true, + "source": "labels" + } + }, + { + "id": "calculateField", + "options": { + "alias": "Row", + "mode": "index", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 9, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "editorMode": "code", + "expr": "{place=\"moon\"}", + "maxLines": 5, + "queryType": "range", + "refId": "A" + } + ], + "title": "Logs with extract key=value and organize", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "auto", + "keepTime": true, + "replace": true, + "source": "Line" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "31": true, + "32": true, + "33": true, + "34": true, + "35": true, + "36": true, + "37": true, + "38": true, + "39": true, + "40": true, + "41": true, + "42": true, + "43": true, + "44": true, + "45": true, + "46": true, + "2023-09-20T14": true, + "33+00": true, + "caller": true, + "main.go": true, + "t": true, + "ts": true + }, + "indexByName": {}, + "renameByName": { + "level": "nivel", + "ts": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "editorMode": "code", + "expr": "{place=\"moon\"}", + "maxLines": 5, + "queryType": "range", + "refId": "do-not-chunk" + } + ], + "title": "Logs with extract key=value and organize, no splitting", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "auto", + "keepTime": true, + "replace": true, + "source": "Line" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "31": true, + "32": true, + "33": true, + "34": true, + "35": true, + "36": true, + "37": true, + "38": true, + "39": true, + "40": true, + "41": true, + "42": true, + "43": true, + "44": true, + "45": true, + "46": true, + "2023-09-20T14": true, + "33+00": true, + "caller": true, + "main.go": true, + "t": true, + "ts": true + }, + "indexByName": {}, + "renameByName": { + "level": "nivel", + "ts": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "PDDA8E780A17E7EF1" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 11, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "editorMode": "code", + "expr": "sum(count_over_time({place=\"moon\"}[$__auto]))", + "maxLines": 5, + "queryType": "range", + "refId": "A", + "step": "1d" + } + ], + "title": "Metric 1d step", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "auto", + "keepTime": true, + "replace": true, + "source": "Line" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "31": true, + "32": true, + "33": true, + "34": true, + "35": true, + "36": true, + "37": true, + "38": true, + "39": true, + "40": true, + "41": true, + "42": true, + "43": true, + "44": true, + "45": true, + "46": true, + "2023-09-20T14": true, + "33+00": true, + "caller": true, + "main.go": true, + "t": true, + "ts": true + }, + "indexByName": {}, + "renameByName": { + "level": "nivel", + "ts": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "PDDA8E780A17E7EF1" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 12, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.0-pre", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "gdev-loki" + }, + "editorMode": "code", + "expr": "sum(count_over_time({place=\"moon\"}[$__auto]))", + "maxLines": 5, + "queryType": "range", + "refId": "do-not-chunk", + "step": "1d" + } + ], + "title": "Metric 1d step, no splitting", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "auto", + "keepTime": true, + "replace": true, + "source": "Line" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "31": true, + "32": true, + "33": true, + "34": true, + "35": true, + "36": true, + "37": true, + "38": true, + "39": true, + "40": true, + "41": true, + "42": true, + "43": true, + "44": true, + "45": true, + "46": true, + "2023-09-20T14": true, + "33+00": true, + "caller": true, + "main.go": true, + "t": true, + "ts": true + }, + "indexByName": {}, + "renameByName": { + "level": "nivel", + "ts": "" + } + } + } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Datasource tests - Loki query splitting", + "uid": "dc4ec947-7e6b-4c3b-be8f-0abec7b0652a", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/public/app/plugins/datasource/loki/metricTimeSplitting.test.ts b/public/app/plugins/datasource/loki/metricTimeSplitting.test.ts index 52bd39e068b..8d836f82894 100644 --- a/public/app/plugins/datasource/loki/metricTimeSplitting.test.ts +++ b/public/app/plugins/datasource/loki/metricTimeSplitting.test.ts @@ -1,15 +1,55 @@ import { splitTimeRange } from './metricTimeSplitting'; describe('metric splitTimeRange', () => { - it('should split time range into chunks', () => { - const start = Date.parse('2022-02-06T14:10:03'); - const end = Date.parse('2022-02-06T14:11:03'); - const step = 10 * 1000; + it('should split time range into chunks with 1day split and duration', () => { + const start = Date.parse('2022-02-06T14:10:03Z'); + const end = Date.parse('2022-02-08T14:11:03Z'); + const step = 24 * 60 * 60 * 1000; // 1 day + const rangeDuration = 24 * 60 * 60 * 1000; // 1 day - expect(splitTimeRange(start, end, step, 25000)).toStrictEqual([ - [Date.parse('2022-02-06T14:10:00'), Date.parse('2022-02-06T14:10:10')], - [Date.parse('2022-02-06T14:10:20'), Date.parse('2022-02-06T14:10:40')], - [Date.parse('2022-02-06T14:10:50'), Date.parse('2022-02-06T14:11:10')], + expect(splitTimeRange(start, end, step, rangeDuration)).toStrictEqual([ + [Date.parse('2022-02-06T00:00:00Z'), Date.parse('2022-02-06T00:00:00Z')], + [Date.parse('2022-02-07T00:00:00Z'), Date.parse('2022-02-07T00:00:00Z')], + [Date.parse('2022-02-08T00:00:00Z'), Date.parse('2022-02-08T00:00:00Z')], + ]); + }); + + it('should split time range into chunks with 1day split and duration and a 5 minute duration', () => { + const start = Date.parse('2022-02-06T14:00:00Z'); + const end = Date.parse('2022-02-06T14:05:00Z'); + const step = 24 * 60 * 60 * 1000; // 1 day + const rangeDuration = 24 * 60 * 60 * 1000; // 1 day + + expect(splitTimeRange(start, end, step, rangeDuration)).toStrictEqual([ + [Date.parse('2022-02-06T00:00:00Z'), Date.parse('2022-02-06T00:00:00Z')], + ]); + }); + + it('should split time range into chunks with 1hour split and 1day duration', () => { + const start = Date.parse('2022-02-06T14:10:03Z'); + const end = Date.parse('2022-02-08T14:11:03Z'); + const step = 60 * 60 * 1000; // 1 hour + const rangeDuration = 24 * 60 * 60 * 1000; // 1 day + + expect(splitTimeRange(start, end, step, rangeDuration)).toStrictEqual([ + [Date.parse('2022-02-06T14:00:00Z'), Date.parse('2022-02-07T13:00:00Z')], + [Date.parse('2022-02-07T14:00:00Z'), Date.parse('2022-02-08T13:00:00Z')], + [Date.parse('2022-02-08T14:00:00Z'), Date.parse('2022-02-08T14:11:03Z')], + ]); + }); + + it('should split time range into chunks with 1hour split and 12h duration', () => { + const start = Date.parse('2022-02-06T14:10:03Z'); + const end = Date.parse('2022-02-08T14:11:03Z'); + const step = 60 * 60 * 1000; // 1 hour + const rangeDuration = 12 * 60 * 60 * 1000; // 12h + + expect(splitTimeRange(start, end, step, rangeDuration)).toStrictEqual([ + [Date.parse('2022-02-06T14:00:00Z'), Date.parse('2022-02-07T01:00:00Z')], + [Date.parse('2022-02-07T02:00:00Z'), Date.parse('2022-02-07T13:00:00Z')], + [Date.parse('2022-02-07T14:00:00Z'), Date.parse('2022-02-08T01:00:00Z')], + [Date.parse('2022-02-08T02:00:00Z'), Date.parse('2022-02-08T13:00:00Z')], + [Date.parse('2022-02-08T14:00:00Z'), Date.parse('2022-02-08T14:11:03Z')], ]); }); diff --git a/public/app/plugins/datasource/loki/metricTimeSplitting.ts b/public/app/plugins/datasource/loki/metricTimeSplitting.ts index facbf199515..0af619673ee 100644 --- a/public/app/plugins/datasource/loki/metricTimeSplitting.ts +++ b/public/app/plugins/datasource/loki/metricTimeSplitting.ts @@ -5,20 +5,6 @@ // we are trying to be compatible with // https://github.com/grafana/loki/blob/089ec1b05f5ec15a8851d0e8230153e0eeb4dcec/pkg/querier/queryrange/split_by_interval.go#L327-L336 -function expandTimeRange(startTime: number, endTime: number, step: number): [number, number] { - // startTime is decreased to the closes multiple-of-step, if necessary - const newStartTime = startTime - (startTime % step); - - // endTime is increased to the closed multiple-of-step, if necessary - let newEndTime = endTime; - const endStepMod = endTime % step; - if (endStepMod !== 0) { - newEndTime += step - endStepMod; - } - - return [newStartTime, newEndTime]; -} - export function splitTimeRange( startTime: number, endTime: number, @@ -33,20 +19,19 @@ export function splitTimeRange( // we make the duration a multiple of `step`, lowering it if necessary const alignedDuration = Math.trunc(idealRangeDuration / step) * step; - const [alignedStartTime, alignedEndTime] = expandTimeRange(startTime, endTime, step); + const alignedStartTime = startTime - (startTime % step); const result: Array<[number, number]> = []; - // we iterate it from the end, because we want to have the potentially smaller chunk at the end, not at the beginning - for (let chunkEndTime = alignedEndTime; chunkEndTime > alignedStartTime; chunkEndTime -= alignedDuration + step) { - // when we get close to the start of the time range, we need to be sure not - // to cross over the startTime - const chunkStartTime = Math.max(chunkEndTime - alignedDuration, alignedStartTime); + // in a previous version we started iterating from the end, to the start. + // However this is not easily possible as end timestamps are always inclusive + // for Loki. So a `2022-02-08T00:00:00Z` end time with a 1day step would mean + // to include the 08.02.2022, which we don't want. So we have to start from + // the start, always ending at the last step before the actual end, or the total end. + for (let chunkStartTime = alignedStartTime; chunkStartTime < endTime; chunkStartTime += alignedDuration) { + const chunkEndTime = Math.min(chunkStartTime + alignedDuration - step, endTime); result.push([chunkStartTime, chunkEndTime]); } - // because we walked backwards, we need to reverse the array - result.reverse(); - return result; } diff --git a/public/app/plugins/datasource/loki/querySplitting.test.ts b/public/app/plugins/datasource/loki/querySplitting.test.ts index 43a58853702..6cf98353d40 100644 --- a/public/app/plugins/datasource/loki/querySplitting.test.ts +++ b/public/app/plugins/datasource/loki/querySplitting.test.ts @@ -82,8 +82,8 @@ describe('runSplitQuery()', () => { intervalMs: 60000, range: expect.objectContaining({ from: expect.objectContaining({ - //2023-02-09T06:00:00.000Z - _i: 1675922400000, + //2023-02-10T05:00:00.000Z + _i: 1676005200000, }), to: expect.objectContaining({ // 2023-02-10T06:00:00.000Z @@ -100,12 +100,12 @@ describe('runSplitQuery()', () => { intervalMs: 60000, range: expect.objectContaining({ from: expect.objectContaining({ - //2023-02-08T05:59:00.000Z - _i: 1675835940000, + // 2023-02-09T05:00:00.000Z + _i: 1675918800000, }), to: expect.objectContaining({ - // 2023-02-09T05:59:00.000Z - _i: 1675922340000, + // 2023-02-10T04:59:00.000Z + _i: 1676005140000, }), }), }) @@ -118,12 +118,12 @@ describe('runSplitQuery()', () => { intervalMs: 60000, range: expect.objectContaining({ from: expect.objectContaining({ - //2023-02-08T05:00:00.000Z + // 2023-02-08T05:00:00.000Z _i: 1675832400000, }), to: expect.objectContaining({ - // 2023-02-08T05:58:00.000Z - _i: 1675835880000, + // 2023-02-09T04:59:00.000Z + _i: 1675918740000, }), }), }) @@ -142,8 +142,8 @@ describe('runSplitQuery()', () => { intervalMs: 60000, range: expect.objectContaining({ from: expect.objectContaining({ - //2023-02-09T06:00:00.000Z - _i: 1675922400000, + //2023-02-10T05:00:00.000Z + _i: 1676005200000, }), to: expect.objectContaining({ // 2023-02-10T06:00:00.000Z @@ -160,12 +160,12 @@ describe('runSplitQuery()', () => { intervalMs: 60000, range: expect.objectContaining({ from: expect.objectContaining({ - //2023-02-08T05:59:50.000Z - _i: 1675835990000, + // 2023-02-09T05:00:00.000Z + _i: 1675918800000, }), to: expect.objectContaining({ - // 2023-02-09T05:59:50.000Z - _i: 1675922390000, + // 2023-02-10T04:59:50.000Z + _i: 1676005190000, }), }), }) @@ -182,8 +182,8 @@ describe('runSplitQuery()', () => { _i: 1675832400000, }), to: expect.objectContaining({ - // 2023-02-08T05:59:40.000Z - _i: 1675835980000, + // 2023-02-09T04:59:50.000Z + _i: 1675918790000, }), }), })