mirror of
https://github.com/grafana/grafana.git
synced 2025-01-13 09:32:12 -06:00
Loki: Fix metric time splitting to split starting with the start time (#80085)
* Loki: Fix metric time splitting to split starting with the start time * add test case to gdev-dashboard * fix splitting test
This commit is contained in:
Sven Grossmann
GitHub
parent
f0cb88e3b5
commit
d1b0e9082d
@ -18,7 +18,6 @@
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": 7004,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
@ -221,7 +220,8 @@
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@ -315,7 +315,8 @@
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@ -496,7 +497,8 @@
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@ -519,7 +521,7 @@
|
||||
},
|
||||
"showHeader": true
|
||||
},
|
||||
"pluginVersion": "10.2.0-61469",
|
||||
"pluginVersion": "10.3.0-pre",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -588,7 +590,8 @@
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@ -611,7 +614,7 @@
|
||||
},
|
||||
"showHeader": true
|
||||
},
|
||||
"pluginVersion": "10.2.0-61469",
|
||||
"pluginVersion": "10.3.0-pre",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -680,7 +683,8 @@
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@ -703,7 +707,7 @@
|
||||
},
|
||||
"showHeader": true
|
||||
},
|
||||
"pluginVersion": "10.2.0-61469",
|
||||
"pluginVersion": "10.3.0-pre",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -795,7 +799,8 @@
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@ -818,7 +823,7 @@
|
||||
},
|
||||
"showHeader": true
|
||||
},
|
||||
"pluginVersion": "10.2.0-61469",
|
||||
"pluginVersion": "10.3.0-pre",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -879,22 +884,256 @@
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "PDDA8E780A17E7EF1"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"cellOptions": {
|
||||
"type": "auto"
|
||||
},
|
||||
"inspect": false
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 36
|
||||
},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"cellHeight": "sm",
|
||||
"footer": {
|
||||
"countRows": false,
|
||||
"fields": "",
|
||||
"reducer": [
|
||||
"sum"
|
||||
],
|
||||
"show": false
|
||||
},
|
||||
"showHeader": true
|
||||
},
|
||||
"pluginVersion": "10.3.0-pre",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "gdev-loki"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(count_over_time({place=\"moon\"}[$__auto]))",
|
||||
"maxLines": 5,
|
||||
"queryType": "range",
|
||||
"refId": "A",
|
||||
"step": "1d"
|
||||
}
|
||||
],
|
||||
"title": "Metric 1d step",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "extractFields",
|
||||
"options": {
|
||||
"format": "auto",
|
||||
"keepTime": true,
|
||||
"replace": true,
|
||||
"source": "Line"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {
|
||||
"31": true,
|
||||
"32": true,
|
||||
"33": true,
|
||||
"34": true,
|
||||
"35": true,
|
||||
"36": true,
|
||||
"37": true,
|
||||
"38": true,
|
||||
"39": true,
|
||||
"40": true,
|
||||
"41": true,
|
||||
"42": true,
|
||||
"43": true,
|
||||
"44": true,
|
||||
"45": true,
|
||||
"46": true,
|
||||
"2023-09-20T14": true,
|
||||
"33+00": true,
|
||||
"caller": true,
|
||||
"main.go": true,
|
||||
"t": true,
|
||||
"ts": true
|
||||
},
|
||||
"indexByName": {},
|
||||
"renameByName": {
|
||||
"level": "nivel",
|
||||
"ts": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "PDDA8E780A17E7EF1"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"cellOptions": {
|
||||
"type": "auto"
|
||||
},
|
||||
"inspect": false
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 36
|
||||
},
|
||||
"id": 12,
|
||||
"options": {
|
||||
"cellHeight": "sm",
|
||||
"footer": {
|
||||
"countRows": false,
|
||||
"fields": "",
|
||||
"reducer": [
|
||||
"sum"
|
||||
],
|
||||
"show": false
|
||||
},
|
||||
"showHeader": true
|
||||
},
|
||||
"pluginVersion": "10.3.0-pre",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "gdev-loki"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(count_over_time({place=\"moon\"}[$__auto]))",
|
||||
"maxLines": 5,
|
||||
"queryType": "range",
|
||||
"refId": "do-not-chunk",
|
||||
"step": "1d"
|
||||
}
|
||||
],
|
||||
"title": "Metric 1d step, no splitting",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "extractFields",
|
||||
"options": {
|
||||
"format": "auto",
|
||||
"keepTime": true,
|
||||
"replace": true,
|
||||
"source": "Line"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {
|
||||
"31": true,
|
||||
"32": true,
|
||||
"33": true,
|
||||
"34": true,
|
||||
"35": true,
|
||||
"36": true,
|
||||
"37": true,
|
||||
"38": true,
|
||||
"39": true,
|
||||
"40": true,
|
||||
"41": true,
|
||||
"42": true,
|
||||
"43": true,
|
||||
"44": true,
|
||||
"45": true,
|
||||
"46": true,
|
||||
"2023-09-20T14": true,
|
||||
"33+00": true,
|
||||
"caller": true,
|
||||
"main.go": true,
|
||||
"t": true,
|
||||
"ts": true
|
||||
},
|
||||
"indexByName": {},
|
||||
"renameByName": {
|
||||
"level": "nivel",
|
||||
"ts": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"refresh": "",
|
||||
"schemaVersion": 38,
|
||||
"schemaVersion": 39,
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"from": "now-5m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Datasource tests - Loki query splitting",
|
||||
"uid": "dc4ec947-7e6b-4c3b-be8f-0abec7b0652a",
|
||||
"version": 13,
|
||||
"version": 2,
|
||||
"weekStart": ""
|
||||
}
|
||||
@ -1,15 +1,55 @@
|
||||
import { splitTimeRange } from './metricTimeSplitting';
|
||||
|
||||
describe('metric splitTimeRange', () => {
|
||||
it('should split time range into chunks', () => {
|
||||
const start = Date.parse('2022-02-06T14:10:03');
|
||||
const end = Date.parse('2022-02-06T14:11:03');
|
||||
const step = 10 * 1000;
|
||||
it('should split time range into chunks with 1day split and duration', () => {
|
||||
const start = Date.parse('2022-02-06T14:10:03Z');
|
||||
const end = Date.parse('2022-02-08T14:11:03Z');
|
||||
const step = 24 * 60 * 60 * 1000; // 1 day
|
||||
const rangeDuration = 24 * 60 * 60 * 1000; // 1 day
|
||||
|
||||
expect(splitTimeRange(start, end, step, 25000)).toStrictEqual([
|
||||
[Date.parse('2022-02-06T14:10:00'), Date.parse('2022-02-06T14:10:10')],
|
||||
[Date.parse('2022-02-06T14:10:20'), Date.parse('2022-02-06T14:10:40')],
|
||||
[Date.parse('2022-02-06T14:10:50'), Date.parse('2022-02-06T14:11:10')],
|
||||
expect(splitTimeRange(start, end, step, rangeDuration)).toStrictEqual([
|
||||
[Date.parse('2022-02-06T00:00:00Z'), Date.parse('2022-02-06T00:00:00Z')],
|
||||
[Date.parse('2022-02-07T00:00:00Z'), Date.parse('2022-02-07T00:00:00Z')],
|
||||
[Date.parse('2022-02-08T00:00:00Z'), Date.parse('2022-02-08T00:00:00Z')],
|
||||
]);
|
||||
});
|
||||
|
||||
it('should split time range into chunks with 1day split and duration and a 5 minute duration', () => {
|
||||
const start = Date.parse('2022-02-06T14:00:00Z');
|
||||
const end = Date.parse('2022-02-06T14:05:00Z');
|
||||
const step = 24 * 60 * 60 * 1000; // 1 day
|
||||
const rangeDuration = 24 * 60 * 60 * 1000; // 1 day
|
||||
|
||||
expect(splitTimeRange(start, end, step, rangeDuration)).toStrictEqual([
|
||||
[Date.parse('2022-02-06T00:00:00Z'), Date.parse('2022-02-06T00:00:00Z')],
|
||||
]);
|
||||
});
|
||||
|
||||
it('should split time range into chunks with 1hour split and 1day duration', () => {
|
||||
const start = Date.parse('2022-02-06T14:10:03Z');
|
||||
const end = Date.parse('2022-02-08T14:11:03Z');
|
||||
const step = 60 * 60 * 1000; // 1 hour
|
||||
const rangeDuration = 24 * 60 * 60 * 1000; // 1 day
|
||||
|
||||
expect(splitTimeRange(start, end, step, rangeDuration)).toStrictEqual([
|
||||
[Date.parse('2022-02-06T14:00:00Z'), Date.parse('2022-02-07T13:00:00Z')],
|
||||
[Date.parse('2022-02-07T14:00:00Z'), Date.parse('2022-02-08T13:00:00Z')],
|
||||
[Date.parse('2022-02-08T14:00:00Z'), Date.parse('2022-02-08T14:11:03Z')],
|
||||
]);
|
||||
});
|
||||
|
||||
it('should split time range into chunks with 1hour split and 12h duration', () => {
|
||||
const start = Date.parse('2022-02-06T14:10:03Z');
|
||||
const end = Date.parse('2022-02-08T14:11:03Z');
|
||||
const step = 60 * 60 * 1000; // 1 hour
|
||||
const rangeDuration = 12 * 60 * 60 * 1000; // 12h
|
||||
|
||||
expect(splitTimeRange(start, end, step, rangeDuration)).toStrictEqual([
|
||||
[Date.parse('2022-02-06T14:00:00Z'), Date.parse('2022-02-07T01:00:00Z')],
|
||||
[Date.parse('2022-02-07T02:00:00Z'), Date.parse('2022-02-07T13:00:00Z')],
|
||||
[Date.parse('2022-02-07T14:00:00Z'), Date.parse('2022-02-08T01:00:00Z')],
|
||||
[Date.parse('2022-02-08T02:00:00Z'), Date.parse('2022-02-08T13:00:00Z')],
|
||||
[Date.parse('2022-02-08T14:00:00Z'), Date.parse('2022-02-08T14:11:03Z')],
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@ -5,20 +5,6 @@
|
||||
// we are trying to be compatible with
|
||||
// https://github.com/grafana/loki/blob/089ec1b05f5ec15a8851d0e8230153e0eeb4dcec/pkg/querier/queryrange/split_by_interval.go#L327-L336
|
||||
|
||||
function expandTimeRange(startTime: number, endTime: number, step: number): [number, number] {
|
||||
// startTime is decreased to the closes multiple-of-step, if necessary
|
||||
const newStartTime = startTime - (startTime % step);
|
||||
|
||||
// endTime is increased to the closed multiple-of-step, if necessary
|
||||
let newEndTime = endTime;
|
||||
const endStepMod = endTime % step;
|
||||
if (endStepMod !== 0) {
|
||||
newEndTime += step - endStepMod;
|
||||
}
|
||||
|
||||
return [newStartTime, newEndTime];
|
||||
}
|
||||
|
||||
export function splitTimeRange(
|
||||
startTime: number,
|
||||
endTime: number,
|
||||
@ -33,20 +19,19 @@ export function splitTimeRange(
|
||||
// we make the duration a multiple of `step`, lowering it if necessary
|
||||
const alignedDuration = Math.trunc(idealRangeDuration / step) * step;
|
||||
|
||||
const [alignedStartTime, alignedEndTime] = expandTimeRange(startTime, endTime, step);
|
||||
const alignedStartTime = startTime - (startTime % step);
|
||||
|
||||
const result: Array<[number, number]> = [];
|
||||
|
||||
// we iterate it from the end, because we want to have the potentially smaller chunk at the end, not at the beginning
|
||||
for (let chunkEndTime = alignedEndTime; chunkEndTime > alignedStartTime; chunkEndTime -= alignedDuration + step) {
|
||||
// when we get close to the start of the time range, we need to be sure not
|
||||
// to cross over the startTime
|
||||
const chunkStartTime = Math.max(chunkEndTime - alignedDuration, alignedStartTime);
|
||||
// in a previous version we started iterating from the end, to the start.
|
||||
// However this is not easily possible as end timestamps are always inclusive
|
||||
// for Loki. So a `2022-02-08T00:00:00Z` end time with a 1day step would mean
|
||||
// to include the 08.02.2022, which we don't want. So we have to start from
|
||||
// the start, always ending at the last step before the actual end, or the total end.
|
||||
for (let chunkStartTime = alignedStartTime; chunkStartTime < endTime; chunkStartTime += alignedDuration) {
|
||||
const chunkEndTime = Math.min(chunkStartTime + alignedDuration - step, endTime);
|
||||
result.push([chunkStartTime, chunkEndTime]);
|
||||
}
|
||||
|
||||
// because we walked backwards, we need to reverse the array
|
||||
result.reverse();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -82,8 +82,8 @@ describe('runSplitQuery()', () => {
|
||||
intervalMs: 60000,
|
||||
range: expect.objectContaining({
|
||||
from: expect.objectContaining({
|
||||
//2023-02-09T06:00:00.000Z
|
||||
_i: 1675922400000,
|
||||
//2023-02-10T05:00:00.000Z
|
||||
_i: 1676005200000,
|
||||
}),
|
||||
to: expect.objectContaining({
|
||||
// 2023-02-10T06:00:00.000Z
|
||||
@ -100,12 +100,12 @@ describe('runSplitQuery()', () => {
|
||||
intervalMs: 60000,
|
||||
range: expect.objectContaining({
|
||||
from: expect.objectContaining({
|
||||
//2023-02-08T05:59:00.000Z
|
||||
_i: 1675835940000,
|
||||
// 2023-02-09T05:00:00.000Z
|
||||
_i: 1675918800000,
|
||||
}),
|
||||
to: expect.objectContaining({
|
||||
// 2023-02-09T05:59:00.000Z
|
||||
_i: 1675922340000,
|
||||
// 2023-02-10T04:59:00.000Z
|
||||
_i: 1676005140000,
|
||||
}),
|
||||
}),
|
||||
})
|
||||
@ -122,8 +122,8 @@ describe('runSplitQuery()', () => {
|
||||
_i: 1675832400000,
|
||||
}),
|
||||
to: expect.objectContaining({
|
||||
// 2023-02-08T05:58:00.000Z
|
||||
_i: 1675835880000,
|
||||
// 2023-02-09T04:59:00.000Z
|
||||
_i: 1675918740000,
|
||||
}),
|
||||
}),
|
||||
})
|
||||
@ -142,8 +142,8 @@ describe('runSplitQuery()', () => {
|
||||
intervalMs: 60000,
|
||||
range: expect.objectContaining({
|
||||
from: expect.objectContaining({
|
||||
//2023-02-09T06:00:00.000Z
|
||||
_i: 1675922400000,
|
||||
//2023-02-10T05:00:00.000Z
|
||||
_i: 1676005200000,
|
||||
}),
|
||||
to: expect.objectContaining({
|
||||
// 2023-02-10T06:00:00.000Z
|
||||
@ -160,12 +160,12 @@ describe('runSplitQuery()', () => {
|
||||
intervalMs: 60000,
|
||||
range: expect.objectContaining({
|
||||
from: expect.objectContaining({
|
||||
//2023-02-08T05:59:50.000Z
|
||||
_i: 1675835990000,
|
||||
// 2023-02-09T05:00:00.000Z
|
||||
_i: 1675918800000,
|
||||
}),
|
||||
to: expect.objectContaining({
|
||||
// 2023-02-09T05:59:50.000Z
|
||||
_i: 1675922390000,
|
||||
// 2023-02-10T04:59:50.000Z
|
||||
_i: 1676005190000,
|
||||
}),
|
||||
}),
|
||||
})
|
||||
@ -182,8 +182,8 @@ describe('runSplitQuery()', () => {
|
||||
_i: 1675832400000,
|
||||
}),
|
||||
to: expect.objectContaining({
|
||||
// 2023-02-08T05:59:40.000Z
|
||||
_i: 1675835980000,
|
||||
// 2023-02-09T04:59:50.000Z
|
||||
_i: 1675918790000,
|
||||
}),
|
||||
}),
|
||||
})
|
||||
|
||||
Loading…
Reference in New Issue
Block a user