mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
57abf39f25
* Logs: Use result range instead of timepicker range for log histogram If a logs datasource does not send histogram data for the requested time range, the logs model computes a timeseries based on the log row counts, bucketed by an automcatically calculated time interval. Even when this histogram time series did not span the whole requested time range it was still rendered in the graph across the whole range, leaving an empty area at its start. Users find this confusing and are lead to believe their log data is missing. This change fixes this by anchoring the start of the timeseries on the first log row's timestamp from the result, and adds this smaller range as `visibleRange` to the logs model and passes it through to the logs component that optionally takes it into account to not render the empty area. The interval (bucket size) is also adjusted to account for a potentially finer resolution on the shorter visible time interval. The bucketsize multiplier was also changed from 10 to 20 to account for the space between the chart's bars. * Aligned visible range with buckets * Extract bucket size calculation and add test
634 lines
16 KiB
TypeScript
634 lines
16 KiB
TypeScript
import {
|
|
DataFrame,
|
|
FieldType,
|
|
LogLevel,
|
|
LogRowModel,
|
|
LogsDedupStrategy,
|
|
LogsMetaKind,
|
|
MutableDataFrame,
|
|
toDataFrame,
|
|
} from '@grafana/data';
|
|
import { dataFrameToLogsModel, dedupLogRows, getSeriesProperties } from './logs_model';
|
|
|
|
describe('dedupLogRows()', () => {
|
|
test('should return rows as is when dedup is set to none', () => {
|
|
const rows: LogRowModel[] = [
|
|
{
|
|
entry: 'WARN test 1.23 on [xxx]',
|
|
},
|
|
{
|
|
entry: 'WARN test 1.23 on [xxx]',
|
|
},
|
|
] as any;
|
|
expect(dedupLogRows(rows, LogsDedupStrategy.none)).toMatchObject(rows);
|
|
});
|
|
|
|
test('should dedup on exact matches', () => {
|
|
const rows: LogRowModel[] = [
|
|
{
|
|
entry: 'WARN test 1.23 on [xxx]',
|
|
},
|
|
{
|
|
entry: 'WARN test 1.23 on [xxx]',
|
|
},
|
|
{
|
|
entry: 'INFO test 2.44 on [xxx]',
|
|
},
|
|
{
|
|
entry: 'WARN test 1.23 on [xxx]',
|
|
},
|
|
] as any;
|
|
expect(dedupLogRows(rows, LogsDedupStrategy.exact)).toEqual([
|
|
{
|
|
duplicates: 1,
|
|
entry: 'WARN test 1.23 on [xxx]',
|
|
},
|
|
{
|
|
duplicates: 0,
|
|
entry: 'INFO test 2.44 on [xxx]',
|
|
},
|
|
{
|
|
duplicates: 0,
|
|
entry: 'WARN test 1.23 on [xxx]',
|
|
},
|
|
]);
|
|
});
|
|
|
|
test('should dedup on number matches', () => {
|
|
const rows: LogRowModel[] = [
|
|
{
|
|
entry: 'WARN test 1.2323423 on [xxx]',
|
|
},
|
|
{
|
|
entry: 'WARN test 1.23 on [xxx]',
|
|
},
|
|
{
|
|
entry: 'INFO test 2.44 on [xxx]',
|
|
},
|
|
{
|
|
entry: 'WARN test 1.23 on [xxx]',
|
|
},
|
|
] as any;
|
|
expect(dedupLogRows(rows, LogsDedupStrategy.numbers)).toEqual([
|
|
{
|
|
duplicates: 1,
|
|
entry: 'WARN test 1.2323423 on [xxx]',
|
|
},
|
|
{
|
|
duplicates: 0,
|
|
entry: 'INFO test 2.44 on [xxx]',
|
|
},
|
|
{
|
|
duplicates: 0,
|
|
entry: 'WARN test 1.23 on [xxx]',
|
|
},
|
|
]);
|
|
});
|
|
|
|
test('should dedup on signature matches', () => {
|
|
const rows: LogRowModel[] = [
|
|
{
|
|
entry: 'WARN test 1.2323423 on [xxx]',
|
|
},
|
|
{
|
|
entry: 'WARN test 1.23 on [xxx]',
|
|
},
|
|
{
|
|
entry: 'INFO test 2.44 on [xxx]',
|
|
},
|
|
{
|
|
entry: 'WARN test 1.23 on [xxx]',
|
|
},
|
|
] as any;
|
|
expect(dedupLogRows(rows, LogsDedupStrategy.signature)).toEqual([
|
|
{
|
|
duplicates: 3,
|
|
entry: 'WARN test 1.2323423 on [xxx]',
|
|
},
|
|
]);
|
|
});
|
|
|
|
test('should return to non-deduped state on same log result', () => {
|
|
const rows: LogRowModel[] = [
|
|
{
|
|
entry: 'INFO 123',
|
|
},
|
|
{
|
|
entry: 'WARN 123',
|
|
},
|
|
{
|
|
entry: 'WARN 123',
|
|
},
|
|
] as any;
|
|
expect(dedupLogRows(rows, LogsDedupStrategy.exact)).toEqual([
|
|
{
|
|
duplicates: 0,
|
|
entry: 'INFO 123',
|
|
},
|
|
{
|
|
duplicates: 1,
|
|
entry: 'WARN 123',
|
|
},
|
|
]);
|
|
|
|
expect(dedupLogRows(rows, LogsDedupStrategy.none)).toEqual(rows);
|
|
});
|
|
});
|
|
|
|
const emptyLogsModel: any = {
|
|
hasUniqueLabels: false,
|
|
rows: [],
|
|
meta: [],
|
|
series: [],
|
|
};
|
|
|
|
describe('dataFrameToLogsModel', () => {
|
|
it('given empty series should return empty logs model', () => {
|
|
expect(dataFrameToLogsModel([] as DataFrame[], 0, 'utc')).toMatchObject(emptyLogsModel);
|
|
});
|
|
|
|
it('given series without correct series name should return empty logs model', () => {
|
|
const series: DataFrame[] = [
|
|
toDataFrame({
|
|
fields: [],
|
|
}),
|
|
];
|
|
expect(dataFrameToLogsModel(series, 0, 'utc')).toMatchObject(emptyLogsModel);
|
|
});
|
|
|
|
it('given series without a time field should return empty logs model', () => {
|
|
const series: DataFrame[] = [
|
|
new MutableDataFrame({
|
|
fields: [
|
|
{
|
|
name: 'message',
|
|
type: FieldType.string,
|
|
values: [],
|
|
},
|
|
],
|
|
}),
|
|
];
|
|
expect(dataFrameToLogsModel(series, 0, 'utc')).toMatchObject(emptyLogsModel);
|
|
});
|
|
|
|
it('given series without a string field should return empty logs model', () => {
|
|
const series: DataFrame[] = [
|
|
new MutableDataFrame({
|
|
fields: [
|
|
{
|
|
name: 'time',
|
|
type: FieldType.time,
|
|
values: [],
|
|
},
|
|
],
|
|
}),
|
|
];
|
|
expect(dataFrameToLogsModel(series, 0, 'utc')).toMatchObject(emptyLogsModel);
|
|
});
|
|
|
|
it('given one series should return expected logs model', () => {
|
|
const series: DataFrame[] = [
|
|
new MutableDataFrame({
|
|
fields: [
|
|
{
|
|
name: 'time',
|
|
type: FieldType.time,
|
|
values: ['2019-04-26T09:28:11.352440161Z', '2019-04-26T14:42:50.991981292Z'],
|
|
},
|
|
{
|
|
name: 'message',
|
|
type: FieldType.string,
|
|
values: [
|
|
't=2019-04-26T11:05:28+0200 lvl=info msg="Initializing DatasourceCacheService" logger=server',
|
|
't=2019-04-26T16:42:50+0200 lvl=eror msg="new token…t unhashed token=56d9fdc5c8b7400bd51b060eea8ca9d7',
|
|
],
|
|
labels: {
|
|
filename: '/var/log/grafana/grafana.log',
|
|
job: 'grafana',
|
|
},
|
|
},
|
|
{
|
|
name: 'id',
|
|
type: FieldType.string,
|
|
values: ['foo', 'bar'],
|
|
},
|
|
],
|
|
meta: {
|
|
limit: 1000,
|
|
},
|
|
}),
|
|
];
|
|
const logsModel = dataFrameToLogsModel(series, 1, 'utc');
|
|
expect(logsModel.hasUniqueLabels).toBeFalsy();
|
|
expect(logsModel.rows).toHaveLength(2);
|
|
expect(logsModel.rows).toMatchObject([
|
|
{
|
|
entry: 't=2019-04-26T11:05:28+0200 lvl=info msg="Initializing DatasourceCacheService" logger=server',
|
|
labels: { filename: '/var/log/grafana/grafana.log', job: 'grafana' },
|
|
logLevel: 'info',
|
|
uniqueLabels: {},
|
|
uid: 'foo',
|
|
},
|
|
{
|
|
entry: 't=2019-04-26T16:42:50+0200 lvl=eror msg="new token…t unhashed token=56d9fdc5c8b7400bd51b060eea8ca9d7',
|
|
labels: { filename: '/var/log/grafana/grafana.log', job: 'grafana' },
|
|
logLevel: 'error',
|
|
uniqueLabels: {},
|
|
uid: 'bar',
|
|
},
|
|
]);
|
|
|
|
expect(logsModel.series).toHaveLength(2);
|
|
expect(logsModel.meta).toHaveLength(2);
|
|
expect(logsModel.meta![0]).toMatchObject({
|
|
label: 'Common labels',
|
|
value: series[0].fields[1].labels,
|
|
kind: LogsMetaKind.LabelsMap,
|
|
});
|
|
expect(logsModel.meta![1]).toMatchObject({
|
|
label: 'Limit',
|
|
value: `1000 (2 returned)`,
|
|
kind: LogsMetaKind.String,
|
|
});
|
|
});
|
|
|
|
it('given one series without labels should return expected logs model', () => {
|
|
const series: DataFrame[] = [
|
|
new MutableDataFrame({
|
|
fields: [
|
|
{
|
|
name: 'time',
|
|
type: FieldType.time,
|
|
values: ['1970-01-01T00:00:01Z'],
|
|
},
|
|
{
|
|
name: 'message',
|
|
type: FieldType.string,
|
|
values: ['WARN boooo'],
|
|
},
|
|
{
|
|
name: 'level',
|
|
type: FieldType.string,
|
|
values: ['dbug'],
|
|
},
|
|
],
|
|
}),
|
|
];
|
|
const logsModel = dataFrameToLogsModel(series, 1, 'utc');
|
|
expect(logsModel.rows).toHaveLength(1);
|
|
expect(logsModel.rows).toMatchObject([
|
|
{
|
|
entry: 'WARN boooo',
|
|
labels: {},
|
|
logLevel: LogLevel.debug,
|
|
uniqueLabels: {},
|
|
},
|
|
]);
|
|
});
|
|
|
|
it('given multiple series with unique times should return expected logs model', () => {
|
|
const series: DataFrame[] = [
|
|
toDataFrame({
|
|
fields: [
|
|
{
|
|
name: 'ts',
|
|
type: FieldType.time,
|
|
values: ['1970-01-01T00:00:01Z'],
|
|
},
|
|
{
|
|
name: 'line',
|
|
type: FieldType.string,
|
|
values: ['WARN boooo'],
|
|
labels: {
|
|
foo: 'bar',
|
|
baz: '1',
|
|
level: 'dbug',
|
|
},
|
|
},
|
|
{
|
|
name: 'id',
|
|
type: FieldType.string,
|
|
values: ['0'],
|
|
},
|
|
],
|
|
}),
|
|
toDataFrame({
|
|
name: 'logs',
|
|
fields: [
|
|
{
|
|
name: 'time',
|
|
type: FieldType.time,
|
|
values: ['1970-01-01T00:00:00Z', '1970-01-01T00:00:02Z'],
|
|
},
|
|
{
|
|
name: 'message',
|
|
type: FieldType.string,
|
|
values: ['INFO 1', 'INFO 2'],
|
|
labels: {
|
|
foo: 'bar',
|
|
baz: '2',
|
|
level: 'err',
|
|
},
|
|
},
|
|
{
|
|
name: 'id',
|
|
type: FieldType.string,
|
|
values: ['1', '2'],
|
|
},
|
|
],
|
|
}),
|
|
];
|
|
const logsModel = dataFrameToLogsModel(series, 1, 'utc');
|
|
expect(logsModel.hasUniqueLabels).toBeTruthy();
|
|
expect(logsModel.rows).toHaveLength(3);
|
|
expect(logsModel.rows).toMatchObject([
|
|
{
|
|
entry: 'INFO 1',
|
|
labels: { foo: 'bar', baz: '2' },
|
|
logLevel: LogLevel.error,
|
|
uniqueLabels: { baz: '2' },
|
|
},
|
|
{
|
|
entry: 'WARN boooo',
|
|
labels: { foo: 'bar', baz: '1' },
|
|
logLevel: LogLevel.debug,
|
|
uniqueLabels: { baz: '1' },
|
|
},
|
|
{
|
|
entry: 'INFO 2',
|
|
labels: { foo: 'bar', baz: '2' },
|
|
logLevel: LogLevel.error,
|
|
uniqueLabels: { baz: '2' },
|
|
},
|
|
]);
|
|
|
|
expect(logsModel.series).toHaveLength(2);
|
|
expect(logsModel.meta).toHaveLength(1);
|
|
expect(logsModel.meta![0]).toMatchObject({
|
|
label: 'Common labels',
|
|
value: {
|
|
foo: 'bar',
|
|
},
|
|
kind: LogsMetaKind.LabelsMap,
|
|
});
|
|
});
|
|
it('given multiple series with equal times should return expected logs model', () => {
|
|
const series: DataFrame[] = [
|
|
toDataFrame({
|
|
fields: [
|
|
{
|
|
name: 'ts',
|
|
type: FieldType.time,
|
|
values: ['1970-01-01T00:00:00Z'],
|
|
},
|
|
{
|
|
name: 'line',
|
|
type: FieldType.string,
|
|
values: ['WARN boooo 1'],
|
|
labels: {
|
|
foo: 'bar',
|
|
baz: '1',
|
|
level: 'dbug',
|
|
},
|
|
},
|
|
{
|
|
name: 'id',
|
|
type: FieldType.string,
|
|
values: ['0'],
|
|
},
|
|
],
|
|
}),
|
|
toDataFrame({
|
|
fields: [
|
|
{
|
|
name: 'ts',
|
|
type: FieldType.time,
|
|
values: ['1970-01-01T00:00:01Z'],
|
|
},
|
|
{
|
|
name: 'line',
|
|
type: FieldType.string,
|
|
values: ['WARN boooo 2'],
|
|
labels: {
|
|
foo: 'bar',
|
|
baz: '2',
|
|
level: 'dbug',
|
|
},
|
|
},
|
|
{
|
|
name: 'id',
|
|
type: FieldType.string,
|
|
values: ['1'],
|
|
},
|
|
],
|
|
}),
|
|
toDataFrame({
|
|
name: 'logs',
|
|
fields: [
|
|
{
|
|
name: 'time',
|
|
type: FieldType.time,
|
|
values: ['1970-01-01T00:00:00Z', '1970-01-01T00:00:01Z'],
|
|
},
|
|
{
|
|
name: 'message',
|
|
type: FieldType.string,
|
|
values: ['INFO 1', 'INFO 2'],
|
|
labels: {
|
|
foo: 'bar',
|
|
baz: '2',
|
|
level: 'err',
|
|
},
|
|
},
|
|
{
|
|
name: 'id',
|
|
type: FieldType.string,
|
|
values: ['2', '3'],
|
|
},
|
|
],
|
|
}),
|
|
];
|
|
const logsModel = dataFrameToLogsModel(series, 1, 'utc');
|
|
expect(logsModel.hasUniqueLabels).toBeTruthy();
|
|
expect(logsModel.rows).toHaveLength(4);
|
|
expect(logsModel.rows).toMatchObject([
|
|
{
|
|
entry: 'WARN boooo 1',
|
|
labels: { foo: 'bar', baz: '1' },
|
|
logLevel: LogLevel.debug,
|
|
uniqueLabels: { baz: '1' },
|
|
},
|
|
{
|
|
entry: 'INFO 1',
|
|
labels: { foo: 'bar', baz: '2' },
|
|
logLevel: LogLevel.error,
|
|
uniqueLabels: { baz: '2' },
|
|
},
|
|
{
|
|
entry: 'WARN boooo 2',
|
|
labels: { foo: 'bar', baz: '2' },
|
|
logLevel: LogLevel.debug,
|
|
uniqueLabels: { baz: '2' },
|
|
},
|
|
{
|
|
entry: 'INFO 2',
|
|
labels: { foo: 'bar', baz: '2' },
|
|
logLevel: LogLevel.error,
|
|
uniqueLabels: { baz: '2' },
|
|
},
|
|
]);
|
|
});
|
|
|
|
it('should fallback to row index if no id', () => {
|
|
const series: DataFrame[] = [
|
|
toDataFrame({
|
|
labels: { foo: 'bar' },
|
|
fields: [
|
|
{
|
|
name: 'ts',
|
|
type: FieldType.time,
|
|
values: ['1970-01-01T00:00:00Z'],
|
|
},
|
|
{
|
|
name: 'line',
|
|
type: FieldType.string,
|
|
values: ['WARN boooo 1'],
|
|
},
|
|
],
|
|
}),
|
|
];
|
|
const logsModel = dataFrameToLogsModel(series, 1, 'utc');
|
|
expect(logsModel.rows[0].uid).toBe('0');
|
|
});
|
|
|
|
it('given multiple series with equal ids should return expected logs model', () => {
|
|
const series: DataFrame[] = [
|
|
toDataFrame({
|
|
fields: [
|
|
{
|
|
name: 'ts',
|
|
type: FieldType.time,
|
|
values: ['1970-01-01T00:00:00Z'],
|
|
},
|
|
{
|
|
name: 'line',
|
|
type: FieldType.string,
|
|
values: ['WARN boooo 1'],
|
|
labels: {
|
|
foo: 'bar',
|
|
baz: '1',
|
|
level: 'dbug',
|
|
},
|
|
},
|
|
{
|
|
name: 'id',
|
|
type: FieldType.string,
|
|
values: ['0'],
|
|
},
|
|
],
|
|
}),
|
|
toDataFrame({
|
|
fields: [
|
|
{
|
|
name: 'ts',
|
|
type: FieldType.time,
|
|
values: ['1970-01-01T00:00:01Z'],
|
|
},
|
|
{
|
|
name: 'line',
|
|
type: FieldType.string,
|
|
values: ['WARN boooo 2'],
|
|
labels: {
|
|
foo: 'bar',
|
|
baz: '2',
|
|
level: 'dbug',
|
|
},
|
|
},
|
|
{
|
|
name: 'id',
|
|
type: FieldType.string,
|
|
values: ['1'],
|
|
},
|
|
],
|
|
}),
|
|
toDataFrame({
|
|
fields: [
|
|
{
|
|
name: 'ts',
|
|
type: FieldType.time,
|
|
values: ['1970-01-01T00:00:01Z'],
|
|
},
|
|
{
|
|
name: 'line',
|
|
type: FieldType.string,
|
|
values: ['WARN boooo 2'],
|
|
labels: {
|
|
foo: 'bar',
|
|
baz: '2',
|
|
level: 'dbug',
|
|
},
|
|
},
|
|
{
|
|
name: 'id',
|
|
type: FieldType.string,
|
|
values: ['1'],
|
|
},
|
|
],
|
|
}),
|
|
];
|
|
const logsModel = dataFrameToLogsModel(series, 0, 'utc');
|
|
expect(logsModel.hasUniqueLabels).toBeTruthy();
|
|
expect(logsModel.rows).toHaveLength(2);
|
|
expect(logsModel.rows).toMatchObject([
|
|
{
|
|
entry: 'WARN boooo 1',
|
|
labels: { foo: 'bar' },
|
|
logLevel: LogLevel.debug,
|
|
uniqueLabels: { baz: '1' },
|
|
},
|
|
{
|
|
entry: 'WARN boooo 2',
|
|
labels: { foo: 'bar' },
|
|
logLevel: LogLevel.debug,
|
|
uniqueLabels: { baz: '2' },
|
|
},
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe('getSeriesProperties()', () => {
|
|
it('sets a minimum bucket size', () => {
|
|
const result = getSeriesProperties([], 2, undefined, 3, 123);
|
|
expect(result.bucketSize).toBe(123);
|
|
});
|
|
|
|
it('does not adjust the bucketSize if there is no range', () => {
|
|
const result = getSeriesProperties([], 30, undefined, 70);
|
|
expect(result.bucketSize).toBe(2100);
|
|
});
|
|
|
|
it('does not adjust the bucketSize if the logs row times match the given range', () => {
|
|
const rows: LogRowModel[] = [
|
|
{ entry: 'foo', timeEpochMs: 10 },
|
|
{ entry: 'bar', timeEpochMs: 20 },
|
|
] as any;
|
|
const range = { from: 10, to: 20 };
|
|
const result = getSeriesProperties(rows, 1, range, 2, 1);
|
|
expect(result.bucketSize).toBe(2);
|
|
expect(result.visibleRange).toMatchObject(range);
|
|
});
|
|
|
|
it('clamps the range and adjusts the bucketSize if the logs row times do not completely cover the given range', () => {
|
|
const rows: LogRowModel[] = [
|
|
{ entry: 'foo', timeEpochMs: 10 },
|
|
{ entry: 'bar', timeEpochMs: 20 },
|
|
] as any;
|
|
const range = { from: 0, to: 30 };
|
|
const result = getSeriesProperties(rows, 3, range, 2, 1);
|
|
// Bucketsize 6 gets shortened to 4 because of new visible range is 20ms vs original range being 30ms
|
|
expect(result.bucketSize).toBe(4);
|
|
// From time is also aligned to bucketSize (divisible by 4)
|
|
expect(result.visibleRange).toMatchObject({ from: 8, to: 30 });
|
|
});
|
|
});
|