grafana/public/app/features/explore/utils/ResultProcessor.ts

138 lines
4.2 KiB
TypeScript
Raw Normal View History

import {
LogsModel,
GraphSeriesXY,
DataFrame,
FieldType,
TimeZone,
getDisplayProcessor,
Explore: Adds Loki explore query editor (#21497) * Explore: updates grafana-data explore query field props with explore mode * Explore: updates query row to pass down explore mode to query fields * Explore: adds LokiExploreQueryEditor * Explore: updates loki query field form to render children * Explore: adds loki explore extra field component * Explore: adds extra field element to loki query field form * Explore: updates loki explore query editor to use extra field element * Explore: moves ExploreMode to grafana-data * Explore: updates query row limit string * Explore: adds maxLines to DataQuery * Explore: adds maxLines to loki datasource runRangeQueryWithFallback * Explore: adds onChangeQueryLimit to LokiExploreQueryEditor * Explore: updates loki explore query editor to render extra field only in logs mode * Explore: fixes query limits for live and legacy queries * Explore: fixes result processor max lines limit in get logs result * Explore: fixes Loki datasource limit test * Explore: removes unnecessary ExploreMode from Loki language provider * Explore: fixes formatting * Explore: updates grafana-data datasource types - replaces strings with explore mode enum * Explore: updates loki explore query field props to take ReactNode * Explore: updates the way we calculate loki query lines limit to fall back to 0 lines on negative or invalid input instead of datasource maxLines * Explore: updates result processor get logs result method to avoid counting invalid/negative line limits * Explore: updates loki result transformer to process only an appropriate slice of a result instead of an entire one * Explore: adds a method for query limit preprocessing/mapping * Explore: updates loki datasource run range query with fallback method to use options.maxDataPoints in dashboards * Explore: removes unnecessary maxlineslimt from getLogsResult in resultProcessor * Explore: moves line limit to metadata * Explore: adds an ability to specify input type of extra field * Explore: updates LokiExploreQueryEditor - adds an input type * Explore: updates LokiExploreQueryEditor to run queries when maxLines is positive * Explore: fixes failing import of ExploreMode * Explore: fixes reducers test imports formatting * Explore: updates Loki extra field with min value set to 0 * Explore: exports LokiExploreExtraFieldProps * Explore: adds render test of LokiExploreQueryEditor * Explore: adds LokiExploreQueryEditor snapshot * Explore: updates LokiExploreQueryEditor onChangeQueryLimit method to prevent it from running when query input is empty - fixes cheatsheet display issue * Explore: updates Loki editor snapshots * Explore: fixes typo in test set name in LokiExploreQueryEditor * Explore: adds a render test of LokiExploreExtraField * Explore: fixes typo in LokiExploreQueryEditor * Explore: updates LokiExploreQueryEditor snapshot due to timezone issues * Explore: updates LokiExploreExtraField to export both functional component and a version using memo * Explore: updates LokiExploreQueryEditor to export both functional component and memoized function * Explore: updates LokiExploreQueryEditor - removes unnecessary react fragment * Explore: updates LokiExploreQueryEditor snapshot * Explore: adds LokiExploreQueryEditor tests for different explore mode cases * Explore: fixes Loki datasource and result transformer * Explore: updates LokiExploreQueryEditor snapshot * Explore: updates LokiExploreQueryEditor tests and test setup * Explore: updates LokiExploreQueryEditor - refactors component * Explore: updates LokiExploreQueryEditor to use default import from LokiExploreExtraField * Explore: updates LokiExploreQueryEditor snapshot * Explore: fixes formatting * Explore: updates LokiExploreQueryEditor max lines change * Explore: updates LokiExploreQueryEditor tests checking ExtraFieldElement * Explore: adds mock loki datasource to LokiExploreQueryEditor * Explore: updates LokiExploreQueryEditor test mock - adds language provider * Explore: updates LokiExploreQueryEditor snapshot * Explore: updates Loki ResultTransformer to filter out rows on limit - logic to be moved into a component with new form styles * Explore: updates LokiExploreQueryEditor tests
2020-02-06 06:34:52 -06:00
ExploreMode,
PreferredVisualisationType,
standardTransformers,
} from '@grafana/data';
Explore: Adds Loki explore query editor (#21497) * Explore: updates grafana-data explore query field props with explore mode * Explore: updates query row to pass down explore mode to query fields * Explore: adds LokiExploreQueryEditor * Explore: updates loki query field form to render children * Explore: adds loki explore extra field component * Explore: adds extra field element to loki query field form * Explore: updates loki explore query editor to use extra field element * Explore: moves ExploreMode to grafana-data * Explore: updates query row limit string * Explore: adds maxLines to DataQuery * Explore: adds maxLines to loki datasource runRangeQueryWithFallback * Explore: adds onChangeQueryLimit to LokiExploreQueryEditor * Explore: updates loki explore query editor to render extra field only in logs mode * Explore: fixes query limits for live and legacy queries * Explore: fixes result processor max lines limit in get logs result * Explore: fixes Loki datasource limit test * Explore: removes unnecessary ExploreMode from Loki language provider * Explore: fixes formatting * Explore: updates grafana-data datasource types - replaces strings with explore mode enum * Explore: updates loki explore query field props to take ReactNode * Explore: updates the way we calculate loki query lines limit to fall back to 0 lines on negative or invalid input instead of datasource maxLines * Explore: updates result processor get logs result method to avoid counting invalid/negative line limits * Explore: updates loki result transformer to process only an appropriate slice of a result instead of an entire one * Explore: adds a method for query limit preprocessing/mapping * Explore: updates loki datasource run range query with fallback method to use options.maxDataPoints in dashboards * Explore: removes unnecessary maxlineslimt from getLogsResult in resultProcessor * Explore: moves line limit to metadata * Explore: adds an ability to specify input type of extra field * Explore: updates LokiExploreQueryEditor - adds an input type * Explore: updates LokiExploreQueryEditor to run queries when maxLines is positive * Explore: fixes failing import of ExploreMode * Explore: fixes reducers test imports formatting * Explore: updates Loki extra field with min value set to 0 * Explore: exports LokiExploreExtraFieldProps * Explore: adds render test of LokiExploreQueryEditor * Explore: adds LokiExploreQueryEditor snapshot * Explore: updates LokiExploreQueryEditor onChangeQueryLimit method to prevent it from running when query input is empty - fixes cheatsheet display issue * Explore: updates Loki editor snapshots * Explore: fixes typo in test set name in LokiExploreQueryEditor * Explore: adds a render test of LokiExploreExtraField * Explore: fixes typo in LokiExploreQueryEditor * Explore: updates LokiExploreQueryEditor snapshot due to timezone issues * Explore: updates LokiExploreExtraField to export both functional component and a version using memo * Explore: updates LokiExploreQueryEditor to export both functional component and memoized function * Explore: updates LokiExploreQueryEditor - removes unnecessary react fragment * Explore: updates LokiExploreQueryEditor snapshot * Explore: adds LokiExploreQueryEditor tests for different explore mode cases * Explore: fixes Loki datasource and result transformer * Explore: updates LokiExploreQueryEditor snapshot * Explore: updates LokiExploreQueryEditor tests and test setup * Explore: updates LokiExploreQueryEditor - refactors component * Explore: updates LokiExploreQueryEditor to use default import from LokiExploreExtraField * Explore: updates LokiExploreQueryEditor snapshot * Explore: fixes formatting * Explore: updates LokiExploreQueryEditor max lines change * Explore: updates LokiExploreQueryEditor tests checking ExtraFieldElement * Explore: adds mock loki datasource to LokiExploreQueryEditor * Explore: updates LokiExploreQueryEditor test mock - adds language provider * Explore: updates LokiExploreQueryEditor snapshot * Explore: updates Loki ResultTransformer to filter out rows on limit - logic to be moved into a component with new form styles * Explore: updates LokiExploreQueryEditor tests
2020-02-06 06:34:52 -06:00
import { ExploreItemState } from 'app/types/explore';
import { sortLogsResult, refreshIntervalToSortOrder } from 'app/core/utils/explore';
import { dataFrameToLogsModel } from 'app/core/logs_model';
import { getGraphSeriesModel } from 'app/plugins/panel/graph2/getGraphSeriesModel';
import { config } from 'app/core/config';
export class ResultProcessor {
constructor(
private state: ExploreItemState,
private dataFrames: DataFrame[],
private intervalMs: number,
private timeZone: TimeZone
) {}
getGraphResult(): GraphSeriesXY[] | null {
if (this.state.mode !== ExploreMode.Metrics) {
QueryProcessing: Observable query interface and RxJS for query & stream processing (#18899) * I needed to learn some rxjs and understand this more, so just playing around * Updated * Removed all the complete calls * Refactoring * StreamHandler -> observable start * progress * simple singal works * Handle update time range * added error handling * wrap old function * minor changes * handle data format in the subscribe function * Use replay subject to return last value to subscribers * Set loading state after no response in 50ms * added missing file * updated comment * Added cancelation of network requests * runRequest: Added unit test scenario framework * Progress on tests * minor refactor of unit tests * updated test * removed some old code * Shared queries work again, and also became so much simplier * unified query and observe methods * implict any fix * Fixed closed subject issue * removed comment * Use last returned data for loading state * WIP: Explore to runRequest makover step1 * Minor progress * Minor progress on explore and runRequest * minor progress * Things are starting to work in explore * Updated prometheus to use new observable query response, greatly simplified code * Revert refId change * Found better solution for key/refId/requestId problem * use observable with loki * tests compile * fix loki query prep * Explore: correct first response handling * Refactorings * Refactoring * Explore: Fixes LoadingState and GraphResults between runs (#18986) * Refactor: Adds state to DataQueryResponse * Fix: Fixes so we do not empty results before new data arrives Fixes: #17409 * Transformations work * observable test data * remove single() from loki promise * Fixed comment * Explore: Fixes failing Loki and Prometheus unit tests (#18995) * Tests: Makes datasource tests work again * Fix: Fixes loki datasource so highligthing works * Chore: Runs Prettier * Fixed query runner tests * Delay loading state indication to 200ms * Fixed test * fixed unit tests * Clear cached calcs * Fixed bug getProcesedDataFrames * Fix the correct test is a better idea * Fix: Fixes so queries in Explore are only run if Graph/Table is shown (#19000) * Fix: Fixes so queries in Explore are only run if Graph/Table is shown Fixes: #18618 * Refactor: Removes unnecessary condition * PanelData: provide legacy data only when needed (#19018) * no legacy * invert logic... now compiles * merge getQueryResponseData and getDataRaw * update comment about query editor * use single getData() function * only send legacy when it is used in explore * pre process rather than post process * pre process rather than post process * Minor refactoring * Add missing tags to test datasource response * MixedDatasource: Adds query observable pattern to MixedDatasource (#19037) * start mixed datasource * Refactor: Refactors into observable parttern * Tests: Fixes tests * Tests: Removes console.log * Refactor: Adds unique requestId
2019-09-12 10:28:46 -05:00
return null;
}
const onlyTimeSeries = this.dataFrames.filter(frame => isTimeSeries(frame, this.state.datasourceInstance?.meta.id));
const timeSeriesToShowInGraph = onlyTimeSeries.filter(frame => shouldShowInVisualisationType(frame, 'graph'));
if (timeSeriesToShowInGraph.length === 0) {
QueryProcessing: Observable query interface and RxJS for query & stream processing (#18899) * I needed to learn some rxjs and understand this more, so just playing around * Updated * Removed all the complete calls * Refactoring * StreamHandler -> observable start * progress * simple singal works * Handle update time range * added error handling * wrap old function * minor changes * handle data format in the subscribe function * Use replay subject to return last value to subscribers * Set loading state after no response in 50ms * added missing file * updated comment * Added cancelation of network requests * runRequest: Added unit test scenario framework * Progress on tests * minor refactor of unit tests * updated test * removed some old code * Shared queries work again, and also became so much simplier * unified query and observe methods * implict any fix * Fixed closed subject issue * removed comment * Use last returned data for loading state * WIP: Explore to runRequest makover step1 * Minor progress * Minor progress on explore and runRequest * minor progress * Things are starting to work in explore * Updated prometheus to use new observable query response, greatly simplified code * Revert refId change * Found better solution for key/refId/requestId problem * use observable with loki * tests compile * fix loki query prep * Explore: correct first response handling * Refactorings * Refactoring * Explore: Fixes LoadingState and GraphResults between runs (#18986) * Refactor: Adds state to DataQueryResponse * Fix: Fixes so we do not empty results before new data arrives Fixes: #17409 * Transformations work * observable test data * remove single() from loki promise * Fixed comment * Explore: Fixes failing Loki and Prometheus unit tests (#18995) * Tests: Makes datasource tests work again * Fix: Fixes loki datasource so highligthing works * Chore: Runs Prettier * Fixed query runner tests * Delay loading state indication to 200ms * Fixed test * fixed unit tests * Clear cached calcs * Fixed bug getProcesedDataFrames * Fix the correct test is a better idea * Fix: Fixes so queries in Explore are only run if Graph/Table is shown (#19000) * Fix: Fixes so queries in Explore are only run if Graph/Table is shown Fixes: #18618 * Refactor: Removes unnecessary condition * PanelData: provide legacy data only when needed (#19018) * no legacy * invert logic... now compiles * merge getQueryResponseData and getDataRaw * update comment about query editor * use single getData() function * only send legacy when it is used in explore * pre process rather than post process * pre process rather than post process * Minor refactoring * Add missing tags to test datasource response * MixedDatasource: Adds query observable pattern to MixedDatasource (#19037) * start mixed datasource * Refactor: Refactors into observable parttern * Tests: Fixes tests * Tests: Removes console.log * Refactor: Adds unique requestId
2019-09-12 10:28:46 -05:00
return null;
}
2019-09-03 05:09:52 -05:00
return getGraphSeriesModel(
timeSeriesToShowInGraph,
this.timeZone,
2019-09-03 05:09:52 -05:00
{},
{ showBars: false, showLines: true, showPoints: false },
{ asTable: false, isVisible: true, placement: 'under' }
);
}
getTableResult(): DataFrame | null {
if (this.state.mode !== ExploreMode.Metrics) {
QueryProcessing: Observable query interface and RxJS for query & stream processing (#18899) * I needed to learn some rxjs and understand this more, so just playing around * Updated * Removed all the complete calls * Refactoring * StreamHandler -> observable start * progress * simple singal works * Handle update time range * added error handling * wrap old function * minor changes * handle data format in the subscribe function * Use replay subject to return last value to subscribers * Set loading state after no response in 50ms * added missing file * updated comment * Added cancelation of network requests * runRequest: Added unit test scenario framework * Progress on tests * minor refactor of unit tests * updated test * removed some old code * Shared queries work again, and also became so much simplier * unified query and observe methods * implict any fix * Fixed closed subject issue * removed comment * Use last returned data for loading state * WIP: Explore to runRequest makover step1 * Minor progress * Minor progress on explore and runRequest * minor progress * Things are starting to work in explore * Updated prometheus to use new observable query response, greatly simplified code * Revert refId change * Found better solution for key/refId/requestId problem * use observable with loki * tests compile * fix loki query prep * Explore: correct first response handling * Refactorings * Refactoring * Explore: Fixes LoadingState and GraphResults between runs (#18986) * Refactor: Adds state to DataQueryResponse * Fix: Fixes so we do not empty results before new data arrives Fixes: #17409 * Transformations work * observable test data * remove single() from loki promise * Fixed comment * Explore: Fixes failing Loki and Prometheus unit tests (#18995) * Tests: Makes datasource tests work again * Fix: Fixes loki datasource so highligthing works * Chore: Runs Prettier * Fixed query runner tests * Delay loading state indication to 200ms * Fixed test * fixed unit tests * Clear cached calcs * Fixed bug getProcesedDataFrames * Fix the correct test is a better idea * Fix: Fixes so queries in Explore are only run if Graph/Table is shown (#19000) * Fix: Fixes so queries in Explore are only run if Graph/Table is shown Fixes: #18618 * Refactor: Removes unnecessary condition * PanelData: provide legacy data only when needed (#19018) * no legacy * invert logic... now compiles * merge getQueryResponseData and getDataRaw * update comment about query editor * use single getData() function * only send legacy when it is used in explore * pre process rather than post process * pre process rather than post process * Minor refactoring * Add missing tags to test datasource response * MixedDatasource: Adds query observable pattern to MixedDatasource (#19037) * start mixed datasource * Refactor: Refactors into observable parttern * Tests: Fixes tests * Tests: Removes console.log * Refactor: Adds unique requestId
2019-09-12 10:28:46 -05:00
return null;
}
const onlyTables = this.dataFrames
.filter((frame: DataFrame) => shouldShowInVisualisationType(frame, 'table'))
.sort((frameA: DataFrame, frameB: DataFrame) => {
const frameARefId = frameA.refId!;
const frameBRefId = frameB.refId!;
if (frameARefId > frameBRefId) {
return 1;
}
if (frameARefId < frameBRefId) {
return -1;
}
return 0;
});
QueryProcessing: Observable query interface and RxJS for query & stream processing (#18899) * I needed to learn some rxjs and understand this more, so just playing around * Updated * Removed all the complete calls * Refactoring * StreamHandler -> observable start * progress * simple singal works * Handle update time range * added error handling * wrap old function * minor changes * handle data format in the subscribe function * Use replay subject to return last value to subscribers * Set loading state after no response in 50ms * added missing file * updated comment * Added cancelation of network requests * runRequest: Added unit test scenario framework * Progress on tests * minor refactor of unit tests * updated test * removed some old code * Shared queries work again, and also became so much simplier * unified query and observe methods * implict any fix * Fixed closed subject issue * removed comment * Use last returned data for loading state * WIP: Explore to runRequest makover step1 * Minor progress * Minor progress on explore and runRequest * minor progress * Things are starting to work in explore * Updated prometheus to use new observable query response, greatly simplified code * Revert refId change * Found better solution for key/refId/requestId problem * use observable with loki * tests compile * fix loki query prep * Explore: correct first response handling * Refactorings * Refactoring * Explore: Fixes LoadingState and GraphResults between runs (#18986) * Refactor: Adds state to DataQueryResponse * Fix: Fixes so we do not empty results before new data arrives Fixes: #17409 * Transformations work * observable test data * remove single() from loki promise * Fixed comment * Explore: Fixes failing Loki and Prometheus unit tests (#18995) * Tests: Makes datasource tests work again * Fix: Fixes loki datasource so highligthing works * Chore: Runs Prettier * Fixed query runner tests * Delay loading state indication to 200ms * Fixed test * fixed unit tests * Clear cached calcs * Fixed bug getProcesedDataFrames * Fix the correct test is a better idea * Fix: Fixes so queries in Explore are only run if Graph/Table is shown (#19000) * Fix: Fixes so queries in Explore are only run if Graph/Table is shown Fixes: #18618 * Refactor: Removes unnecessary condition * PanelData: provide legacy data only when needed (#19018) * no legacy * invert logic... now compiles * merge getQueryResponseData and getDataRaw * update comment about query editor * use single getData() function * only send legacy when it is used in explore * pre process rather than post process * pre process rather than post process * Minor refactoring * Add missing tags to test datasource response * MixedDatasource: Adds query observable pattern to MixedDatasource (#19037) * start mixed datasource * Refactor: Refactors into observable parttern * Tests: Fixes tests * Tests: Removes console.log * Refactor: Adds unique requestId
2019-09-12 10:28:46 -05:00
if (onlyTables.length === 0) {
return null;
}
const hasOnlyTimeseries = onlyTables.every(df => isTimeSeries(df));
// If we have only timeseries we do join on default time column which makes more sense. If we are showing
// non timeseries or some mix of data we are not trying to join on anything and just try to merge them in
// single table, which may not make sense in most cases, but it's up to the user to query something sensible.
const transformer = hasOnlyTimeseries
? standardTransformers.seriesToColumnsTransformer.transformer({})
: standardTransformers.mergeTransformer.transformer({});
const data = transformer(onlyTables)[0];
// set display processor
for (const field of data.fields) {
field.display = getDisplayProcessor({
field,
theme: config.theme,
DateTime: adding support to select preferred timezone for presentation of date and time values. (#23586) * added moment timezone package. * added a qnd way of selecting timezone. * added a first draft to display how it can be used. * fixed failing tests. * made moment.local to be in utc when running tests. * added tests to verify that the timeZone support works as expected. * Fixed so we use the formatter in the graph context menu. * changed so we will format d3 according to timeZone. * changed from class base to function based for easier consumption. * fixed so tests got green. * renamed to make it shorter. * fixed formatting in logRow. * removed unused value. * added time formatter to flot. * fixed failing tests. * changed so history will use the formatting with support for timezone. * added todo. * added so we append the correct abbrivation behind time. * added time zone abbrevation in timepicker. * adding timezone in rangeutil tool. * will use timezone when formatting range. * changed so we use new functions to format date so timezone is respected. * wip - dashboard settings. * changed so the time picker settings is in react. * added force update. * wip to get the react graph to work. * fixed formatting and parsing on the timepicker. * updated snap to be correct. * fixed so we format values properly in time picker. * make sure we pass timezone on all the proper places. * fixed so we use correct timeZone in explore. * fixed failing tests. * fixed so we always parse from local to selected timezone. * removed unused variable. * reverted back. * trying to fix issue with directive. * fixed issue. * fixed strict null errors. * fixed so we still can select default. * make sure we reads the time zone from getTimezone
2020-04-27 08:28:06 -05:00
timeZone: this.timeZone,
});
}
return data;
2019-09-03 05:09:52 -05:00
}
getLogsResult(): LogsModel | null {
if (this.state.mode !== ExploreMode.Logs) {
return null;
}
2019-09-03 05:09:52 -05:00
const newResults = dataFrameToLogsModel(this.dataFrames, this.intervalMs, this.timeZone, this.state.absoluteRange);
const sortOrder = refreshIntervalToSortOrder(this.state.refreshInterval);
const sortedNewResults = sortLogsResult(newResults, sortOrder);
const rows = sortedNewResults.rows;
const series = sortedNewResults.series;
return { ...sortedNewResults, rows, series };
2019-09-03 05:09:52 -05:00
}
}
2019-09-03 08:57:00 -05:00
function isTimeSeries(frame: DataFrame, datasource?: string): boolean {
// TEMP: Temporary hack. Remove when logs/metrics unification is done
if (datasource && datasource === 'cloudwatch') {
return isTimeSeriesCloudWatch(frame);
}
2019-09-03 08:57:00 -05:00
if (frame.fields.length === 2) {
if (frame.fields[0].type === FieldType.time) {
2019-09-03 08:57:00 -05:00
return true;
}
}
return false;
}
function shouldShowInVisualisationType(frame: DataFrame, visualisation: PreferredVisualisationType) {
if (frame.meta?.preferredVisualisationType && frame.meta?.preferredVisualisationType !== visualisation) {
return false;
}
return true;
}
// TEMP: Temporary hack. Remove when logs/metrics unification is done
function isTimeSeriesCloudWatch(frame: DataFrame): boolean {
return (
frame.fields.some(field => field.type === FieldType.time) &&
frame.fields.some(field => field.type === FieldType.number)
);
}