mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Prometheus: Incremental querying profile data updates (#70701)
* add prometheusIncrementalQueryInstrumentation feature flag, update instrumentation data
This commit is contained in:
Galen Kistler
GitHub
parent
f96ceb7804
commit
daf9f9cd19
@ -74,53 +74,54 @@ Some features are enabled by default. You can disable these feature by setting t
|
||||
These features are early in their development lifecycle and so are not yet supported in Grafana Cloud.
|
||||
Experimental features might be changed or removed without prior notice.
|
||||
|
||||
| Feature toggle name | Description |
|
||||
| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------ |
|
||||
| `live-service-web-worker` | This will use a webworker thread to processes events rather than the main thread |
|
||||
| `queryOverLive` | Use Grafana Live WebSocket to execute backend queries |
|
||||
| `lokiExperimentalStreaming` | Support new streaming approach for loki (prototype, needs special loki build) |
|
||||
| `storage` | Configurable storage for dashboards, datasources, and resources |
|
||||
| `newTraceViewHeader` | Shows the new trace view header |
|
||||
| `datasourceQueryMultiStatus` | Introduce HTTP 207 Multi Status for api/ds/query |
|
||||
| `traceToMetrics` | Enable trace to metrics links |
|
||||
| `prometheusWideSeries` | Enable wide series responses in the Prometheus datasource |
|
||||
| `canvasPanelNesting` | Allow elements nesting |
|
||||
| `scenes` | Experimental framework to build interactive dashboards |
|
||||
| `disableSecretsCompatibility` | Disable duplicated secret storage in legacy tables |
|
||||
| `logRequestsInstrumentedAsUnknown` | Logs the path for requests that are instrumented as unknown |
|
||||
| `showDashboardValidationWarnings` | Show warnings when dashboards do not validate against the schema |
|
||||
| `mysqlAnsiQuotes` | Use double quotes to escape keyword in a MySQL query |
|
||||
| `nestedFolderPicker` | Enables the still in-development new folder picker to support nested folders |
|
||||
| `showTraceId` | Show trace ids for requests |
|
||||
| `alertingBacktesting` | Rule backtesting API for alerting |
|
||||
| `editPanelCSVDragAndDrop` | Enables drag and drop for CSV and Excel files |
|
||||
| `lokiQuerySplitting` | Split large interval queries into subqueries with smaller time intervals |
|
||||
| `lokiQuerySplittingConfig` | Give users the option to configure split durations for Loki queries |
|
||||
| `individualCookiePreferences` | Support overriding cookie preferences per user |
|
||||
| `onlyExternalOrgRoleSync` | Prohibits a user from changing organization roles synced with external auth providers |
|
||||
| `traceqlSearch` | Enables the 'TraceQL Search' tab for the Tempo datasource which provides a UI to generate TraceQL queries |
|
||||
| `timeSeriesTable` | Enable time series table transformer & sparkline cell type |
|
||||
| `prometheusResourceBrowserCache` | Displays browser caching options in Prometheus data source configuration |
|
||||
| `influxdbBackendMigration` | Query InfluxDB InfluxQL without the proxy |
|
||||
| `clientTokenRotation` | Replaces the current in-request token rotation so that the client initiates the rotation |
|
||||
| `disableSSEDataplane` | Disables dataplane specific processing in server side expressions. |
|
||||
| `alertStateHistoryLokiSecondary` | Enable Grafana to write alert state history to an external Loki instance in addition to Grafana annotations. |
|
||||
| `alertStateHistoryLokiPrimary` | Enable a remote Loki instance as the primary source for state history reads. |
|
||||
| `alertStateHistoryLokiOnly` | Disable Grafana alerts from emitting annotations when a remote Loki instance is available. |
|
||||
| `unifiedRequestLog` | Writes error logs to the request logger |
|
||||
| `pyroscopeFlameGraph` | Changes flame graph to pyroscope one |
|
||||
| `extraThemes` | Enables extra themes |
|
||||
| `lokiPredefinedOperations` | Adds predefined query operations to Loki query editor |
|
||||
| `pluginsFrontendSandbox` | Enables the plugins frontend sandbox |
|
||||
| `frontendSandboxMonitorOnly` | Enables monitor only in the plugin frontend sandbox (if enabled) |
|
||||
| `cloudWatchLogsMonacoEditor` | Enables the Monaco editor for CloudWatch Logs queries |
|
||||
| `exploreScrollableLogsContainer` | Improves the scrolling behavior of logs in Explore |
|
||||
| `recordedQueriesMulti` | Enables writing multiple items from a single query within Recorded Queries |
|
||||
| `pluginsDynamicAngularDetectionPatterns` | Enables fetching Angular detection patterns for plugins from GCOM and fallback to hardcoded ones |
|
||||
| `alertingLokiRangeToInstant` | Rewrites eligible loki range queries to instant queries |
|
||||
| `flameGraphV2` | New version of flame graph with new features |
|
||||
| `elasticToggleableFilters` | Enable support to toggle filters off from the query through the Logs Details component |
|
||||
| `vizAndWidgetSplit` | Split panels between vizualizations and widgets |
|
||||
| Feature toggle name | Description |
|
||||
| ------------------------------------------- | ------------------------------------------------------------------------------------------------------------ |
|
||||
| `live-service-web-worker` | This will use a webworker thread to processes events rather than the main thread |
|
||||
| `queryOverLive` | Use Grafana Live WebSocket to execute backend queries |
|
||||
| `lokiExperimentalStreaming` | Support new streaming approach for loki (prototype, needs special loki build) |
|
||||
| `storage` | Configurable storage for dashboards, datasources, and resources |
|
||||
| `newTraceViewHeader` | Shows the new trace view header |
|
||||
| `datasourceQueryMultiStatus` | Introduce HTTP 207 Multi Status for api/ds/query |
|
||||
| `traceToMetrics` | Enable trace to metrics links |
|
||||
| `prometheusWideSeries` | Enable wide series responses in the Prometheus datasource |
|
||||
| `canvasPanelNesting` | Allow elements nesting |
|
||||
| `scenes` | Experimental framework to build interactive dashboards |
|
||||
| `disableSecretsCompatibility` | Disable duplicated secret storage in legacy tables |
|
||||
| `logRequestsInstrumentedAsUnknown` | Logs the path for requests that are instrumented as unknown |
|
||||
| `showDashboardValidationWarnings` | Show warnings when dashboards do not validate against the schema |
|
||||
| `mysqlAnsiQuotes` | Use double quotes to escape keyword in a MySQL query |
|
||||
| `nestedFolderPicker` | Enables the still in-development new folder picker to support nested folders |
|
||||
| `showTraceId` | Show trace ids for requests |
|
||||
| `alertingBacktesting` | Rule backtesting API for alerting |
|
||||
| `editPanelCSVDragAndDrop` | Enables drag and drop for CSV and Excel files |
|
||||
| `lokiQuerySplitting` | Split large interval queries into subqueries with smaller time intervals |
|
||||
| `lokiQuerySplittingConfig` | Give users the option to configure split durations for Loki queries |
|
||||
| `individualCookiePreferences` | Support overriding cookie preferences per user |
|
||||
| `onlyExternalOrgRoleSync` | Prohibits a user from changing organization roles synced with external auth providers |
|
||||
| `traceqlSearch` | Enables the 'TraceQL Search' tab for the Tempo datasource which provides a UI to generate TraceQL queries |
|
||||
| `timeSeriesTable` | Enable time series table transformer & sparkline cell type |
|
||||
| `prometheusResourceBrowserCache` | Displays browser caching options in Prometheus data source configuration |
|
||||
| `influxdbBackendMigration` | Query InfluxDB InfluxQL without the proxy |
|
||||
| `clientTokenRotation` | Replaces the current in-request token rotation so that the client initiates the rotation |
|
||||
| `disableSSEDataplane` | Disables dataplane specific processing in server side expressions. |
|
||||
| `alertStateHistoryLokiSecondary` | Enable Grafana to write alert state history to an external Loki instance in addition to Grafana annotations. |
|
||||
| `alertStateHistoryLokiPrimary` | Enable a remote Loki instance as the primary source for state history reads. |
|
||||
| `alertStateHistoryLokiOnly` | Disable Grafana alerts from emitting annotations when a remote Loki instance is available. |
|
||||
| `unifiedRequestLog` | Writes error logs to the request logger |
|
||||
| `pyroscopeFlameGraph` | Changes flame graph to pyroscope one |
|
||||
| `extraThemes` | Enables extra themes |
|
||||
| `lokiPredefinedOperations` | Adds predefined query operations to Loki query editor |
|
||||
| `pluginsFrontendSandbox` | Enables the plugins frontend sandbox |
|
||||
| `frontendSandboxMonitorOnly` | Enables monitor only in the plugin frontend sandbox (if enabled) |
|
||||
| `cloudWatchLogsMonacoEditor` | Enables the Monaco editor for CloudWatch Logs queries |
|
||||
| `exploreScrollableLogsContainer` | Improves the scrolling behavior of logs in Explore |
|
||||
| `recordedQueriesMulti` | Enables writing multiple items from a single query within Recorded Queries |
|
||||
| `pluginsDynamicAngularDetectionPatterns` | Enables fetching Angular detection patterns for plugins from GCOM and fallback to hardcoded ones |
|
||||
| `alertingLokiRangeToInstant` | Rewrites eligible loki range queries to instant queries |
|
||||
| `flameGraphV2` | New version of flame graph with new features |
|
||||
| `elasticToggleableFilters` | Enable support to toggle filters off from the query through the Logs Details component |
|
||||
| `vizAndWidgetSplit` | Split panels between vizualizations and widgets |
|
||||
| `prometheusIncrementalQueryInstrumentation` | Adds RudderStack events to incremental queries |
|
||||
|
||||
## Development feature toggles
|
||||
|
||||
|
||||
@ -108,4 +108,5 @@ export interface FeatureToggles {
|
||||
flameGraphV2?: boolean;
|
||||
elasticToggleableFilters?: boolean;
|
||||
vizAndWidgetSplit?: boolean;
|
||||
prometheusIncrementalQueryInstrumentation?: boolean;
|
||||
}
|
||||
|
||||
@ -614,5 +614,12 @@ var (
|
||||
FrontendOnly: true,
|
||||
Owner: grafanaDashboardsSquad,
|
||||
},
|
||||
{
|
||||
Name: "prometheusIncrementalQueryInstrumentation",
|
||||
Description: "Adds RudderStack events to incremental queries",
|
||||
FrontendOnly: true,
|
||||
Stage: FeatureStageExperimental,
|
||||
Owner: grafanaObservabilityMetricsSquad,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
@ -89,3 +89,4 @@ alertingLokiRangeToInstant,experimental,@grafana/alerting-squad,false,false,fals
|
||||
flameGraphV2,experimental,@grafana/observability-traces-and-profiling,false,false,false,true
|
||||
elasticToggleableFilters,experimental,@grafana/observability-logs,false,false,false,true
|
||||
vizAndWidgetSplit,experimental,@grafana/dashboards-squad,false,false,false,true
|
||||
prometheusIncrementalQueryInstrumentation,experimental,@grafana/observability-metrics,false,false,false,true
|
||||
|
||||
|
@ -366,4 +366,8 @@ const (
|
||||
// FlagVizAndWidgetSplit
|
||||
// Split panels between vizualizations and widgets
|
||||
FlagVizAndWidgetSplit = "vizAndWidgetSplit"
|
||||
|
||||
// FlagPrometheusIncrementalQueryInstrumentation
|
||||
// Adds RudderStack events to incremental queries
|
||||
FlagPrometheusIncrementalQueryInstrumentation = "prometheusIncrementalQueryInstrumentation"
|
||||
)
|
||||
|
||||
@ -29,8 +29,8 @@ export class TimeSrv {
|
||||
oldRefresh?: string;
|
||||
timeModel?: TimeModel;
|
||||
timeAtLoad: RawTimeRange;
|
||||
refreshMS?: number;
|
||||
private autoRefreshBlocked?: boolean;
|
||||
private refreshMS?: number;
|
||||
|
||||
constructor(private contextSrv: ContextSrv) {
|
||||
// default time
|
||||
|
||||
@ -9,9 +9,10 @@ import {
|
||||
parseDuration,
|
||||
} from '@grafana/data/src';
|
||||
import { faro } from '@grafana/faro-web-sdk';
|
||||
import { config } from '@grafana/runtime/src';
|
||||
import { config, reportInteraction } from '@grafana/runtime/src';
|
||||
import { amendTable, Table, trimTable } from 'app/features/live/data/amendTimeSeries';
|
||||
|
||||
import { getTimeSrv } from '../../../../features/dashboard/services/TimeSrv';
|
||||
import { PromQuery } from '../types';
|
||||
|
||||
// dashboardUID + panelId + refId
|
||||
@ -54,6 +55,9 @@ interface ProfileData extends DatasourceProfileData {
|
||||
bytes: number | null;
|
||||
dashboardUID: string;
|
||||
panelId?: number;
|
||||
from: string;
|
||||
queryRangeSeconds: number;
|
||||
refreshIntervalMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -77,8 +81,8 @@ export class QueryCache<T extends SupportedQueryTypes> {
|
||||
private perfObeserver?: PerformanceObserver;
|
||||
private shouldProfile: boolean;
|
||||
|
||||
// send profile events every 5 minutes
|
||||
sendEventsInterval = 60000 * 5;
|
||||
// send profile events every 10 minutes
|
||||
sendEventsInterval = 60000 * 10;
|
||||
|
||||
pendingRequestIdsToTargSigs = new Map<RequestID, ProfileData>();
|
||||
|
||||
@ -92,9 +96,11 @@ export class QueryCache<T extends SupportedQueryTypes> {
|
||||
panelId: string;
|
||||
dashId: string;
|
||||
expr: string;
|
||||
interval: string;
|
||||
refreshIntervalMs: number;
|
||||
sent: boolean;
|
||||
datasource: string;
|
||||
from: string;
|
||||
queryRangeSeconds: number;
|
||||
}
|
||||
>();
|
||||
|
||||
@ -114,7 +120,10 @@ export class QueryCache<T extends SupportedQueryTypes> {
|
||||
this.overlapWindowMs = durationToMilliseconds(duration);
|
||||
}
|
||||
|
||||
if (config.grafanaJavascriptAgent.enabled && options.profileFunction !== undefined) {
|
||||
if (
|
||||
(config.grafanaJavascriptAgent.enabled || config.featureToggles?.prometheusIncrementalQueryInstrumentation) &&
|
||||
options.profileFunction !== undefined
|
||||
) {
|
||||
this.profile();
|
||||
this.shouldProfile = true;
|
||||
} else {
|
||||
@ -167,8 +176,10 @@ export class QueryCache<T extends SupportedQueryTypes> {
|
||||
panelId: currentRequest.panelId?.toString() ?? '',
|
||||
dashId: currentRequest.dashboardUID ?? '',
|
||||
expr: currentRequest.expr ?? '',
|
||||
interval: currentRequest.interval ?? '',
|
||||
refreshIntervalMs: currentRequest.refreshIntervalMs ?? 0,
|
||||
sent: false,
|
||||
from: currentRequest.from ?? '',
|
||||
queryRangeSeconds: currentRequest.queryRangeSeconds ?? 0,
|
||||
});
|
||||
|
||||
// We don't need to save each subsequent request, only the first one
|
||||
@ -201,25 +212,36 @@ export class QueryCache<T extends SupportedQueryTypes> {
|
||||
|
||||
for (let [key, value] of entries) {
|
||||
if (!value.sent) {
|
||||
this.pendingAccumulatedEvents.set(key, { ...value, sent: true });
|
||||
faro.api.pushEvent(
|
||||
'incremental query response size',
|
||||
{
|
||||
datasource: value.datasource.toString(),
|
||||
requestCount: value.requestCount.toString(),
|
||||
savedBytesTotal: value.savedBytesTotal.toString(),
|
||||
initialRequestSize: value.initialRequestSize.toString(),
|
||||
lastRequestSize: value.lastRequestSize.toString(),
|
||||
panelId: value.panelId.toString(),
|
||||
dashId: value.dashId.toString(),
|
||||
expr: value.expr.toString(),
|
||||
interval: value.interval.toString(),
|
||||
},
|
||||
'no-interaction',
|
||||
{
|
||||
const event = {
|
||||
datasource: value.datasource.toString(),
|
||||
requestCount: value.requestCount.toString(),
|
||||
savedBytesTotal: value.savedBytesTotal.toString(),
|
||||
initialRequestSize: value.initialRequestSize.toString(),
|
||||
lastRequestSize: value.lastRequestSize.toString(),
|
||||
panelId: value.panelId.toString(),
|
||||
dashId: value.dashId.toString(),
|
||||
expr: value.expr.toString(),
|
||||
refreshIntervalMs: value.refreshIntervalMs.toString(),
|
||||
from: value.from.toString(),
|
||||
queryRangeSeconds: value.queryRangeSeconds.toString(),
|
||||
};
|
||||
|
||||
if (config.featureToggles.prometheusIncrementalQueryInstrumentation) {
|
||||
reportInteraction('grafana_incremental_queries_profile', event);
|
||||
} else if (faro.api.pushEvent) {
|
||||
faro.api.pushEvent('incremental query response size', event, 'no-interaction', {
|
||||
skipDedupe: true,
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
this.pendingAccumulatedEvents.set(key, {
|
||||
...value,
|
||||
sent: true,
|
||||
requestCount: 0,
|
||||
savedBytesTotal: 0,
|
||||
initialRequestSize: 0,
|
||||
lastRequestSize: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
@ -238,6 +260,8 @@ export class QueryCache<T extends SupportedQueryTypes> {
|
||||
let doPartialQuery = shouldCache;
|
||||
let prevTo: TimestampMs | undefined = undefined;
|
||||
|
||||
const refreshIntervalMs = getTimeSrv().refreshMS;
|
||||
|
||||
// pre-compute reqTargSigs
|
||||
const reqTargSigs = new Map<TargetIdent, TargetSig>();
|
||||
request.targets.forEach((targ) => {
|
||||
@ -251,6 +275,9 @@ export class QueryCache<T extends SupportedQueryTypes> {
|
||||
bytes: null,
|
||||
panelId: request.panelId,
|
||||
dashboardUID: request.dashboardUID ?? '',
|
||||
from: request.rangeRaw?.from.toString() ?? '',
|
||||
queryRangeSeconds: request.range.to.diff(request.range.from, 'seconds') ?? '',
|
||||
refreshIntervalMs: refreshIntervalMs ?? 0,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user