Query Caching: Add per-panel query caching TTL (#61968)

* *Create Caching Config interface and OSS impl *Create front-end facing DS Cache config *Populate Caching Config on Datasource DTO *Update OSS wire deps * fix unit test * handle query caching TTL override on the frontend * Make sure the override works with pubdash * move caching config to the right place in the ds info * move caching config logic to enterprise index hook * move queryCachingTTL to pubdash query payload * Remove from metadata (not needed) * rename struct and add comment * remove invalid wire dependency * manual revert of 395c74b * fix frontend test * fix backend test * fix tests for real this time * truly fix frontend test * fix back end unit test for real
2024-11-25 18:30:41 -06:00 · 2023-02-02 23:39:54 -05:00 · 2023-02-02 23:39:54 -05:00 · 7391793504
commit 7391793504
parent 9eeea8f5ea
17 changed files with 101 additions and 12 deletions
--- a/packages/grafana-data/src/types/datasource.ts
+++ b/packages/grafana-data/src/types/datasource.ts
@ -145,6 +145,10 @@ interface PluginMetaQueryOptions {
  maxDataPoints?: boolean;
  minInterval?: boolean;
 }
+interface PluginQueryCachingConfig {
+  enabled?: boolean;
+  TTLMs?: number;
+}

 export interface DataSourcePluginComponents<
  DSType extends DataSourceApi<TQuery, TOptions>,
@ -224,6 +228,7 @@ abstract class DataSourceApi<
    this.id = instanceSettings.id;
    this.type = instanceSettings.type;
    this.meta = instanceSettings.meta;
+    this.cachingConfig = instanceSettings.cachingConfig;
    this.uid = instanceSettings.uid;
  }

@ -301,6 +306,12 @@ abstract class DataSourceApi<
   */
  meta: DataSourcePluginMeta;

+  /**
+   * Information about the datasource's query caching configuration
+   * When the caching feature is disabled, this config will always be falsy
+   */
+  cachingConfig?: PluginQueryCachingConfig;
+
  /**
   * Used by alerting to check if query contains template variables
   */
@ -487,6 +498,7 @@ export interface DataQueryRequest<TQuery extends DataQuery = DataQuery> {
  app: CoreApp | string;

  cacheTimeout?: string | null;
+  queryCachingTTL?: number | null;
  rangeRaw?: RawTimeRange;
  timeInfo?: string; // The query time description (blue text in the upper right)
  panelId?: number;
@ -586,6 +598,7 @@ export interface DataSourceInstanceSettings<T extends DataSourceJsonData = DataS
  type: string;
  name: string;
  meta: DataSourcePluginMeta;
+  cachingConfig?: PluginQueryCachingConfig;
  readOnly: boolean;
  url?: string;
  jsonData: T;
--- a/packages/grafana-data/src/types/queryRunner.ts
+++ b/packages/grafana-data/src/types/queryRunner.ts
@ -23,6 +23,7 @@ export interface QueryRunnerOptions {
  minInterval: string | undefined | null;
  scopedVars?: ScopedVars;
  cacheTimeout?: string;
+  queryCachingTTL?: number;
  app?: string;
 }

--- a/packages/grafana-runtime/src/utils/DataSourceWithBackend.test.ts
+++ b/packages/grafana-runtime/src/utils/DataSourceWithBackend.test.ts
@ -66,6 +66,7 @@ describe('DataSourceWithBackend', () => {
              "datasourceId": 1234,
              "intervalMs": 5000,
              "maxDataPoints": 10,
+              "queryCachingTTL": undefined,
              "refId": "A",
            },
            {
@ -76,6 +77,7 @@ describe('DataSourceWithBackend', () => {
              "datasourceId": undefined,
              "intervalMs": 5000,
              "maxDataPoints": 10,
+              "queryCachingTTL": undefined,
              "refId": "B",
            },
          ],
@ -133,6 +135,7 @@ describe('DataSourceWithBackend', () => {
              "datasourceId": 1234,
              "intervalMs": 5000,
              "maxDataPoints": 10,
+              "queryCachingTTL": undefined,
              "refId": "A",
            },
            {
@ -143,6 +146,7 @@ describe('DataSourceWithBackend', () => {
              "datasourceId": undefined,
              "intervalMs": 5000,
              "maxDataPoints": 10,
+              "queryCachingTTL": undefined,
              "refId": "B",
            },
          ],
--- a/packages/grafana-runtime/src/utils/DataSourceWithBackend.ts
+++ b/packages/grafana-runtime/src/utils/DataSourceWithBackend.ts
@ -120,7 +120,7 @@ class DataSourceWithBackend<
   * Ideally final -- any other implementation may not work as expected
   */
  query(request: DataQueryRequest<TQuery>): Observable<DataQueryResponse> {
-    const { intervalMs, maxDataPoints, range, requestId, hideFromInspector = false } = request;
+    const { intervalMs, maxDataPoints, queryCachingTTL, range, requestId, hideFromInspector = false } = request;
    let targets = request.targets;

    if (this.filterQuery) {
@ -172,6 +172,7 @@ class DataSourceWithBackend<
        datasourceId, // deprecated!
        intervalMs,
        maxDataPoints,
+        queryCachingTTL,
      };
    });

--- a/pkg/api/http_server.go
+++ b/pkg/api/http_server.go
@ -259,8 +259,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	accesscontrolService accesscontrol.Service, dashboardThumbsService thumbs.DashboardThumbService, navTreeService navtree.Service,
 	annotationRepo annotations.Repository, tagService tag.Service, searchv2HTTPService searchV2.SearchHTTPService,
 	queryLibraryHTTPService querylibrary.HTTPService, queryLibraryService querylibrary.Service, oauthTokenService oauthtoken.OAuthTokenService,
-	statsService stats.Service, authnService authn.Service,
-	pluginsCDNService *pluginscdn.Service,
+	statsService stats.Service, authnService authn.Service, pluginsCDNService *pluginscdn.Service,
 	k8saccess k8saccess.K8SAccess, // required so that the router is registered
 	starApi *starApi.API,
 ) (*HTTPServer, error) {
--- a/pkg/plugins/models.go
+++ b/pkg/plugins/models.go
@ -226,6 +226,9 @@ type DataSourceDTO struct {
 	BasicAuth       string `json:"basicAuth,omitempty"`
 	WithCredentials bool   `json:"withCredentials,omitempty"`

+	// This is populated by an Enterprise hook
+	CachingConfig QueryCachingConfig `json:"cachingConfig,omitempty"`
+
 	// InfluxDB
 	Username string `json:"username,omitempty"`
 	Password string `json:"password,omitempty"`
@ -288,3 +291,8 @@ type Permission struct {
 	Action string `json:"action"`
 	Scope  string `json:"scope"`
 }
+
+type QueryCachingConfig struct {
+	Enabled bool  `json:"enabled"`
+	TTLMS   int64 `json:"TTLMs"`
+}
--- a/pkg/services/publicdashboards/models/models.go
+++ b/pkg/services/publicdashboards/models/models.go
@ -123,9 +123,10 @@ type SavePublicDashboardDTO struct {
 }

 type PublicDashboardQueryDTO struct {
-	IntervalMs    int64
-	MaxDataPoints int64
-	TimeRange     TimeSettings
+	IntervalMs      int64
+	MaxDataPoints   int64
+	QueryCachingTTL int64
+	TimeRange       TimeSettings
 }

 type AnnotationsQueryDTO struct {
--- a/pkg/services/publicdashboards/service/query.go
+++ b/pkg/services/publicdashboards/service/query.go
@ -164,6 +164,7 @@ func (pd *PublicDashboardServiceImpl) buildMetricRequest(ctx context.Context, da
 	for i := range queries {
 		queries[i].Set("intervalMs", safeInterval)
 		queries[i].Set("maxDataPoints", safeResolution)
+		queries[i].Set("queryCachingTTL", reqDTO.QueryCachingTTL)
 	}

 	return dtos.MetricRequest{
--- a/pkg/services/publicdashboards/service/query_test.go
+++ b/pkg/services/publicdashboards/service/query_test.go
@ -888,9 +888,10 @@ func TestBuildMetricRequest(t *testing.T) {
 					"type": "mysql",
 					"uid":  "ds1",
 				},
-				"intervalMs":    int64(10000000),
-				"maxDataPoints": int64(200),
-				"refId":         "A",
+				"intervalMs":      int64(10000000),
+				"maxDataPoints":   int64(200),
+				"queryCachingTTL": int64(0),
+				"refId":           "A",
 			}),
 			reqDTO.Queries[0],
 		)
@ -902,9 +903,10 @@ func TestBuildMetricRequest(t *testing.T) {
 					"type": "prometheus",
 					"uid":  "ds2",
 				},
-				"intervalMs":    int64(10000000),
-				"maxDataPoints": int64(200),
-				"refId":         "B",
+				"intervalMs":      int64(10000000),
+				"maxDataPoints":   int64(200),
+				"queryCachingTTL": int64(0),
+				"refId":           "B",
 			}),
 			reqDTO.Queries[1],
 		)
--- a/public/app/angular/panel/metrics_panel_ctrl.ts
+++ b/public/app/angular/panel/metrics_panel_ctrl.ts
@ -203,6 +203,7 @@ class MetricsPanelCtrl extends PanelCtrl {
      publicDashboardAccessToken: this.dashboard.meta.publicDashboardAccessToken,
      scopedVars: panel.scopedVars,
      cacheTimeout: panel.cacheTimeout,
+      queryCachingTTL: panel.queryCachingTTL,
      transformations: panel.transformations,
    });
  }
--- a/public/app/features/dashboard/components/PanelEditor/PanelEditorQueries.tsx
+++ b/public/app/features/dashboard/components/PanelEditor/PanelEditorQueries.tsx
@ -33,6 +33,7 @@ export class PanelEditorQueries extends PureComponent<Props> {
        type: datasourceSettings?.type,
        uid: datasourceSettings?.uid,
      },
+      queryCachingTTL: datasourceSettings?.cachingConfig?.enabled ? panel.queryCachingTTL : undefined,
      queries: panel.targets,
      maxDataPoints: panel.maxDataPoints,
      minInterval: panel.interval,
--- a/public/app/features/dashboard/services/PublicDashboardDataSource.ts
+++ b/public/app/features/dashboard/services/PublicDashboardDataSource.ts
@ -86,6 +86,7 @@ export class PublicDashboardDataSource extends DataSourceApi<DataQuery, DataSour
      requestId,
      publicDashboardAccessToken,
      panelId,
+      queryCachingTTL,
      range: { from: fromRange, to: toRange },
    } = request;
    let queries: DataQuery[];
@ -110,6 +111,7 @@ export class PublicDashboardDataSource extends DataSourceApi<DataQuery, DataSour
      const body = {
        intervalMs,
        maxDataPoints,
+        queryCachingTTL,
        timeRange: { from: fromRange.valueOf().toString(), to: toRange.valueOf().toString() },
      };

--- a/public/app/features/dashboard/state/PanelModel.ts
+++ b/public/app/features/dashboard/state/PanelModel.ts
@ -104,6 +104,7 @@ const mustKeepProps: { [str: string]: boolean } = {
  hasRefreshed: true,
  events: true,
  cacheTimeout: true,
+  queryCachingTTL: true,
  cachedPluginOptions: true,
  transparent: true,
  pluginVersion: true,
@ -184,6 +185,8 @@ export class PanelModel implements DataConfigSource, IPanelModel {
  hasSavedPanelEditChange?: boolean;
  hasRefreshed?: boolean;
  cacheTimeout?: string | null;
+  queryCachingTTL?: number | null;
+
  cachedPluginOptions: Record<string, PanelOptionsCache> = {};
  legend?: { show: boolean; sort?: string; sortDesc?: boolean };
  plugin?: PanelPlugin;
@ -366,6 +369,7 @@ export class PanelModel implements DataConfigSource, IPanelModel {
      minInterval: this.interval,
      scopedVars: this.scopedVars,
      cacheTimeout: this.cacheTimeout,
+      queryCachingTTL: this.queryCachingTTL,
      transformations: this.transformations,
      app: this.isEditing ? CoreApp.PanelEditor : this.isViewing ? CoreApp.PanelViewer : CoreApp.Dashboard,
    });
@ -532,6 +536,7 @@ export class PanelModel implements DataConfigSource, IPanelModel {
    }

    this.cacheTimeout = options.cacheTimeout;
+    this.queryCachingTTL = options.queryCachingTTL;
    this.timeFrom = options.timeRange?.from;
    this.timeShift = options.timeRange?.shift;
    this.hideTimeOverride = options.timeRange?.hide;
--- a/public/app/features/query/components/QueryGroupOptions.tsx
+++ b/public/app/features/query/components/QueryGroupOptions.tsx
@ -111,6 +111,21 @@ export class QueryGroupOptionsEditor extends PureComponent<Props, State> {
    });
  };

+  onQueryCachingTTLBlur = (event: ChangeEvent<HTMLInputElement>) => {
+    const { options, onChange } = this.props;
+
+    let ttl: number | null = parseInt(event.target.value, 10);
+
+    if (isNaN(ttl) || ttl === 0) {
+      ttl = null;
+    }
+
+    onChange({
+      ...options,
+      queryCachingTTL: ttl,
+    });
+  };
+
  onMaxDataPointsBlur = (event: ChangeEvent<HTMLInputElement>) => {
    const { options, onChange } = this.props;

@ -168,6 +183,34 @@ export class QueryGroupOptionsEditor extends PureComponent<Props, State> {
    );
  }

+  renderQueryCachingTTLOption() {
+    const { dataSource, options } = this.props;
+
+    const tooltip = `Cache time-to-live: How long results from this queries in this panel will be cached, in milliseconds. Defaults to the TTL in the caching configuration for this datasource.`;
+
+    if (!dataSource.cachingConfig?.enabled) {
+      return null;
+    }
+
+    return (
+      <div className="gf-form-inline">
+        <div className="gf-form">
+          <InlineFormLabel width={9} tooltip={tooltip}>
+            Cache TTL
+          </InlineFormLabel>
+          <Input
+            type="number"
+            className="width-6"
+            placeholder={`${dataSource.cachingConfig.TTLMs}`}
+            spellCheck={false}
+            onBlur={this.onQueryCachingTTLBlur}
+            defaultValue={options.queryCachingTTL ?? undefined}
+          />
+        </div>
+      </div>
+    );
+  }
+
  renderMaxDataPointsOption() {
    const { data, options } = this.props;
    const realMd = data.request?.maxDataPoints;
@ -312,6 +355,7 @@ export class QueryGroupOptionsEditor extends PureComponent<Props, State> {
        {this.renderMaxDataPointsOption()}
        {this.renderIntervalOption()}
        {this.renderCacheTimeoutOption()}
+        {this.renderQueryCachingTTLOption()}

        <div className="gf-form">
          <InlineFormLabel width={9}>Relative time</InlineFormLabel>
--- a/public/app/features/query/state/PanelQueryRunner.ts
+++ b/public/app/features/query/state/PanelQueryRunner.ts
@ -58,6 +58,7 @@ export interface QueryRunnerOptions<
  minInterval: string | undefined | null;
  scopedVars?: ScopedVars;
  cacheTimeout?: string | null;
+  queryCachingTTL?: number | null;
  transformations?: DataTransformerConfig[];
  app?: CoreApp;
 }
@ -209,6 +210,7 @@ export class PanelQueryRunner {
      timeRange,
      timeInfo,
      cacheTimeout,
+      queryCachingTTL,
      maxDataPoints,
      scopedVars,
      minInterval,
@ -236,6 +238,7 @@ export class PanelQueryRunner {
      maxDataPoints: maxDataPoints,
      scopedVars: scopedVars || {},
      cacheTimeout,
+      queryCachingTTL,
      startTime: Date.now(),
      rangeRaw: timeRange.raw,
    };
--- a/public/app/features/query/state/QueryRunner.ts
+++ b/public/app/features/query/state/QueryRunner.ts
@ -45,6 +45,7 @@ export class QueryRunner implements QueryRunnerSrv {
      timeRange,
      timeInfo,
      cacheTimeout,
+      queryCachingTTL,
      maxDataPoints,
      scopedVars,
      minInterval,
@ -68,6 +69,7 @@ export class QueryRunner implements QueryRunnerSrv {
      maxDataPoints: maxDataPoints,
      scopedVars: scopedVars || {},
      cacheTimeout,
+      queryCachingTTL,
      startTime: Date.now(),
    };

--- a/public/app/types/query.ts
+++ b/public/app/types/query.ts
@ -7,6 +7,7 @@ export interface QueryGroupOptions {
  maxDataPoints?: number | null;
  minInterval?: string | null;
  cacheTimeout?: string | null;
+  queryCachingTTL?: number | null;
  timeRange?: {
    from?: string | null;
    shift?: string | null;