Query Caching: Add per-panel query caching TTL (#61968)

* *Create Caching Config interface and OSS impl
*Create front-end facing DS Cache config
*Populate Caching Config on Datasource DTO
*Update OSS wire deps

* fix unit test

* handle query caching TTL override on the frontend

* Make sure the override works with pubdash

* move caching config to the right place in the ds info

* move caching config logic to enterprise index hook

* move queryCachingTTL to pubdash query payload

* Remove  from metadata (not needed)

* rename struct and add comment

* remove invalid wire dependency

* manual revert of 395c74b

* fix frontend test

* fix backend test

* fix tests for real this time

* truly fix frontend test

* fix back end unit test for real

packages/grafana-data/src/types/datasource.ts

@@ -145,6 +145,10 @@ interface PluginMetaQueryOptions {
   maxDataPoints?: boolean;
   minInterval?: boolean;
 }
+interface PluginQueryCachingConfig {
+  enabled?: boolean;
+  TTLMs?: number;
+}
 
 export interface DataSourcePluginComponents<
   DSType extends DataSourceApi<TQuery, TOptions>,
@@ -224,6 +228,7 @@ abstract class DataSourceApi<
     this.id = instanceSettings.id;
     this.type = instanceSettings.type;
     this.meta = instanceSettings.meta;
+    this.cachingConfig = instanceSettings.cachingConfig;
     this.uid = instanceSettings.uid;
   }
 
@@ -301,6 +306,12 @@ abstract class DataSourceApi<
    */
   meta: DataSourcePluginMeta;
 
+  /**
+   * Information about the datasource's query caching configuration
+   * When the caching feature is disabled, this config will always be falsy
+   */
+  cachingConfig?: PluginQueryCachingConfig;
+
   /**
    * Used by alerting to check if query contains template variables
    */
@@ -487,6 +498,7 @@ export interface DataQueryRequest<TQuery extends DataQuery = DataQuery> {
   app: CoreApp | string;
 
   cacheTimeout?: string | null;
+  queryCachingTTL?: number | null;
   rangeRaw?: RawTimeRange;
   timeInfo?: string; // The query time description (blue text in the upper right)
   panelId?: number;
@@ -586,6 +598,7 @@ export interface DataSourceInstanceSettings<T extends DataSourceJsonData = DataS
   type: string;
   name: string;
   meta: DataSourcePluginMeta;
+  cachingConfig?: PluginQueryCachingConfig;
   readOnly: boolean;
   url?: string;
   jsonData: T;

packages/grafana-data/src/types/queryRunner.ts

@@ -23,6 +23,7 @@ export interface QueryRunnerOptions {
   minInterval: string | undefined | null;
   scopedVars?: ScopedVars;
   cacheTimeout?: string;
+  queryCachingTTL?: number;
   app?: string;
 }
 

packages/grafana-runtime/src/utils/DataSourceWithBackend.test.ts

@@ -66,6 +66,7 @@ describe('DataSourceWithBackend', () => {
               "datasourceId": 1234,
               "intervalMs": 5000,
               "maxDataPoints": 10,
+              "queryCachingTTL": undefined,
               "refId": "A",
             },
             {
@@ -76,6 +77,7 @@ describe('DataSourceWithBackend', () => {
               "datasourceId": undefined,
               "intervalMs": 5000,
               "maxDataPoints": 10,
+              "queryCachingTTL": undefined,
               "refId": "B",
             },
           ],
@@ -133,6 +135,7 @@ describe('DataSourceWithBackend', () => {
               "datasourceId": 1234,
               "intervalMs": 5000,
               "maxDataPoints": 10,
+              "queryCachingTTL": undefined,
               "refId": "A",
             },
             {
@@ -143,6 +146,7 @@ describe('DataSourceWithBackend', () => {
               "datasourceId": undefined,
               "intervalMs": 5000,
               "maxDataPoints": 10,
+              "queryCachingTTL": undefined,
               "refId": "B",
             },
           ],

packages/grafana-runtime/src/utils/DataSourceWithBackend.ts

@@ -120,7 +120,7 @@ class DataSourceWithBackend<
    * Ideally final -- any other implementation may not work as expected
    */
   query(request: DataQueryRequest<TQuery>): Observable<DataQueryResponse> {
-    const { intervalMs, maxDataPoints, range, requestId, hideFromInspector = false } = request;
+    const { intervalMs, maxDataPoints, queryCachingTTL, range, requestId, hideFromInspector = false } = request;
     let targets = request.targets;
 
     if (this.filterQuery) {
@@ -172,6 +172,7 @@ class DataSourceWithBackend<
         datasourceId, // deprecated!
         intervalMs,
         maxDataPoints,
+        queryCachingTTL,
       };
     });
 

pkg/api/http_server.go

@@ -259,8 +259,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	accesscontrolService accesscontrol.Service, dashboardThumbsService thumbs.DashboardThumbService, navTreeService navtree.Service,
 	annotationRepo annotations.Repository, tagService tag.Service, searchv2HTTPService searchV2.SearchHTTPService,
 	queryLibraryHTTPService querylibrary.HTTPService, queryLibraryService querylibrary.Service, oauthTokenService oauthtoken.OAuthTokenService,
-	statsService stats.Service, authnService authn.Service,
+	statsService stats.Service, authnService authn.Service, pluginsCDNService *pluginscdn.Service,
-	pluginsCDNService *pluginscdn.Service,
 	k8saccess k8saccess.K8SAccess, // required so that the router is registered
 	starApi *starApi.API,
 ) (*HTTPServer, error) {

pkg/plugins/models.go

@@ -226,6 +226,9 @@ type DataSourceDTO struct {
 	BasicAuth       string `json:"basicAuth,omitempty"`
 	WithCredentials bool   `json:"withCredentials,omitempty"`
 
+	// This is populated by an Enterprise hook
+	CachingConfig QueryCachingConfig `json:"cachingConfig,omitempty"`
+
 	// InfluxDB
 	Username string `json:"username,omitempty"`
 	Password string `json:"password,omitempty"`
@@ -288,3 +291,8 @@ type Permission struct {
 	Action string `json:"action"`
 	Scope  string `json:"scope"`
 }
+
+type QueryCachingConfig struct {
+	Enabled bool  `json:"enabled"`
+	TTLMS   int64 `json:"TTLMs"`
+}

pkg/services/publicdashboards/models/models.go

@@ -123,9 +123,10 @@ type SavePublicDashboardDTO struct {
 }
 
 type PublicDashboardQueryDTO struct {
-	IntervalMs    int64
+	IntervalMs      int64
-	MaxDataPoints int64
+	MaxDataPoints   int64
-	TimeRange     TimeSettings
+	QueryCachingTTL int64
+	TimeRange       TimeSettings
 }
 
 type AnnotationsQueryDTO struct {

pkg/services/publicdashboards/service/query.go

@@ -164,6 +164,7 @@ func (pd *PublicDashboardServiceImpl) buildMetricRequest(ctx context.Context, da
 	for i := range queries {
 		queries[i].Set("intervalMs", safeInterval)
 		queries[i].Set("maxDataPoints", safeResolution)
+		queries[i].Set("queryCachingTTL", reqDTO.QueryCachingTTL)
 	}
 
 	return dtos.MetricRequest{

pkg/services/publicdashboards/service/query_test.go

@@ -888,9 +888,10 @@ func TestBuildMetricRequest(t *testing.T) {
 					"type": "mysql",
 					"uid":  "ds1",
 				},
-				"intervalMs":    int64(10000000),
+				"intervalMs":      int64(10000000),
-				"maxDataPoints": int64(200),
+				"maxDataPoints":   int64(200),
-				"refId":         "A",
+				"queryCachingTTL": int64(0),
+				"refId":           "A",
 			}),
 			reqDTO.Queries[0],
 		)
@@ -902,9 +903,10 @@ func TestBuildMetricRequest(t *testing.T) {
 					"type": "prometheus",
 					"uid":  "ds2",
 				},
-				"intervalMs":    int64(10000000),
+				"intervalMs":      int64(10000000),
-				"maxDataPoints": int64(200),
+				"maxDataPoints":   int64(200),
-				"refId":         "B",
+				"queryCachingTTL": int64(0),
+				"refId":           "B",
 			}),
 			reqDTO.Queries[1],
 		)

public/app/angular/panel/metrics_panel_ctrl.ts

@@ -203,6 +203,7 @@ class MetricsPanelCtrl extends PanelCtrl {
       publicDashboardAccessToken: this.dashboard.meta.publicDashboardAccessToken,
       scopedVars: panel.scopedVars,
       cacheTimeout: panel.cacheTimeout,
+      queryCachingTTL: panel.queryCachingTTL,
       transformations: panel.transformations,
     });
   }

public/app/features/dashboard/components/PanelEditor/PanelEditorQueries.tsx

@@ -33,6 +33,7 @@ export class PanelEditorQueries extends PureComponent<Props> {
         type: datasourceSettings?.type,
         uid: datasourceSettings?.uid,
       },
+      queryCachingTTL: datasourceSettings?.cachingConfig?.enabled ? panel.queryCachingTTL : undefined,
       queries: panel.targets,
       maxDataPoints: panel.maxDataPoints,
       minInterval: panel.interval,

public/app/features/dashboard/services/PublicDashboardDataSource.ts

@@ -86,6 +86,7 @@ export class PublicDashboardDataSource extends DataSourceApi<DataQuery, DataSour
       requestId,
       publicDashboardAccessToken,
       panelId,
+      queryCachingTTL,
       range: { from: fromRange, to: toRange },
     } = request;
     let queries: DataQuery[];
@@ -110,6 +111,7 @@ export class PublicDashboardDataSource extends DataSourceApi<DataQuery, DataSour
       const body = {
         intervalMs,
         maxDataPoints,
+        queryCachingTTL,
         timeRange: { from: fromRange.valueOf().toString(), to: toRange.valueOf().toString() },
       };
 

public/app/features/dashboard/state/PanelModel.ts

@@ -104,6 +104,7 @@ const mustKeepProps: { [str: string]: boolean } = {
   hasRefreshed: true,
   events: true,
   cacheTimeout: true,
+  queryCachingTTL: true,
   cachedPluginOptions: true,
   transparent: true,
   pluginVersion: true,
@@ -184,6 +185,8 @@ export class PanelModel implements DataConfigSource, IPanelModel {
   hasSavedPanelEditChange?: boolean;
   hasRefreshed?: boolean;
   cacheTimeout?: string | null;
+  queryCachingTTL?: number | null;
+
   cachedPluginOptions: Record<string, PanelOptionsCache> = {};
   legend?: { show: boolean; sort?: string; sortDesc?: boolean };
   plugin?: PanelPlugin;
@@ -366,6 +369,7 @@ export class PanelModel implements DataConfigSource, IPanelModel {
       minInterval: this.interval,
       scopedVars: this.scopedVars,
       cacheTimeout: this.cacheTimeout,
+      queryCachingTTL: this.queryCachingTTL,
       transformations: this.transformations,
       app: this.isEditing ? CoreApp.PanelEditor : this.isViewing ? CoreApp.PanelViewer : CoreApp.Dashboard,
     });
@@ -532,6 +536,7 @@ export class PanelModel implements DataConfigSource, IPanelModel {
     }
 
     this.cacheTimeout = options.cacheTimeout;
+    this.queryCachingTTL = options.queryCachingTTL;
     this.timeFrom = options.timeRange?.from;
     this.timeShift = options.timeRange?.shift;
     this.hideTimeOverride = options.timeRange?.hide;

public/app/features/query/components/QueryGroupOptions.tsx

@@ -111,6 +111,21 @@ export class QueryGroupOptionsEditor extends PureComponent<Props, State> {
     });
   };
 
+  onQueryCachingTTLBlur = (event: ChangeEvent<HTMLInputElement>) => {
+    const { options, onChange } = this.props;
+
+    let ttl: number | null = parseInt(event.target.value, 10);
+
+    if (isNaN(ttl) || ttl === 0) {
+      ttl = null;
+    }
+
+    onChange({
+      ...options,
+      queryCachingTTL: ttl,
+    });
+  };
+
   onMaxDataPointsBlur = (event: ChangeEvent<HTMLInputElement>) => {
     const { options, onChange } = this.props;
 
@@ -168,6 +183,34 @@ export class QueryGroupOptionsEditor extends PureComponent<Props, State> {
     );
   }
 
+  renderQueryCachingTTLOption() {
+    const { dataSource, options } = this.props;
+
+    const tooltip = `Cache time-to-live: How long results from this queries in this panel will be cached, in milliseconds. Defaults to the TTL in the caching configuration for this datasource.`;
+
+    if (!dataSource.cachingConfig?.enabled) {
+      return null;
+    }
+
+    return (
+      <div className="gf-form-inline">
+        <div className="gf-form">
+          <InlineFormLabel width={9} tooltip={tooltip}>
+            Cache TTL
+          </InlineFormLabel>
+          <Input
+            type="number"
+            className="width-6"
+            placeholder={`${dataSource.cachingConfig.TTLMs}`}
+            spellCheck={false}
+            onBlur={this.onQueryCachingTTLBlur}
+            defaultValue={options.queryCachingTTL ?? undefined}
+          />
+        </div>
+      </div>
+    );
+  }
+
   renderMaxDataPointsOption() {
     const { data, options } = this.props;
     const realMd = data.request?.maxDataPoints;
@@ -312,6 +355,7 @@ export class QueryGroupOptionsEditor extends PureComponent<Props, State> {
         {this.renderMaxDataPointsOption()}
         {this.renderIntervalOption()}
         {this.renderCacheTimeoutOption()}
+        {this.renderQueryCachingTTLOption()}
 
         <div className="gf-form">
           <InlineFormLabel width={9}>Relative time</InlineFormLabel>

public/app/features/query/state/PanelQueryRunner.ts

@@ -58,6 +58,7 @@ export interface QueryRunnerOptions<
   minInterval: string | undefined | null;
   scopedVars?: ScopedVars;
   cacheTimeout?: string | null;
+  queryCachingTTL?: number | null;
   transformations?: DataTransformerConfig[];
   app?: CoreApp;
 }
@@ -209,6 +210,7 @@ export class PanelQueryRunner {
       timeRange,
       timeInfo,
       cacheTimeout,
+      queryCachingTTL,
       maxDataPoints,
       scopedVars,
       minInterval,
@@ -236,6 +238,7 @@ export class PanelQueryRunner {
       maxDataPoints: maxDataPoints,
       scopedVars: scopedVars || {},
       cacheTimeout,
+      queryCachingTTL,
       startTime: Date.now(),
       rangeRaw: timeRange.raw,
     };

public/app/features/query/state/QueryRunner.ts

@@ -45,6 +45,7 @@ export class QueryRunner implements QueryRunnerSrv {
       timeRange,
       timeInfo,
       cacheTimeout,
+      queryCachingTTL,
       maxDataPoints,
       scopedVars,
       minInterval,
@@ -68,6 +69,7 @@ export class QueryRunner implements QueryRunnerSrv {
       maxDataPoints: maxDataPoints,
       scopedVars: scopedVars || {},
       cacheTimeout,
+      queryCachingTTL,
       startTime: Date.now(),
     };
 

public/app/types/query.ts

@@ -7,6 +7,7 @@ export interface QueryGroupOptions {
   maxDataPoints?: number | null;
   minInterval?: string | null;
   cacheTimeout?: string | null;
+  queryCachingTTL?: number | null;
   timeRange?: {
     from?: string | null;
     shift?: string | null;