Loki: Add predefined operations functionality behind feature flag (#69379)

* Add feature toggle for lokiPredefinedOperations

* Add config editor for predefined operations

* Add predefined patterns to editor

* Update content in tooltip

* Update public/app/plugins/datasource/loki/configuration/QuerySettings.tsx

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

* Update public/app/plugins/datasource/loki/components/LokiQueryEditor.tsx

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

* Add experimental badge

---------

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -110,6 +110,7 @@ Alpha features might be changed or removed without prior notice.
 | `authenticationConfigUI`           | Enables authentication configuration UI                                                                      |
 | `pluginsAPIManifestKey`            | Use grafana.com API to retrieve the public manifest key                                                      |
 | `extraThemes`                      | Enables extra themes                                                                                         |
+| `lokiPredefinedOperations`         | Adds predefined query operations to Loki query editor                                                        |
 
 ## Development feature toggles
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -97,4 +97,5 @@ export interface FeatureToggles {
   enableDatagridEditing?: boolean;
   dataSourcePageHeader?: boolean;
   extraThemes?: boolean;
+  lokiPredefinedOperations?: boolean;
 }

pkg/services/featuremgmt/registry.go

@@ -535,5 +535,12 @@ var (
 			State:        FeatureStateAlpha,
 			Owner:        grafanaUserEssentialsSquad,
 		},
+		{
+			Name:         "lokiPredefinedOperations",
+			Description:  "Adds predefined query operations to Loki query editor",
+			FrontendOnly: true,
+			State:        FeatureStateAlpha,
+			Owner:        grafanaObservabilityLogsSquad,
+		},
 	}
 )

pkg/services/featuremgmt/toggles_gen.csv

@@ -78,3 +78,4 @@ faroDatasourceSelector,beta,@grafana/app-o11y,false,false,false,true
 enableDatagridEditing,beta,@grafana/grafana-bi-squad,false,false,false,true
 dataSourcePageHeader,beta,@grafana/enterprise-datasources,false,false,false,true
 extraThemes,alpha,@grafana/user-essentials,false,false,false,true
+lokiPredefinedOperations,alpha,@grafana/observability-logs,false,false,false,true

pkg/services/featuremgmt/toggles_gen.go

@@ -322,4 +322,8 @@ const (
 	// FlagExtraThemes
 	// Enables extra themes
 	FlagExtraThemes = "extraThemes"
+
+	// FlagLokiPredefinedOperations
+	// Adds predefined query operations to Loki query editor
+	FlagLokiPredefinedOperations = "lokiPredefinedOperations"
 )

public/app/plugins/datasource/loki/components/LokiQueryEditor.tsx

@@ -5,7 +5,7 @@ import { usePrevious } from 'react-use';
 import { CoreApp, LoadingState } from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
 import { EditorHeader, EditorRows, FlexItem, Space, Stack } from '@grafana/experimental';
-import { reportInteraction } from '@grafana/runtime';
+import { config, reportInteraction } from '@grafana/runtime';
 import { Button, ConfirmModal } from '@grafana/ui';
 import { QueryEditorModeToggle } from 'app/plugins/datasource/prometheus/querybuilder/shared/QueryEditorModeToggle';
 import { QueryHeaderSwitch } from 'app/plugins/datasource/prometheus/querybuilder/shared/QueryHeaderSwitch';
@@ -38,9 +38,13 @@ export const LokiQueryEditor = React.memo<LokiQueryEditorProps>((props) => {
   const { flag: explain, setFlag: setExplain } = useFlag(lokiQueryEditorExplainKey);
 
   const timerange = datasource.getTimeRange();
+  const predefinedOperations = datasource.predefinedOperations;
   const previousTimerange = usePrevious(timerange);
 
   const query = getQueryWithDefaults(props.query);
+  if (config.featureToggles.lokiPredefinedOperations && !query.expr && predefinedOperations) {
+    query.expr = `{} ${predefinedOperations}`;
+  }
   const previousQuery = usePrevious(query.expr);
 
   // This should be filled in from the defaults by now.

public/app/plugins/datasource/loki/configuration/ConfigEditor.tsx

@@ -7,7 +7,7 @@ import { AlertingSettings, DataSourceHttpSettings } from '@grafana/ui';
 import { LokiOptions } from '../types';
 
 import { DerivedFields } from './DerivedFields';
-import { MaxLinesField } from './MaxLinesField';
+import { QuerySettings } from './QuerySettings';
 
 export type Props = DataSourcePluginOptionsEditorProps<LokiOptions>;
 
@@ -24,6 +24,7 @@ const makeJsonUpdater =
   };
 
 const setMaxLines = makeJsonUpdater('maxLines');
+const setPredefinedOperations = makeJsonUpdater('predefinedOperations');
 const setDerivedFields = makeJsonUpdater('derivedFields');
 
 export const ConfigEditor = (props: Props) => {
@@ -41,9 +42,11 @@ export const ConfigEditor = (props: Props) => {
 
       <AlertingSettings<LokiOptions> options={options} onOptionsChange={onOptionsChange} />
 
-      <MaxLinesField
-        value={options.jsonData.maxLines || ''}
-        onChange={(value) => onOptionsChange(setMaxLines(options, value))}
+      <QuerySettings
+        maxLines={options.jsonData.maxLines || ''}
+        onMaxLinedChange={(value) => onOptionsChange(setMaxLines(options, value))}
+        predefinedOperations={options.jsonData.predefinedOperations || ''}
+        onPredefinedOperationsChange={(value) => onOptionsChange(setPredefinedOperations(options, value))}
       />
 
       <DerivedFields

public/app/plugins/datasource/loki/configuration/MaxLinesField.tsx

@@ -1,46 +0,0 @@
-import React from 'react';
-
-import { LegacyForms } from '@grafana/ui';
-const { FormField } = LegacyForms;
-
-type Props = {
-  value: string;
-  onChange: (value: string) => void;
-};
-
-export const MaxLinesField = (props: Props) => {
-  const { value, onChange } = props;
-  return (
-    <>
-      <h3 className="page-heading">Queries</h3>
-      <div className="gf-form-group">
-        <div className="gf-form-inline">
-          <div className="gf-form">
-            <FormField
-              label="Maximum lines"
-              labelWidth={11}
-              inputWidth={20}
-              inputEl={
-                <input
-                  type="number"
-                  className="gf-form-input width-8 gf-form-input--has-help-icon"
-                  value={value}
-                  onChange={(event) => onChange(event.currentTarget.value)}
-                  spellCheck={false}
-                  placeholder="1000"
-                />
-              }
-              tooltip={
-                <>
-                  Loki queries must contain a limit of the maximum number of lines returned (default: 1000). Increase
-                  this limit to have a bigger result set for ad-hoc analysis. Decrease this limit if your browser
-                  becomes sluggish when displaying the log results.
-                </>
-              }
-            />
-          </div>
-        </div>
-      </div>
-    </>
-  );
-};

public/app/plugins/datasource/loki/configuration/QuerySettings.tsx

@@ -0,0 +1,83 @@
+import React from 'react';
+
+import { config } from '@grafana/runtime';
+import { Badge, LegacyForms } from '@grafana/ui';
+
+const { FormField } = LegacyForms;
+
+type Props = {
+  maxLines: string;
+  onMaxLinedChange: (value: string) => void;
+  predefinedOperations: string;
+  onPredefinedOperationsChange: (value: string) => void;
+};
+
+export const QuerySettings = (props: Props) => {
+  const { maxLines, onMaxLinedChange, predefinedOperations, onPredefinedOperationsChange } = props;
+  return (
+    <>
+      <h3 className="page-heading">Queries</h3>
+      <div className="gf-form-group">
+        <div className="gf-form-inline">
+          <div className="gf-form">
+            <FormField
+              label="Maximum lines"
+              labelWidth={11}
+              inputWidth={20}
+              inputEl={
+                <input
+                  type="number"
+                  className="gf-form-input width-8 gf-form-input--has-help-icon"
+                  value={maxLines}
+                  onChange={(event) => onMaxLinedChange(event.currentTarget.value)}
+                  spellCheck={false}
+                  placeholder="1000"
+                />
+              }
+              tooltip={
+                <>
+                  Loki queries must contain a limit of the maximum number of lines returned (default: 1000). Increase
+                  this limit to have a bigger result set for ad-hoc analysis. Decrease this limit if your browser
+                  becomes sluggish when displaying the log results.
+                </>
+              }
+            />
+          </div>
+        </div>
+        {config.featureToggles.lokiPredefinedOperations && (
+          <div className="gf-form-inline">
+            <div className="gf-form">
+              <FormField
+                label="Predefined operations"
+                labelWidth={11}
+                inputEl={
+                  <input
+                    type="string"
+                    className="gf-form-input width-20 gf-form-input--has-help-icon"
+                    value={predefinedOperations}
+                    onChange={(event) => onPredefinedOperationsChange(event.currentTarget.value)}
+                    spellCheck={false}
+                    placeholder="| unpack | line_format"
+                  />
+                }
+                tooltip={
+                  <div>
+                    {
+                      'Predefined operations are used as an initial state for your queries. They are useful, if you want to unpack, parse or format all log lines. Currently we support only log operations starting with |. For example: | unpack | line_format "{{.message}}".'
+                    }
+                  </div>
+                }
+              />
+              <Badge
+                text="Experimental"
+                color="orange"
+                icon="exclamation-triangle"
+                tooltip="Predefined operations is an experimental feature that may change in the future."
+              />
+            </div>
+          </div>
+        )}
+      </div>
+    </>
+  );
+};

public/app/plugins/datasource/loki/datasource.ts

@@ -134,6 +134,7 @@ export class LokiDatasource
   languageProvider: LanguageProvider;
   logContextProvider: LogContextProvider;
   maxLines: number;
+  predefinedOperations: string;
 
   constructor(
     private instanceSettings: DataSourceInstanceSettings<LokiOptions>,
@@ -145,6 +146,7 @@ export class LokiDatasource
     this.languageProvider = new LanguageProvider(this);
     const settingsData = instanceSettings.jsonData || {};
     this.maxLines = parseInt(settingsData.maxLines ?? '0', 10) || DEFAULT_MAX_LINES;
+    this.predefinedOperations = settingsData.predefinedOperations ?? '';
     this.annotations = {
       QueryEditor: LokiAnnotationsQueryEditor,
     };

public/app/plugins/datasource/loki/types.ts

@@ -48,6 +48,7 @@ export interface LokiOptions extends DataSourceJsonData {
   derivedFields?: DerivedFieldConfig[];
   alertmanager?: string;
   keepCookies?: string[];
+  predefinedOperations?: string;
 }
 
 export interface LokiStats {