Explore: Add transformations to correlation data links (#61799)

* bring in source from database

* bring in transformations from database

* add regex transformations to scopevar

* Consolidate types, add better example, cleanup

* Add var only if match

* Change ScopedVar to not require text, do not leak transformation-made variables between links

* Add mappings and start implementing logfmt

* Add mappings and start implementing logfmt

* Remove mappings, turn off global regex

* Add example yaml and omit transformations if empty

* Fix the yaml

* Add logfmt transformation

* Cleanup transformations and yaml

* add transformation field to FE types and use it, safeStringify logfmt values

* Add tests, only safe stringify if non-string, fix bug with safe stringify where it would return empty string with false value

* Add test for transformation field

* Do not add null transformations object

* Break out transformation logic, add tests to backend code

* Fix lint errors I understand 😅

* Fix the backend lint error

* Remove unnecessary code and mark new Transformations object as internal

* Add support for named capture groups

* Remove type assertion

* Remove variable name from transformation

* Add test for overriding regexes

* Add back variable name field, but change to mapValue

* fix go api test

* Change transformation types to enum, add better provisioning checks for bad type name and format

* Check for expression with regex transformations

public/app/core/utils/explore.ts

@@ -192,7 +192,7 @@ export const safeParseJson = (text?: string): any | undefined => {
 };
 
 export const safeStringifyValue = (value: any, space?: number) => {
-  if (!value) {
+  if (value === undefined || value === null) {
     return '';
   }
 

public/app/features/correlations/transformations.ts

@@ -0,0 +1,34 @@
+import logfmt from 'logfmt';
+
+import { ScopedVars, DataLinkTransformationConfig, SupportedTransformationTypes } from '@grafana/data';
+import { safeStringifyValue } from 'app/core/utils/explore';
+
+export const getTransformationVars = (
+  transformation: DataLinkTransformationConfig,
+  fieldValue: string,
+  fieldName: string
+): ScopedVars => {
+  let transformationScopedVars: ScopedVars = {};
+  let transformVal: { [key: string]: string | boolean | null | undefined } = {};
+  if (transformation.type === SupportedTransformationTypes.Regex && transformation.expression) {
+    const regexp = new RegExp(transformation.expression, 'gi');
+    const matches = fieldValue.matchAll(regexp);
+    for (const match of matches) {
+      if (match.groups) {
+        transformVal = match.groups;
+      } else {
+        transformVal[transformation.mapValue || fieldName] = match[1] || match[0];
+      }
+    }
+  } else if (transformation.type === SupportedTransformationTypes.Logfmt) {
+    transformVal = logfmt.parse(fieldValue);
+  }
+
+  Object.keys(transformVal).forEach((key) => {
+    const transformValString =
+      typeof transformVal[key] === 'string' ? transformVal[key] : safeStringifyValue(transformVal[key]);
+    transformationScopedVars[key] = { value: transformValString };
+  });
+
+  return transformationScopedVars;
+};

public/app/features/correlations/types.ts

@@ -1,3 +1,5 @@
+import { DataLinkTransformationConfig } from '@grafana/data';
+
 export interface AddCorrelationResponse {
   correlation: Correlation;
 }
@@ -5,10 +7,12 @@ export interface AddCorrelationResponse {
 export type GetCorrelationsResponse = Correlation[];
 
 type CorrelationConfigType = 'query';
+
 export interface CorrelationConfig {
   field: string;
   target: object;
   type: CorrelationConfigType;
+  transformations?: DataLinkTransformationConfig[];
 }
 
 export interface Correlation {

public/app/features/correlations/utils.ts

@@ -39,6 +39,7 @@ const decorateDataFrameWithInternalDataLinks = (dataFrame: DataFrame, correlatio
             query: correlation.config?.target,
             datasourceUid: correlation.target.uid,
             datasourceName: correlation.target.name,
+            transformations: correlation.config?.transformations,
           },
           url: '',
           title: correlation.label || correlation.target.name,

public/app/features/explore/utils/links.test.ts

@@ -6,6 +6,7 @@ import {
   Field,
   FieldType,
   InterpolateFunction,
+  SupportedTransformationTypes,
   TimeRange,
   toDataFrame,
 } from '@grafana/data';
@@ -240,6 +241,233 @@ describe('getFieldLinksForExplore', () => {
     );
   });
 
+  it('returns internal links with logfmt and regex transformation', () => {
+    const transformationLink: DataLink = {
+      title: '',
+      url: '',
+      internal: {
+        query: { query: 'http_requests{app=${application} env=${environment}}' },
+        datasourceUid: 'uid_1',
+        datasourceName: 'test_ds',
+        transformations: [
+          { type: SupportedTransformationTypes.Logfmt },
+          { type: SupportedTransformationTypes.Regex, expression: 'host=(dev|prod)', mapValue: 'environment' },
+        ],
+      },
+    };
+
+    const { field, range, dataFrame } = setup(transformationLink, true, {
+      name: 'msg',
+      type: FieldType.string,
+      values: new ArrayVector(['application=foo host=dev-001', 'application=bar host=prod-003']),
+      config: {
+        links: [transformationLink],
+      },
+    });
+
+    const links = [
+      getFieldLinksForExplore({ field, rowIndex: 0, range, dataFrame }),
+      getFieldLinksForExplore({ field, rowIndex: 1, range, dataFrame }),
+    ];
+    expect(links[0]).toHaveLength(1);
+    expect(links[0][0].href).toBe(
+      `/explore?left=${encodeURIComponent(
+        '{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=foo env=dev}"}]}'
+      )}`
+    );
+    expect(links[1]).toHaveLength(1);
+    expect(links[1][0].href).toBe(
+      `/explore?left=${encodeURIComponent(
+        '{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=bar env=prod}"}]}'
+      )}`
+    );
+  });
+
+  it('returns internal links with 2 unnamed regex transformations and use the last transformation', () => {
+    const transformationLink: DataLink = {
+      title: '',
+      url: '',
+      internal: {
+        query: { query: 'http_requests{env=${msg}}' },
+        datasourceUid: 'uid_1',
+        datasourceName: 'test_ds',
+        transformations: [
+          { type: SupportedTransformationTypes.Regex, expression: 'fieldA=(asparagus|broccoli)' },
+          { type: SupportedTransformationTypes.Regex, expression: 'fieldB=(apple|banana)' },
+        ],
+      },
+    };
+
+    const { field, range, dataFrame } = setup(transformationLink, true, {
+      name: 'msg',
+      type: FieldType.string,
+      values: new ArrayVector(['fieldA=asparagus fieldB=banana', 'fieldA=broccoli fieldB=apple']),
+      config: {
+        links: [transformationLink],
+      },
+    });
+
+    const links = [
+      getFieldLinksForExplore({ field, rowIndex: 0, range, dataFrame }),
+      getFieldLinksForExplore({ field, rowIndex: 1, range, dataFrame }),
+    ];
+    expect(links[0]).toHaveLength(1);
+    expect(links[0][0].href).toBe(
+      `/explore?left=${encodeURIComponent(
+        '{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{env=banana}"}]}'
+      )}`
+    );
+    expect(links[1]).toHaveLength(1);
+    expect(links[1][0].href).toBe(
+      `/explore?left=${encodeURIComponent(
+        '{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{env=apple}"}]}'
+      )}`
+    );
+  });
+
+  it('returns internal links with logfmt with stringified booleans', () => {
+    const transformationLink: DataLink = {
+      title: '',
+      url: '',
+      internal: {
+        query: { query: 'http_requests{app=${application} isOnline=${online}}' },
+        datasourceUid: 'uid_1',
+        datasourceName: 'test_ds',
+        transformations: [{ type: SupportedTransformationTypes.Logfmt }],
+      },
+    };
+
+    const { field, range, dataFrame } = setup(transformationLink, true, {
+      name: 'msg',
+      type: FieldType.string,
+      values: new ArrayVector(['application=foo online=true', 'application=bar online=false']),
+      config: {
+        links: [transformationLink],
+      },
+    });
+
+    const links = [
+      getFieldLinksForExplore({ field, rowIndex: 0, range, dataFrame }),
+      getFieldLinksForExplore({ field, rowIndex: 1, range, dataFrame }),
+    ];
+    expect(links[0]).toHaveLength(1);
+    expect(links[0][0].href).toBe(
+      `/explore?left=${encodeURIComponent(
+        '{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=foo isOnline=true}"}]}'
+      )}`
+    );
+    expect(links[1]).toHaveLength(1);
+    expect(links[1][0].href).toBe(
+      `/explore?left=${encodeURIComponent(
+        '{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=bar isOnline=false}"}]}'
+      )}`
+    );
+  });
+
+  it('returns internal links with logfmt with correct data on transformation-defined field', () => {
+    const transformationLink: DataLink = {
+      title: '',
+      url: '',
+      internal: {
+        query: { query: 'http_requests{app=${application}}' },
+        datasourceUid: 'uid_1',
+        datasourceName: 'test_ds',
+        transformations: [{ type: SupportedTransformationTypes.Logfmt, field: 'fieldNamedInTransformation' }],
+      },
+    };
+
+    // fieldWithLink has the transformation, but the transformation has defined fieldNamedInTransformation as its field to transform
+    const { field, range, dataFrame } = setup(
+      transformationLink,
+      true,
+      {
+        name: 'fieldWithLink',
+        type: FieldType.string,
+        values: new ArrayVector(['application=link', 'application=link2']),
+        config: {
+          links: [transformationLink],
+        },
+      },
+      [
+        {
+          name: 'fieldNamedInTransformation',
+          type: FieldType.string,
+          values: new ArrayVector(['application=transform', 'application=transform2']),
+          config: {},
+        },
+      ]
+    );
+
+    const links = [
+      getFieldLinksForExplore({ field, rowIndex: 0, range, dataFrame }),
+      getFieldLinksForExplore({ field, rowIndex: 1, range, dataFrame }),
+    ];
+    expect(links[0]).toHaveLength(1);
+    expect(links[0][0].href).toBe(
+      `/explore?left=${encodeURIComponent(
+        '{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=transform}"}]}'
+      )}`
+    );
+    expect(links[1]).toHaveLength(1);
+    expect(links[1][0].href).toBe(
+      `/explore?left=${encodeURIComponent(
+        '{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=transform2}"}]}'
+      )}`
+    );
+  });
+
+  it('returns internal links with regex named capture groups', () => {
+    const transformationLink: DataLink = {
+      title: '',
+      url: '',
+      internal: {
+        query: { query: 'http_requests{app=${application} env=${environment}}' },
+        datasourceUid: 'uid_1',
+        datasourceName: 'test_ds',
+        transformations: [
+          {
+            type: SupportedTransformationTypes.Regex,
+            expression: '(?=.*(?<application>(grafana|loki)))(?=.*(?<environment>(dev|prod)))',
+          },
+        ],
+      },
+    };
+
+    const { field, range, dataFrame } = setup(transformationLink, true, {
+      name: 'msg',
+      type: FieldType.string,
+      values: new ArrayVector(['foo loki prod', 'dev bar grafana', 'prod grafana foo']),
+      config: {
+        links: [transformationLink],
+      },
+    });
+
+    const links = [
+      getFieldLinksForExplore({ field, rowIndex: 0, range, dataFrame }),
+      getFieldLinksForExplore({ field, rowIndex: 1, range, dataFrame }),
+      getFieldLinksForExplore({ field, rowIndex: 2, range, dataFrame }),
+    ];
+    expect(links[0]).toHaveLength(1);
+    expect(links[0][0].href).toBe(
+      `/explore?left=${encodeURIComponent(
+        '{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=loki env=prod}"}]}'
+      )}`
+    );
+    expect(links[1]).toHaveLength(1);
+    expect(links[1][0].href).toBe(
+      `/explore?left=${encodeURIComponent(
+        '{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=grafana env=dev}"}]}'
+      )}`
+    );
+
+    expect(links[2]).toHaveLength(1);
+    expect(links[2][0].href).toBe(
+      `/explore?left=${encodeURIComponent(
+        '{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=grafana env=prod}"}]}'
+      )}`
+    );
+  });
+
   it('returns no internal links when target contains empty template variables', () => {
     const { field, range, dataFrame } = setup({
       title: '',

public/app/features/explore/utils/links.ts

@@ -15,6 +15,7 @@ import {
 } from '@grafana/data';
 import { getTemplateSrv } from '@grafana/runtime';
 import { contextSrv } from 'app/core/services/context_srv';
+import { getTransformationVars } from 'app/features/correlations/transformations';
 
 import { getLinkSrv } from '../../panel/panellinks/link_srv';
 
@@ -71,7 +72,7 @@ export const getFieldLinksForExplore = (options: {
   dataFrame?: DataFrame;
 }): Array<LinkModel<Field>> => {
   const { field, vars, splitOpenFn, range, rowIndex, dataFrame } = options;
-  const scopedVars: any = { ...(vars || {}) };
+  const scopedVars: ScopedVars = { ...(vars || {}) };
   scopedVars['__value'] = {
     value: {
       raw: field.values.get(rowIndex),
@@ -127,10 +128,28 @@ export const getFieldLinksForExplore = (options: {
         }
         return linkModel;
       } else {
+        let internalLinkSpecificVars: ScopedVars = {};
+        if (link.internal?.transformations) {
+          link.internal?.transformations.forEach((transformation) => {
+            let fieldValue;
+            if (transformation.field) {
+              const transformField = dataFrame?.fields.find((field) => field.name === transformation.field);
+              fieldValue = transformField?.values.get(rowIndex);
+            } else {
+              fieldValue = field.values.get(rowIndex);
+            }
+
+            internalLinkSpecificVars = {
+              ...internalLinkSpecificVars,
+              ...getTransformationVars(transformation, fieldValue, field.name),
+            };
+          });
+        }
+
         return mapInternalLinkToExplore({
           link,
           internalLink: link.internal,
-          scopedVars: scopedVars,
+          scopedVars: { ...scopedVars, ...internalLinkSpecificVars },
           range,
           field,
           onClickFn: splitOpenFn,