IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Explore: Add transformations to correlation data links (#61799)

Browse Source 
* bring in source from database

* bring in transformations from database

* add regex transformations to scopevar

* Consolidate types, add better example, cleanup

* Add var only if match

* Change ScopedVar to not require text, do not leak transformation-made variables between links

* Add mappings and start implementing logfmt

* Add mappings and start implementing logfmt

* Remove mappings, turn off global regex

* Add example yaml and omit transformations if empty

* Fix the yaml

* Add logfmt transformation

* Cleanup transformations and yaml

* add transformation field to FE types and use it, safeStringify logfmt values

* Add tests, only safe stringify if non-string, fix bug with safe stringify where it would return empty string with false value

* Add test for transformation field

* Do not add null transformations object

* Break out transformation logic, add tests to backend code

* Fix lint errors I understand 😅

* Fix the backend lint error

* Remove unnecessary code and mark new Transformations object as internal

* Add support for named capture groups

* Remove type assertion

* Remove variable name from transformation

* Add test for overriding regexes

* Add back variable name field, but change to mapValue

* fix go api test

* Change transformation types to enum, add better provisioning checks for bad type name and format

* Check for expression with regex transformations
This commit is contained in:
Kristina 2023-02-22 06:53:03 -06:00 committed by GitHub
parent f64b7fe8a7
commit 06dfe2156f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 377 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.betterer.results
View File

@ -3574,9 +3574,6 @@ exports[`better eslint`] = {
[0, 0, 0, "Do not use any type assertions.", "4"],
[0, 0, 0, "Do not use any type assertions.", "5"]
],
"public/app/features/explore/utils/links.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"]
],
"public/app/features/expressions/ExpressionDatasource.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"],

2
package.json
View File

@ -140,7 +140,7 @@
"@types/js-yaml": "^4.0.5",
"@types/jsurl": "^1.2.28",
"@types/lodash": "4.14.187",
"@types/logfmt": "^1.2.1",
"@types/logfmt": "^1.2.3",
"@types/mousetrap": "1.6.10",
"@types/node": "18.14.0",
"@types/ol-ext": "npm:@siedlerchr/types-ol-ext@3.0.6",

2
packages/grafana-data/src/types/ScopedVars.ts
View File

@ -1,5 +1,5 @@
export interface ScopedVar<T = any> {
text: any;
text?: any;
value: T;
[key: string]: any;
}

15
packages/grafana-data/src/types/dataLink.ts
View File

@ -40,12 +40,27 @@ export interface DataLink<T extends DataQuery = any> {
internal?: InternalDataLink<T>;
}
/** @internal */
export enum SupportedTransformationTypes {
Regex = 'regex',
Logfmt = 'logfmt',
}
/** @internal */
export interface DataLinkTransformationConfig {
type: SupportedTransformationTypes;
field?: string;
expression?: string;
mapValue?: string;
}
/** @internal */
export interface InternalDataLink<T extends DataQuery = any> {
query: T;
datasourceUid: string;
datasourceName: string; // used as a title if `DataLink.title` is empty
panelsState?: ExplorePanelsState;
transformations?: DataLinkTransformationConfig[];
}
export type LinkTarget = '_blank' | '_self' | undefined;

3
pkg/services/correlations/database.go
View File

@ -129,6 +129,9 @@ func (s CorrelationsService) updateCorrelation(ctx context.Context, cmd UpdateCo
if cmd.Config.Target != nil {
correlation.Config.Target = *cmd.Config.Target
}
if cmd.Config.Transformations != nil {
correlation.Config.Transformations = cmd.Config.Transformations
}
}
updateCount, err := session.Where("uid = ? AND source_uid = ?", correlation.UID, correlation.SourceUID).Limit(1).Update(correlation)

50
pkg/services/correlations/models.go
View File

@ -14,10 +14,21 @@ var (
ErrCorrelationNotFound = errors.New("correlation not found")
ErrUpdateCorrelationEmptyParams = errors.New("not enough parameters to edit correlation")
ErrInvalidConfigType = errors.New("invalid correlation config type")
ErrInvalidTransformationType = errors.New("invalid transformation type")
ErrTransformationNotNested = errors.New("transformations must be nested under config")
ErrTransformationRegexReqExp = errors.New("regex transformations require expression")
)
type CorrelationConfigType string
type Transformation struct {
//Enum: regex,logfmt
Type string `json:"type"`
Expression string `json:"expression,omitempty"`
Field string `json:"field,omitempty"`
MapValue string `json:"mapValue,omitempty"`
}
const (
ConfigTypeQuery CorrelationConfigType = "query"
)
@ -29,6 +40,19 @@ func (t CorrelationConfigType) Validate() error {
return nil
}
func (t Transformations) Validate() error {
for _, v := range t {
if v.Type != "regex" && v.Type != "logfmt" {
return fmt.Errorf("%s: \"%s\"", ErrInvalidTransformationType, t)
} else if v.Type == "regex" && len(v.Expression) == 0 {
return fmt.Errorf("%s: \"%s\"", ErrTransformationRegexReqExp, t)
}
}
return nil
}
type Transformations []Transformation
// swagger:model
type CorrelationConfig struct {
// Field used to attach the correlation link
@ -42,21 +66,28 @@ type CorrelationConfig struct {
// required:true
// example: { "expr": "job=app" }
Target map[string]interface{} `json:"target" binding:"Required"`
// Source data transformations
// required:false
// example: [{"type": "logfmt"}]
Transformations Transformations `json:"transformations,omitempty"`
}
func (c CorrelationConfig) MarshalJSON() ([]byte, error) {
target := c.Target
transformations := c.Transformations
if target == nil {
target = map[string]interface{}{}
}
return json.Marshal(struct {
Type CorrelationConfigType `json:"type"`
Field string `json:"field"`
Target map[string]interface{} `json:"target"`
Type CorrelationConfigType `json:"type"`
Field string `json:"field"`
Target map[string]interface{} `json:"target"`
Transformations Transformations `json:"transformations,omitempty"`
}{
Type: ConfigTypeQuery,
Field: c.Field,
Target: target,
Type: ConfigTypeQuery,
Field: c.Field,
Target: target,
Transformations: transformations,
})
}
@ -117,6 +148,10 @@ func (c CreateCorrelationCommand) Validate() error {
if c.TargetUID == nil && c.Config.Type == ConfigTypeQuery {
return fmt.Errorf("correlations of type \"%s\" must have a targetUID", ConfigTypeQuery)
}
if err := c.Config.Transformations.Validate(); err != nil {
return err
}
return nil
}
@ -151,6 +186,9 @@ type CorrelationConfigUpdateDTO struct {
// Target data query
// example: { "expr": "job=app" }
Target *map[string]interface{} `json:"target"`
// Source data transformations
// example: [{"type": "logfmt"},{"type":"regex","expression":"(Superman|Batman)", "variable":"name"}]
Transformations []Transformation `json:"transformations"`
}
func (c CorrelationConfigUpdateDTO) Validate() error {

4
pkg/services/provisioning/datasources/datasources.go
View File

@ -152,6 +152,10 @@ func makeCreateCorrelationCommand(correlation map[string]interface{}, SourceUID
createCommand.TargetUID = &targetUID
}
if correlation["transformations"] != nil {
return correlations.CreateCorrelationCommand{}, correlations.ErrTransformationNotNested
}
if correlation["config"] != nil {
jsonbody, err := json.Marshal(correlation["config"])
if err != nil {

10
pkg/tests/api/correlations/correlations_create_test.go
View File

@ -237,6 +237,8 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
label := "a label"
fieldName := "fieldName"
configType := correlations.ConfigTypeQuery
transformation := correlations.Transformation{Type: "logfmt"}
transformation2 := correlations.Transformation{Type: "regex", Expression: "testExpression", MapValue: "testVar"}
res := ctx.Post(PostParams{
url: fmt.Sprintf("/api/datasources/uid/%s/correlations", writableDs),
body: fmt.Sprintf(`{
@ -246,7 +248,11 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
"config": {
"type": "%s",
"field": "%s",
"target": { "expr": "foo" }
"target": { "expr": "foo" },
"transformations": [
{"type": "logfmt"},
{"type": "regex", "expression": "testExpression", "mapValue": "testVar"}
]
}
}`, writableDs, description, label, configType, fieldName),
user: adminUser,
@ -268,6 +274,8 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
require.Equal(t, configType, response.Result.Config.Type)
require.Equal(t, fieldName, response.Result.Config.Field)
require.Equal(t, map[string]interface{}{"expr": "foo"}, response.Result.Config.Target)
require.Equal(t, transformation, response.Result.Config.Transformations[0])
require.Equal(t, transformation2, response.Result.Config.Transformations[1])
require.NoError(t, res.Body.Close())
})

3
pkg/tests/api/correlations/correlations_read_test.go
View File

@ -79,6 +79,9 @@ func TestIntegrationReadCorrelation(t *testing.T) {
Type: correlations.ConfigTypeQuery,
Field: "foo",
Target: map[string]interface{}{},
Transformations: []correlations.Transformation{
{Type: "logfmt"},
},
},
})

4
pkg/tests/api/correlations/correlations_update_test.go
View File

@ -287,7 +287,8 @@ func TestIntegrationUpdateCorrelation(t *testing.T) {
"config": {
"field": "field",
"type": "query",
"target": { "expr": "bar" }
"target": { "expr": "bar" },
"transformations": [ {"type": "logfmt"} ]
}
}`,
})
@ -305,6 +306,7 @@ func TestIntegrationUpdateCorrelation(t *testing.T) {
require.Equal(t, "1", response.Result.Description)
require.Equal(t, "field", response.Result.Config.Field)
require.Equal(t, map[string]interface{}{"expr": "bar"}, response.Result.Config.Target)
require.Equal(t, correlations.Transformation{Type: "logfmt"}, response.Result.Config.Transformations[0])
require.NoError(t, res.Body.Close())
// partially updating only label

2
public/app/core/utils/explore.ts
View File

@ -192,7 +192,7 @@ export const safeParseJson = (text?: string): any | undefined => {
};
export const safeStringifyValue = (value: any, space?: number) => {
if (!value) {
if (value === undefined || value === null) {
return '';
}

34
public/app/features/correlations/transformations.ts Normal file
View File

@ -0,0 +1,34 @@
import logfmt from 'logfmt';
import { ScopedVars, DataLinkTransformationConfig, SupportedTransformationTypes } from '@grafana/data';
import { safeStringifyValue } from 'app/core/utils/explore';
export const getTransformationVars = (
transformation: DataLinkTransformationConfig,
fieldValue: string,
fieldName: string
): ScopedVars => {
let transformationScopedVars: ScopedVars = {};
let transformVal: { [key: string]: string | boolean | null | undefined } = {};
if (transformation.type === SupportedTransformationTypes.Regex && transformation.expression) {
const regexp = new RegExp(transformation.expression, 'gi');
const matches = fieldValue.matchAll(regexp);
for (const match of matches) {
if (match.groups) {
transformVal = match.groups;
} else {
transformVal[transformation.mapValue || fieldName] = match[1] || match[0];
}
}
} else if (transformation.type === SupportedTransformationTypes.Logfmt) {
transformVal = logfmt.parse(fieldValue);
}
Object.keys(transformVal).forEach((key) => {
const transformValString =
typeof transformVal[key] === 'string' ? transformVal[key] : safeStringifyValue(transformVal[key]);
transformationScopedVars[key] = { value: transformValString };
});
return transformationScopedVars;
};

4
public/app/features/correlations/types.ts
View File

@ -1,3 +1,5 @@
import { DataLinkTransformationConfig } from '@grafana/data';
export interface AddCorrelationResponse {
correlation: Correlation;
}
@ -5,10 +7,12 @@ export interface AddCorrelationResponse {
export type GetCorrelationsResponse = Correlation[];
type CorrelationConfigType = 'query';
export interface CorrelationConfig {
field: string;
target: object;
type: CorrelationConfigType;
transformations?: DataLinkTransformationConfig[];
}
export interface Correlation {

1
public/app/features/correlations/utils.ts
View File

@ -39,6 +39,7 @@ const decorateDataFrameWithInternalDataLinks = (dataFrame: DataFrame, correlatio
query: correlation.config?.target,
datasourceUid: correlation.target.uid,
datasourceName: correlation.target.name,
transformations: correlation.config?.transformations,
},
url: '',
title: correlation.label || correlation.target.name,

228
public/app/features/explore/utils/links.test.ts
View File

@ -6,6 +6,7 @@ import {
Field,
FieldType,
InterpolateFunction,
SupportedTransformationTypes,
TimeRange,
toDataFrame,
} from '@grafana/data';
@ -240,6 +241,233 @@ describe('getFieldLinksForExplore', () => {
);
});
it('returns internal links with logfmt and regex transformation', () => {
const transformationLink: DataLink = {
title: '',
url: '',
internal: {
query: { query: 'http_requests{app=${application} env=${environment}}' },
datasourceUid: 'uid_1',
datasourceName: 'test_ds',
transformations: [
{ type: SupportedTransformationTypes.Logfmt },
{ type: SupportedTransformationTypes.Regex, expression: 'host=(dev|prod)', mapValue: 'environment' },
],
},
};
const { field, range, dataFrame } = setup(transformationLink, true, {
name: 'msg',
type: FieldType.string,
values: new ArrayVector(['application=foo host=dev-001', 'application=bar host=prod-003']),
config: {
links: [transformationLink],
},
});
const links = [
getFieldLinksForExplore({ field, rowIndex: 0, range, dataFrame }),
getFieldLinksForExplore({ field, rowIndex: 1, range, dataFrame }),
];
expect(links[0]).toHaveLength(1);
expect(links[0][0].href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=foo env=dev}"}]}'
)}`
);
expect(links[1]).toHaveLength(1);
expect(links[1][0].href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=bar env=prod}"}]}'
)}`
);
});
it('returns internal links with 2 unnamed regex transformations and use the last transformation', () => {
const transformationLink: DataLink = {
title: '',
url: '',
internal: {
query: { query: 'http_requests{env=${msg}}' },
datasourceUid: 'uid_1',
datasourceName: 'test_ds',
transformations: [
{ type: SupportedTransformationTypes.Regex, expression: 'fieldA=(asparagus|broccoli)' },
{ type: SupportedTransformationTypes.Regex, expression: 'fieldB=(apple|banana)' },
],
},
};
const { field, range, dataFrame } = setup(transformationLink, true, {
name: 'msg',
type: FieldType.string,
values: new ArrayVector(['fieldA=asparagus fieldB=banana', 'fieldA=broccoli fieldB=apple']),
config: {
links: [transformationLink],
},
});
const links = [
getFieldLinksForExplore({ field, rowIndex: 0, range, dataFrame }),
getFieldLinksForExplore({ field, rowIndex: 1, range, dataFrame }),
];
expect(links[0]).toHaveLength(1);
expect(links[0][0].href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{env=banana}"}]}'
)}`
);
expect(links[1]).toHaveLength(1);
expect(links[1][0].href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{env=apple}"}]}'
)}`
);
});
it('returns internal links with logfmt with stringified booleans', () => {
const transformationLink: DataLink = {
title: '',
url: '',
internal: {
query: { query: 'http_requests{app=${application} isOnline=${online}}' },
datasourceUid: 'uid_1',
datasourceName: 'test_ds',
transformations: [{ type: SupportedTransformationTypes.Logfmt }],
},
};
const { field, range, dataFrame } = setup(transformationLink, true, {
name: 'msg',
type: FieldType.string,
values: new ArrayVector(['application=foo online=true', 'application=bar online=false']),
config: {
links: [transformationLink],
},
});
const links = [
getFieldLinksForExplore({ field, rowIndex: 0, range, dataFrame }),
getFieldLinksForExplore({ field, rowIndex: 1, range, dataFrame }),
];
expect(links[0]).toHaveLength(1);
expect(links[0][0].href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=foo isOnline=true}"}]}'
)}`
);
expect(links[1]).toHaveLength(1);
expect(links[1][0].href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=bar isOnline=false}"}]}'
)}`
);
});
it('returns internal links with logfmt with correct data on transformation-defined field', () => {
const transformationLink: DataLink = {
title: '',
url: '',
internal: {
query: { query: 'http_requests{app=${application}}' },
datasourceUid: 'uid_1',
datasourceName: 'test_ds',
transformations: [{ type: SupportedTransformationTypes.Logfmt, field: 'fieldNamedInTransformation' }],
},
};
// fieldWithLink has the transformation, but the transformation has defined fieldNamedInTransformation as its field to transform
const { field, range, dataFrame } = setup(
transformationLink,
true,
{
name: 'fieldWithLink',
type: FieldType.string,
values: new ArrayVector(['application=link', 'application=link2']),
config: {
links: [transformationLink],
},
},
[
{
name: 'fieldNamedInTransformation',
type: FieldType.string,
values: new ArrayVector(['application=transform', 'application=transform2']),
config: {},
},
]
);
const links = [
getFieldLinksForExplore({ field, rowIndex: 0, range, dataFrame }),
getFieldLinksForExplore({ field, rowIndex: 1, range, dataFrame }),
];
expect(links[0]).toHaveLength(1);
expect(links[0][0].href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=transform}"}]}'
)}`
);
expect(links[1]).toHaveLength(1);
expect(links[1][0].href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=transform2}"}]}'
)}`
);
});
it('returns internal links with regex named capture groups', () => {
const transformationLink: DataLink = {
title: '',
url: '',
internal: {
query: { query: 'http_requests{app=${application} env=${environment}}' },
datasourceUid: 'uid_1',
datasourceName: 'test_ds',
transformations: [
{
type: SupportedTransformationTypes.Regex,
expression: '(?=.*(?<application>(grafana|loki)))(?=.*(?<environment>(dev|prod)))',
},
],
},
};
const { field, range, dataFrame } = setup(transformationLink, true, {
name: 'msg',
type: FieldType.string,
values: new ArrayVector(['foo loki prod', 'dev bar grafana', 'prod grafana foo']),
config: {
links: [transformationLink],
},
});
const links = [
getFieldLinksForExplore({ field, rowIndex: 0, range, dataFrame }),
getFieldLinksForExplore({ field, rowIndex: 1, range, dataFrame }),
getFieldLinksForExplore({ field, rowIndex: 2, range, dataFrame }),
];
expect(links[0]).toHaveLength(1);
expect(links[0][0].href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=loki env=prod}"}]}'
)}`
);
expect(links[1]).toHaveLength(1);
expect(links[1][0].href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=grafana env=dev}"}]}'
)}`
);
expect(links[2]).toHaveLength(1);
expect(links[2][0].href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"now-1h","to":"now"},"datasource":"uid_1","queries":[{"query":"http_requests{app=grafana env=prod}"}]}'
)}`
);
});
it('returns no internal links when target contains empty template variables', () => {
const { field, range, dataFrame } = setup({
title: '',

23
public/app/features/explore/utils/links.ts
View File

@ -15,6 +15,7 @@ import {
} from '@grafana/data';
import { getTemplateSrv } from '@grafana/runtime';
import { contextSrv } from 'app/core/services/context_srv';
import { getTransformationVars } from 'app/features/correlations/transformations';
import { getLinkSrv } from '../../panel/panellinks/link_srv';
@ -71,7 +72,7 @@ export const getFieldLinksForExplore = (options: {
dataFrame?: DataFrame;
}): Array<LinkModel<Field>> => {
const { field, vars, splitOpenFn, range, rowIndex, dataFrame } = options;
const scopedVars: any = { ...(vars || {}) };
const scopedVars: ScopedVars = { ...(vars || {}) };
scopedVars['__value'] = {
value: {
raw: field.values.get(rowIndex),
@ -127,10 +128,28 @@ export const getFieldLinksForExplore = (options: {
}
return linkModel;
} else {
let internalLinkSpecificVars: ScopedVars = {};
if (link.internal?.transformations) {
link.internal?.transformations.forEach((transformation) => {
let fieldValue;
if (transformation.field) {
const transformField = dataFrame?.fields.find((field) => field.name === transformation.field);
fieldValue = transformField?.values.get(rowIndex);
} else {
fieldValue = field.values.get(rowIndex);
}
internalLinkSpecificVars = {
...internalLinkSpecificVars,
...getTransformationVars(transformation, fieldValue, field.name),
};
});
}
return mapInternalLinkToExplore({
link,
internalLink: link.internal,
scopedVars: scopedVars,
scopedVars: { ...scopedVars, ...internalLinkSpecificVars },
range,
field,
onClickFn: splitOpenFn,

10
yarn.lock
View File

@ -11497,12 +11497,12 @@ __metadata:
languageName: node
linkType: hard
"@types/logfmt@npm:^1.2.1":
version: 1.2.2
resolution: "@types/logfmt@npm:1.2.2"
"@types/logfmt@npm:^1.2.3":
version: 1.2.3
resolution: "@types/logfmt@npm:1.2.3"
dependencies:
"@types/node": "*"
checksum: 3f75ce7cc79886b6c3a2b7c467c49d72752a494cf316b87315c7609f9d3f29ffc6d400b6ae9e81c3997ef1e8b0a442974b60e86b3e45a8174b9b0415aba40701
checksum: d5872ab0432c687dc95a4c3a1c21c8eca24553415ef6a34f6cbbe0eefc4b7b8fb8b2af80df4a53fcf7cc7b212569df568bed1b17f7c2a976c4416f4a67b285de
languageName: node
linkType: hard
@ -22002,7 +22002,7 @@ __metadata:
"@types/js-yaml": ^4.0.5
"@types/jsurl": ^1.2.28
"@types/lodash": 4.14.187
"@types/logfmt": ^1.2.1
"@types/logfmt": ^1.2.3
"@types/mousetrap": 1.6.10
"@types/node": 18.14.0
"@types/ol-ext": "npm:@siedlerchr/types-ol-ext@3.0.6"