Explore: Add transformations to correlation data links (#61799)

* bring in source from database

* bring in transformations from database

* add regex transformations to scopevar

* Consolidate types, add better example, cleanup

* Add var only if match

* Change ScopedVar to not require text, do not leak transformation-made variables between links

* Add mappings and start implementing logfmt

* Add mappings and start implementing logfmt

* Remove mappings, turn off global regex

* Add example yaml and omit transformations if empty

* Fix the yaml

* Add logfmt transformation

* Cleanup transformations and yaml

* add transformation field to FE types and use it, safeStringify logfmt values

* Add tests, only safe stringify if non-string, fix bug with safe stringify where it would return empty string with false value

* Add test for transformation field

* Do not add null transformations object

* Break out transformation logic, add tests to backend code

* Fix lint errors I understand 😅

* Fix the backend lint error

* Remove unnecessary code and mark new Transformations object as internal

* Add support for named capture groups

* Remove type assertion

* Remove variable name from transformation

* Add test for overriding regexes

* Add back variable name field, but change to mapValue

* fix go api test

* Change transformation types to enum, add better provisioning checks for bad type name and format

* Check for expression with regex transformations

pkg/services/correlations/models.go

@@ -14,10 +14,21 @@ var (
 	ErrCorrelationNotFound                = errors.New("correlation not found")
 	ErrUpdateCorrelationEmptyParams       = errors.New("not enough parameters to edit correlation")
 	ErrInvalidConfigType                  = errors.New("invalid correlation config type")
+	ErrInvalidTransformationType          = errors.New("invalid transformation type")
+	ErrTransformationNotNested            = errors.New("transformations must be nested under config")
+	ErrTransformationRegexReqExp          = errors.New("regex transformations require expression")
 )
 
 type CorrelationConfigType string
 
+type Transformation struct {
+	//Enum: regex,logfmt
+	Type       string `json:"type"`
+	Expression string `json:"expression,omitempty"`
+	Field      string `json:"field,omitempty"`
+	MapValue   string `json:"mapValue,omitempty"`
+}
+
 const (
 	ConfigTypeQuery CorrelationConfigType = "query"
 )
@@ -29,6 +40,19 @@ func (t CorrelationConfigType) Validate() error {
 	return nil
 }
 
+func (t Transformations) Validate() error {
+	for _, v := range t {
+		if v.Type != "regex" && v.Type != "logfmt" {
+			return fmt.Errorf("%s: \"%s\"", ErrInvalidTransformationType, t)
+		} else if v.Type == "regex" && len(v.Expression) == 0 {
+			return fmt.Errorf("%s: \"%s\"", ErrTransformationRegexReqExp, t)
+		}
+	}
+	return nil
+}
+
+type Transformations []Transformation
+
 // swagger:model
 type CorrelationConfig struct {
 	// Field used to attach the correlation link
@@ -42,21 +66,28 @@ type CorrelationConfig struct {
 	// required:true
 	// example: { "expr": "job=app" }
 	Target map[string]interface{} `json:"target" binding:"Required"`
+	// Source data transformations
+	// required:false
+	// example: [{"type": "logfmt"}]
+	Transformations Transformations `json:"transformations,omitempty"`
 }
 
 func (c CorrelationConfig) MarshalJSON() ([]byte, error) {
 	target := c.Target
+	transformations := c.Transformations
 	if target == nil {
 		target = map[string]interface{}{}
 	}
 	return json.Marshal(struct {
-		Type   CorrelationConfigType  `json:"type"`
-		Field  string                 `json:"field"`
-		Target map[string]interface{} `json:"target"`
+		Type            CorrelationConfigType  `json:"type"`
+		Field           string                 `json:"field"`
+		Target          map[string]interface{} `json:"target"`
+		Transformations Transformations        `json:"transformations,omitempty"`
 	}{
-		Type:   ConfigTypeQuery,
-		Field:  c.Field,
-		Target: target,
+		Type:            ConfigTypeQuery,
+		Field:           c.Field,
+		Target:          target,
+		Transformations: transformations,
 	})
 }
 
@@ -117,6 +148,10 @@ func (c CreateCorrelationCommand) Validate() error {
 	if c.TargetUID == nil && c.Config.Type == ConfigTypeQuery {
 		return fmt.Errorf("correlations of type \"%s\" must have a targetUID", ConfigTypeQuery)
 	}
+
+	if err := c.Config.Transformations.Validate(); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -151,6 +186,9 @@ type CorrelationConfigUpdateDTO struct {
 	// Target data query
 	// example: { "expr": "job=app" }
 	Target *map[string]interface{} `json:"target"`
+	// Source data transformations
+	// example: [{"type": "logfmt"},{"type":"regex","expression":"(Superman|Batman)", "variable":"name"}]
+	Transformations []Transformation `json:"transformations"`
 }
 
 func (c CorrelationConfigUpdateDTO) Validate() error {