Glue: Add DB migration & support provisioning for user-defined correlations config (#55560)

* Allow provisioning correlation config

* Simplify code

* Fix reading correlations test

* Fix linting errors

* Fix linting errors

* remove simpleJson

* Clean up

* Fix tests

* Update swagger docs

* Fix linting

* Fix linting

* Clean up swagger definitions

Co-authored-by: Elfo404 <me@giordanoricci.com>

devenv/datasources.yaml

@@ -52,6 +52,7 @@ datasources:
       implementation: prometheus
 
   - name: gdev-prometheus
+    uid: gdev-prometheus
     type: prometheus
     access: proxy
     url: http://localhost:9090
@@ -237,6 +238,13 @@ datasources:
       - targetUID: gdev-zipkin
         label: "Zipkin traces"
         description: "Related traces stored in Zipkin"
+      - targetUID: gdev-prometheus
+        label: "Logs to metrics"
+        description: "Related metrics stored in Prometheus"
+        config:
+          target:
+            expr: "{ job=\"test\" }"
+          field: "labels"
     jsonData:
       manageAlerts: false
       derivedFields:

pkg/services/correlations/database.go

@@ -16,6 +16,7 @@ func (s CorrelationsService) createCorrelation(ctx context.Context, cmd CreateCo
 		TargetUID:   cmd.TargetUID,
 		Label:       cmd.Label,
 		Description: cmd.Description,
+		Config:      cmd.Config,
 	}
 
 	err := s.SQLStore.WithTransactionalDbSession(ctx, func(session *sqlstore.DBSession) error {

pkg/services/correlations/models.go

@@ -13,7 +13,22 @@ var (
 	ErrUpdateCorrelationEmptyParams       = errors.New("not enough parameters to edit correlation")
 )
 
+// CorrelationConfigTarget is the target data query specific to target data source (Correlation.TargetUID)
+// swagger:model
+type CorrelationConfigTarget struct{}
+
+// swagger:model
+type CorrelationConfig struct {
+	// Field used to attach the correlation link
+	// required:true
+	Field string `json:"field"`
+	// Target data query
+	// required:true
+	Target CorrelationConfigTarget `json:"target"`
+}
+
 // Correlation is the model for correlations definitions
+// swagger:model
 type Correlation struct {
 	// Unique identifier of the correlation
 	// example: 50xhMlg9k
@@ -30,6 +45,9 @@ type Correlation struct {
 	// Description of the correlation
 	// example: Logs to Traces
 	Description string `json:"description" xorm:"description"`
+	// Correlation Configuration
+	// example: { field: "job", target: { query: "job=app" } }
+	Config CorrelationConfig `json:"config" xorm:"jsonb config"`
 }
 
 // CreateCorrelationResponse is the response struct for CreateCorrelationCommand
@@ -56,6 +74,9 @@ type CreateCorrelationCommand struct {
 	// Optional description of the correlation
 	// example: Logs to Traces
 	Description string `json:"description"`
+	// Arbitrary configuration object handled in frontend
+	// example: { field: "job", target: { query: "job=app" } }
+	Config CorrelationConfig `json:"config"`
 }
 
 // swagger:model

pkg/services/provisioning/datasources/datasources.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"fmt"
 
+	jsoniter "github.com/json-iterator/go"
+
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/correlations"
 	"github.com/grafana/grafana/pkg/services/datasources"
@@ -134,20 +136,37 @@ func (dc *DatasourceProvisioner) applyChanges(ctx context.Context, configPath st
 	return nil
 }
 
-func makeCreateCorrelationCommand(correlation map[string]interface{}, SourceUid string, OrgId int64) (correlations.CreateCorrelationCommand, error) {
+func makeCreateCorrelationCommand(correlation map[string]interface{}, SourceUID string, OrgId int64) (correlations.CreateCorrelationCommand, error) {
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	targetUID, ok := correlation["targetUID"].(string)
 	if !ok {
 		return correlations.CreateCorrelationCommand{}, fmt.Errorf("correlation missing targetUID")
 	}
 
-	return correlations.CreateCorrelationCommand{
-		SourceUID:         SourceUid,
+	createCommand := correlations.CreateCorrelationCommand{
+		SourceUID:         SourceUID,
 		TargetUID:         targetUID,
 		Label:             correlation["label"].(string),
 		Description:       correlation["description"].(string),
 		OrgId:             OrgId,
 		SkipReadOnlyCheck: true,
-	}, nil
+	}
+
+	if correlation["config"] != nil {
+		jsonbody, err := json.Marshal(correlation["config"])
+		if err != nil {
+			return correlations.CreateCorrelationCommand{}, err
+		}
+
+		config := correlations.CorrelationConfig{}
+		if err := json.Unmarshal(jsonbody, &config); err != nil {
+			return correlations.CreateCorrelationCommand{}, err
+		}
+
+		createCommand.Config = config
+	}
+
+	return createCommand, nil
 }
 
 func (dc *DatasourceProvisioner) deleteDatasources(ctx context.Context, dsToDelete []*deleteDatasourceConfig) error {

pkg/services/sqlstore/migrations/correlations_mig.go

@@ -25,4 +25,8 @@ func addCorrelationsMigrations(mg *Migrator) {
 
 	mg.AddMigration("add index correlations.uid", NewAddIndexMigration(correlationsV1, correlationsV1.Indices[0]))
 	mg.AddMigration("add index correlations.source_uid", NewAddIndexMigration(correlationsV1, correlationsV1.Indices[1]))
+
+	mg.AddMigration("add correlation config column", NewAddColumnMigration(correlationsV1, &Column{
+		Name: "config", Type: DB_Text, Nullable: true,
+	}))
 }

pkg/tests/api/correlations/correlations_create_test.go

@@ -211,7 +211,7 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
 		require.NoError(t, res.Body.Close())
 	})
 
-	t.Run("Should correctly create a correlation", func(t *testing.T) {
+	t.Run("Should correctly create a correlation without a config", func(t *testing.T) {
 		description := "a description"
 		label := "a label"
 		res := ctx.Post(PostParams{
@@ -240,4 +240,68 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
 
 		require.NoError(t, res.Body.Close())
 	})
+
+	t.Run("Should correctly create a correlation with a correct config", func(t *testing.T) {
+		description := "a description"
+		label := "a label"
+		res := ctx.Post(PostParams{
+			url: fmt.Sprintf("/api/datasources/uid/%s/correlations", writableDs),
+			body: fmt.Sprintf(`{
+					"targetUID": "%s",
+					"description": "%s",
+					"label": "%s",
+					"config": {
+						"field": "fieldName",
+						"target": { "expr": "foo" }
+					}
+				}`, writableDs, description, label),
+			user: adminUser,
+		})
+		require.Equal(t, http.StatusOK, res.StatusCode)
+
+		responseBody, err := io.ReadAll(res.Body)
+		require.NoError(t, err)
+
+		var response correlations.CreateCorrelationResponseBody
+		err = json.Unmarshal(responseBody, &response)
+		require.NoError(t, err)
+
+		require.Equal(t, "Correlation created", response.Message)
+		require.Equal(t, writableDs, response.Result.SourceUID)
+		require.Equal(t, writableDs, response.Result.TargetUID)
+		require.Equal(t, description, response.Result.Description)
+		require.Equal(t, label, response.Result.Label)
+		require.Equal(t, "fieldName", response.Result.Config.Field)
+
+		require.NoError(t, res.Body.Close())
+	})
+
+	t.Run("Should not create a correlation with incorrect config", func(t *testing.T) {
+		description := "a description"
+		label := "a label"
+		res := ctx.Post(PostParams{
+			url: fmt.Sprintf("/api/datasources/uid/%s/correlations", writableDs),
+			body: fmt.Sprintf(`{
+					"targetUID": "%s",
+					"description": "%s",
+					"label": "%s",
+					"config": {
+						"field": 2
+					}
+				}`, writableDs, description, label),
+			user: adminUser,
+		})
+		require.Equal(t, http.StatusBadRequest, res.StatusCode)
+
+		responseBody, err := io.ReadAll(res.Body)
+		require.NoError(t, err)
+
+		var response errorResponseBody
+		err = json.Unmarshal(responseBody, &response)
+		require.NoError(t, err)
+
+		require.Contains(t, response.Message, "bad request data")
+
+		require.NoError(t, res.Body.Close())
+	})
 }

pkg/tests/api/correlations/correlations_read_test.go

@@ -75,6 +75,10 @@ func TestIntegrationReadCorrelation(t *testing.T) {
 		SourceUID: dsWithCorrelations.Uid,
 		TargetUID: dsWithCorrelations.Uid,
 		OrgId:     dsWithCorrelations.OrgId,
+		Config: correlations.CorrelationConfig{
+			Field:  "foo",
+			Target: struct{}{},
+		},
 	})
 
 	createDsCommand = &datasources.AddDataSourceCommand{

public/api-merged.json

@@ -11383,6 +11383,9 @@
       "description": "Correlation is the model for correlations definitions",
       "type": "object",
       "properties": {
+        "config": {
+          "$ref": "#/definitions/CorrelationConfig"
+        },
         "description": {
           "description": "Description of the correlation",
           "type": "string",
@@ -11410,6 +11413,26 @@
         }
       }
     },
+    "CorrelationConfig": {
+      "type": "object",
+      "required": [
+        "field",
+        "target"
+      ],
+      "properties": {
+        "field": {
+          "description": "Field used to attach the correlation link",
+          "type": "string"
+        },
+        "target": {
+          "$ref": "#/definitions/CorrelationConfigTarget"
+        }
+      }
+    },
+    "CorrelationConfigTarget": {
+      "description": "CorrelationConfigTarget is the target data query specific to target data source (Correlation.TargetUID)",
+      "type": "object"
+    },
     "CreateAlertNotificationCommand": {
       "type": "object",
       "properties": {
@@ -11449,6 +11472,9 @@
       "description": "CreateCorrelationCommand is the command for creating a correlation",
       "type": "object",
       "properties": {
+        "config": {
+          "$ref": "#/definitions/CorrelationConfig"
+        },
         "description": {
           "description": "Optional description of the correlation",
           "type": "string",
@@ -20100,4 +20126,4 @@
       "name": "service_accounts"
     }
   ]
-}
+}

public/api-spec.json

@@ -10403,6 +10403,9 @@
       "description": "Correlation is the model for correlations definitions",
       "type": "object",
       "properties": {
+        "config": {
+          "$ref": "#/definitions/CorrelationConfig"
+        },
         "description": {
           "description": "Description of the correlation",
           "type": "string",
@@ -10430,6 +10433,26 @@
         }
       }
     },
+    "CorrelationConfig": {
+      "type": "object",
+      "required": [
+        "field",
+        "target"
+      ],
+      "properties": {
+        "field": {
+          "description": "Field used to attach the correlation link",
+          "type": "string"
+        },
+        "target": {
+          "$ref": "#/definitions/CorrelationConfigTarget"
+        }
+      }
+    },
+    "CorrelationConfigTarget": {
+      "description": "CorrelationConfigTarget is the target data query specific to target data source (Correlation.TargetUID)",
+      "type": "object"
+    },
     "CreateAlertNotificationCommand": {
       "type": "object",
       "properties": {
@@ -10469,6 +10492,9 @@
       "description": "CreateCorrelationCommand is the command for creating a correlation",
       "type": "object",
       "properties": {
+        "config": {
+          "$ref": "#/definitions/CorrelationConfig"
+        },
         "description": {
           "description": "Optional description of the correlation",
           "type": "string",
@@ -16125,4 +16151,4 @@
       "name": "service_accounts"
     }
   ]
-}
+}

public/openapi3.json

@@ -2825,6 +2825,9 @@
       "Correlation": {
         "description": "Correlation is the model for correlations definitions",
         "properties": {
+          "config": {
+            "$ref": "#/components/schemas/CorrelationConfig"
+          },
           "description": {
             "description": "Description of the correlation",
             "example": "Logs to Traces",
@@ -2853,6 +2856,26 @@
         },
         "type": "object"
       },
+      "CorrelationConfig": {
+        "properties": {
+          "field": {
+            "description": "Field used to attach the correlation link",
+            "type": "string"
+          },
+          "target": {
+            "$ref": "#/components/schemas/CorrelationConfigTarget"
+          }
+        },
+        "required": [
+          "field",
+          "target"
+        ],
+        "type": "object"
+      },
+      "CorrelationConfigTarget": {
+        "description": "CorrelationConfigTarget is the target data query specific to target data source (Correlation.TargetUID)",
+        "type": "object"
+      },
       "CreateAlertNotificationCommand": {
         "properties": {
           "disableResolveMessage": {
@@ -2891,6 +2914,9 @@
       "CreateCorrelationCommand": {
         "description": "CreateCorrelationCommand is the command for creating a correlation",
         "properties": {
+          "config": {
+            "$ref": "#/components/schemas/CorrelationConfig"
+          },
           "description": {
             "description": "Optional description of the correlation",
             "example": "Logs to Traces",
@@ -21415,4 +21441,4 @@
       "name": "service_accounts"
     }
   ]
-}
+}