IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2024-12-23 23:50:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Add clean_upgrade config and deprecate force_migration (#78324)

Browse Source 
* Alerting: Add clean_upgrade config and deprecate force_migration

Upgrading to UA and rolling back will no longer delete any data by default. 
Instead, each set of tables will remain unchanged when switching between 
legacy and UA. As such, the force_migration config has been deprecated 
and no extra configuration is required to roll back to legacy anymore.

If clean_upgrade is set to true when upgrading from legacy alerting to Unified
Alerting, grafana will first delete all existing Unified Alerting resources,
thus re-upgrading all organizations from scratch. If false or unset,
organizations that have previously upgraded will not lose their existing Unified
 Alerting data when switching between legacy and Unified Alerting.

 Similar to force_migration, it should be kept false when not needed as it may
 cause unintended data-loss if left enabled.

---------

Co-authored-by: Christopher Moyer <35463610+chri2547@users.noreply.github.com>
This commit is contained in:
Matthew Jacobson 2023-11-30 11:01:11 -05:00 committed by GitHub
parent 0f0249abea
commit 5a80962de9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 144 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
conf/defaults.ini
View File

@ -10,6 +10,7 @@ app_mode = production
instance_name = ${HOSTNAME}
# force migration will run migrations that might cause dataloss
# Deprecated, use clean_upgrade option in [unified_alerting.upgrade] instead.
force_migration = false
#################################### Paths ###############################
@ -1238,6 +1239,13 @@ loki_basic_auth_password =
# ex.
# mylabelkey = mylabelvalue
[unified_alerting.upgrade]
# If set to true when upgrading from legacy alerting to Unified Alerting, grafana will first delete all existing
# Unified Alerting resources, thus re-upgrading all organizations from scratch. If false or unset, organizations that
# have previously upgraded will not lose their existing Unified Alerting data when switching between legacy and
# Unified Alerting. Should be kept false when not needed as it may cause unintended data-loss if left enabled.
clean_upgrade = false
# NOTE: this configuration options are not used yet.
[remote.alertmanager]

8
conf/sample.ini
View File

@ -10,6 +10,7 @@
;instance_name = ${HOSTNAME}
# force migration will run migrations that might cause dataloss
# Deprecated, use clean_upgrade option in [unified_alerting.upgrade] instead.
;force_migration = false
#################################### Paths ####################################
@ -1155,6 +1156,13 @@
# Any number of label key-value-pairs can be provided.
; mylabelkey = mylabelvalue
[unified_alerting.upgrade]
# If set to true when upgrading from legacy alerting to Unified Alerting, grafana will first delete all existing
# Unified Alerting resources, thus re-upgrading all organizations from scratch. If false or unset, organizations that
# have previously upgraded will not lose their existing Unified Alerting data when switching between legacy and
# Unified Alerting. Should be kept false when not needed as it may cause unintended data-loss if left enabled.
;clean_upgrade = false
#################################### Alerting ############################
[alerting]
# Disable legacy alerting engine & UI features

15
docs/sources/alerting/set-up/migrating-alerts/_index.md
View File

@ -23,7 +23,7 @@ Existing installations that do not use legacy alerting will have Grafana Alertin
Likewise, existing installations that use legacy alerting will be automatically upgraded to Grafana Alerting unless you have opted out of Grafana Alerting before migration takes place. During the upgrade, legacy alerts are migrated to the new alerts type and no alerts or alerting data are lost.
Once the upgrade has taken place, you still have the option to roll back to legacy alerting. However, we do not recommend choosing this option. If you do choose to roll back, Grafana will restore your alerts to the alerts you had at the point in time when the upgrade took place. All new alerts and changes made exclusively in Grafana Alerting will be deleted.
Once the upgrade has taken place, you still have the option to roll back to legacy alerting. However, we do not recommend choosing this option. If you do choose to roll back, Grafana will restore your alerts to the alerts you had at the point in time when the upgrade took place.
{{% admonition type="note" %}}
Cloud customers, who do not want to upgrade to Grafana Alerting, should contact customer support.
@ -91,13 +91,9 @@ If you want to turn alerting back on, you can remove both flags to enable Grafan
Once the upgrade has taken place, you still have the option to roll back to legacy alerting. If you choose to roll back, Grafana will restore your alerts to the alerts you had at the point in time when the upgrade took place.
All new alerts and changes made exclusively in Grafana Alerting will be deleted.
To roll back to legacy alerting, enter the following in your configuration:
```toml
force_migration = true
[alerting]
enabled = true
@ -105,7 +101,14 @@ enabled = true
enabled = false
```
> **Note**: We do not recommend this option. If you choose to roll back, Grafana will restore your alerts to the alerts you had at the point in time when the upgrade took place. All new alerts and changes made exclusively in Grafana Alerting will be deleted.
> **Note**: The next time you upgrade to Grafana Alerting, Grafana will restore your Grafana Alerting alerts and configuration to those you had before rolling back.
If, after rolling back, you wish to delete any existing Grafana Alerting configuration and upgrade your legacy alerting configuration again from scratch, you can enable the `clean_upgrade` option:
```toml
[unified_alerting.upgrade]
clean_upgrade = true
```
## Opt in

32
docs/sources/setup-grafana/configure-grafana/_index.md
View File

@ -148,11 +148,19 @@ Options are `production` and `development`. Default is `production`. _Do not_ ch
Set the name of the grafana-server instance. Used in logging, internal metrics, and clustering info. Defaults to: `${HOSTNAME}`, which will be replaced with
environment variable `HOSTNAME`, if that is empty or does not exist Grafana will try to use system calls to get the machine name.
### force_migration
## force_migration
Force migration will run migrations that might cause data loss. Default is `false`.
{{% admonition type="note" %}}
This option is deprecated - [See `clean_upgrade` option]({{< relref "#clean_upgrade" >}}) instead.
{{% /admonition %}}
Set force_migration=true in your grafana.ini and restart Grafana to roll back and delete Unified Alerting configuration data. Any alert rules created while using Unified Alerting will be deleted by rolling back.
When you restart Grafana to rollback from Grafana Alerting to legacy alerting, delete any existing Grafana Alerting data, such as alert rules, contact points, and notification policies. Default is `false`.
If `false` or unset, existing Grafana Alerting data is not changed or deleted when rolling back to legacy alerting.
{{% admonition type="note" %}}
It should be kept false or unset when not needed, as it may cause unintended data loss if left enabled.
{{% /admonition %}}
<hr />
@ -1489,7 +1497,7 @@ For more information about the Grafana alerts, refer to [About Grafana Alerting]
### enabled
Enable or disable Grafana Alerting. If disabled, all your legacy alerting data will be available again, but the data you created using Grafana Alerting will be deleted. Set force_migration=true to avoid deletion of data. The default value is `true`.
Enable or disable Grafana Alerting. If disabled, all your legacy alerting data will be available again. The default value is `true`.
Alerting Rules migrated from dashboards and panels will include a link back via the `annotations`.
@ -1629,6 +1637,22 @@ For example: `disabled_labels=grafana_folder`
<hr>
## [unified_alerting.upgrade]
For more information about upgrading to Grafana Alerting, refer to [Upgrade Alerting](/docs/grafana/next/alerting/set-up/migrating-alerts/).
### clean_upgrade
When you restart Grafana to upgrade from legacy alerting to Grafana Alerting, delete any existing Grafana Alerting data from a previous upgrade, such as alert rules, contact points, and notification policies. Default is `false`.
If `false` or unset, existing Grafana Alerting data is not changed or deleted when you switch between legacy and Unified Alerting.
{{% admonition type="note" %}}
It should be kept false when not needed, as it may cause unintended data loss if left enabled.
{{% /admonition %}}
<hr>
## [alerting]
For more information about the legacy dashboard alerting feature in Grafana, refer to [the legacy Grafana alerts](/docs/grafana/v8.5/alerting/old-alerting/).

39
pkg/services/ngalert/migration/migration_test.go
View File

@ -50,27 +50,30 @@ func TestServiceStart(t *testing.T) {
expected: migrationStore.UnifiedAlerting,
},
{
name: "when unified alerting disabled, migration is already run and force migration is enabled, then revert migration",
name: "when unified alerting disabled, migration is already run and CleanUpgrade is enabled, then revert migration",
config: &setting.Cfg{
UnifiedAlerting: setting.UnifiedAlertingSettings{
Enabled: pointer(false),
Upgrade: setting.UnifiedAlertingUpgradeSettings{
CleanUpgrade: true,
},
},
ForceMigration: true,
},
starting: migrationStore.UnifiedAlerting,
expected: migrationStore.Legacy,
},
{
name: "when unified alerting disabled, migration is already run and force migration is disabled, then the migration should panic",
name: "when unified alerting disabled, migration is already run and CleanUpgrade is disabled, then the migration status should set to false",
config: &setting.Cfg{
UnifiedAlerting: setting.UnifiedAlertingSettings{
Enabled: pointer(false),
Upgrade: setting.UnifiedAlertingUpgradeSettings{
CleanUpgrade: false,
},
},
ForceMigration: false,
},
starting: migrationStore.UnifiedAlerting,
expected: migrationStore.UnifiedAlerting,
expectedErr: true,
starting: migrationStore.UnifiedAlerting,
expected: migrationStore.Legacy,
},
{
name: "when unified alerting enabled and migration is already run, then do nothing",
@ -92,6 +95,28 @@ func TestServiceStart(t *testing.T) {
starting: migrationStore.Legacy,
expected: migrationStore.Legacy,
},
{
name: "when unified alerting disabled, migration is already run and force migration is enabled, then revert migration",
config: &setting.Cfg{
UnifiedAlerting: setting.UnifiedAlertingSettings{
Enabled: pointer(false),
},
ForceMigration: true,
},
starting: migrationStore.UnifiedAlerting,
expected: migrationStore.Legacy,
},
{
name: "when unified alerting disabled, migration is already run and force migration is disabled, then the migration status should set to false",
config: &setting.Cfg{
UnifiedAlerting: setting.UnifiedAlertingSettings{
Enabled: pointer(false),
},
ForceMigration: false,
},
starting: migrationStore.UnifiedAlerting,
expected: migrationStore.Legacy,
},
}
sqlStore := db.InitTestDB(t)

18
pkg/services/ngalert/migration/service.go
View File

@ -16,9 +16,6 @@ import (
// actionName is the unique row-level lock name for serverlock.ServerLockService.
const actionName = "alerting migration"
//nolint:stylecheck
var ForceMigrationError = fmt.Errorf("Grafana has already been migrated to Unified Alerting. Any alert rules created while using Unified Alerting will be deleted by rolling back. Set force_migration=true in your grafana.ini and restart Grafana to roll back and delete Unified Alerting configuration data.")
type UpgradeService interface {
Run(ctx context.Context) error
}
@ -84,6 +81,7 @@ func newTransition(currentType migrationStore.AlertingType, cfg *setting.Cfg) tr
CurrentType: currentType,
DesiredType: desiredType,
CleanOnDowngrade: cfg.ForceMigration,
CleanOnUpgrade: cfg.UnifiedAlerting.Upgrade.CleanUpgrade,
}
}
@ -92,6 +90,7 @@ type transition struct {
CurrentType migrationStore.AlertingType
DesiredType migrationStore.AlertingType
CleanOnDowngrade bool
CleanOnUpgrade bool
}
// isNoChange returns true if the migration is a no-op.
@ -111,30 +110,27 @@ func (t transition) isDowngrading() bool {
// shouldClean returns true if the migration should delete all unified alerting data.
func (t transition) shouldClean() bool {
return t.isDowngrading() && t.CleanOnDowngrade
return t.isDowngrading() && t.CleanOnDowngrade || t.isUpgrading() && t.CleanOnUpgrade
}
// applyTransition applies the transition to the database.
// If the transition is a no-op, nothing will be done.
// If the transition is a downgrade and CleanOnDowngrade is false, nothing will be done.
// If the transition is a downgrade and CleanOnDowngrade is true, all unified alerting data will be deleted.
// If the transition is a downgrade and CleanOnDowngrade is false, an error will be returned.
// If the transition is an upgrade, all orgs will be migrated.
// If the transition is an upgrade and CleanOnUpgrade is false, all orgs will be migrated.
// If the transition is an upgrade and CleanOnUpgrade is true, all unified alerting data will be deleted and then all orgs will be migrated.
func (ms *migrationService) applyTransition(ctx context.Context, t transition) error {
l := ms.log.New(
"CurrentType", t.CurrentType,
"DesiredType", t.DesiredType,
"CleanOnDowngrade", t.CleanOnDowngrade,
"CleanOnUpgrade", t.CleanOnUpgrade,
)
if t.isNoChange() {
l.Info("Migration already complete")
return nil
}
// Safeguard to prevent accidental data loss when reverting from UA to LA.
if t.isDowngrading() && !ms.cfg.ForceMigration {
return ForceMigrationError
}
if t.shouldClean() {
l.Info("Cleaning up unified alerting data")
if err := ms.migrationStore.RevertAllOrgs(ctx); err != nil {

57
pkg/services/ngalert/migration/service_test.go
View File

@ -197,9 +197,9 @@ func TestServiceRevert(t *testing.T) {
// Run migration.
ctx := context.Background()
cfg := &setting.Cfg{
ForceMigration: true,
UnifiedAlerting: setting.UnifiedAlertingSettings{
Enabled: pointer(true),
Upgrade: setting.UnifiedAlertingUpgradeSettings{},
},
}
service := NewTestMigrationService(t, sqlStore, cfg)
@ -280,7 +280,7 @@ func TestServiceRevert(t *testing.T) {
}
})
t.Run("ForceMigration story", func(t *testing.T) {
t.Run("CleanUpgrade story", func(t *testing.T) {
sqlStore := db.InitTestDB(t)
x := sqlStore.GetEngine()
@ -304,14 +304,45 @@ func TestServiceRevert(t *testing.T) {
checkMigrationStatus(t, ctx, service, 1, true)
checkAlertRulesCount(t, x, 1, 1)
// Disable UA without ForceMigration.
// This run should throw an error.
// Disable UA.
// This run should just set migration status to false.
service.cfg.UnifiedAlerting.Enabled = pointer(false)
require.ErrorContains(t, service.Run(ctx), ForceMigrationError.Error())
checkAlertingType(t, ctx, service, migrationStore.UnifiedAlerting)
require.NoError(t, service.Run(ctx))
checkAlertingType(t, ctx, service, migrationStore.Legacy)
checkMigrationStatus(t, ctx, service, 1, true)
checkAlertRulesCount(t, x, 1, 1)
// Add another alert.
// Enable UA without clean flag.
// This run should not remigrate org, new alert is not migrated.
_, alertErr := x.Insert(createAlert(t, 1, 1, 2, "alert2", []string{"notifier1"}))
require.NoError(t, alertErr)
service.cfg.UnifiedAlerting.Enabled = pointer(true)
require.NoError(t, service.Run(ctx))
checkAlertingType(t, ctx, service, migrationStore.UnifiedAlerting)
checkMigrationStatus(t, ctx, service, 1, true)
checkAlertRulesCount(t, x, 1, 1) // Still 1
// Disable UA with clean flag.
// This run should not revert UA data.
service.cfg.UnifiedAlerting.Enabled = pointer(false)
service.cfg.UnifiedAlerting.Upgrade.CleanUpgrade = true
require.NoError(t, service.Run(ctx))
checkAlertingType(t, ctx, service, migrationStore.Legacy)
checkMigrationStatus(t, ctx, service, 1, true)
checkAlertRulesCount(t, x, 1, 1) // Still 1
// Enable UA with clean flag.
// This run should revert and remigrate org, new alert is migrated.
service.cfg.UnifiedAlerting.Enabled = pointer(true)
require.NoError(t, service.Run(ctx))
checkAlertingType(t, ctx, service, migrationStore.UnifiedAlerting)
checkMigrationStatus(t, ctx, service, 1, true)
checkAlertRulesCount(t, x, 1, 2) // Now we have 2
// The following tests ForceMigration which is deprecated and will be removed in v11.
service.cfg.UnifiedAlerting.Upgrade.CleanUpgrade = false
// Disable UA with force flag.
// This run should not revert UA data.
service.cfg.UnifiedAlerting.Enabled = pointer(false)
@ -319,19 +350,7 @@ func TestServiceRevert(t *testing.T) {
require.NoError(t, service.Run(ctx))
checkAlertingType(t, ctx, service, migrationStore.Legacy)
checkMigrationStatus(t, ctx, service, 1, false)
checkAlertRulesCount(t, x, 1, 0) // Alerts are gone.
// Add another alert.
_, alertErr := x.Insert(createAlert(t, 1, 1, 2, "alert2", []string{"notifier1"}))
require.NoError(t, alertErr)
// Enable UA.
// This run should remigrate org, new alert is migrated.
service.cfg.UnifiedAlerting.Enabled = pointer(true)
require.NoError(t, service.Run(ctx))
checkAlertingType(t, ctx, service, migrationStore.UnifiedAlerting)
checkMigrationStatus(t, ctx, service, 1, true)
checkAlertRulesCount(t, x, 1, 2) // Now we have 2
checkAlertRulesCount(t, x, 1, 0)
})
}

2
pkg/setting/setting.go
View File

@ -416,6 +416,7 @@ type Cfg struct {
StackID string
Slug string
// Deprecated
ForceMigration bool
// Analytics
@ -1061,6 +1062,7 @@ func (cfg *Cfg) Load(args CommandLineArgs) error {
cfg.Env = Env
cfg.StackID = valueAsString(iniFile.Section("environment"), "stack_id", "")
cfg.Slug = valueAsString(iniFile.Section("environment"), "stack_slug", "")
//nolint:staticcheck
cfg.ForceMigration = iniFile.Section("").Key("force_migration").MustBool(false)
InstanceName = valueAsString(iniFile.Section(""), "instance_name", "unknown_instance_name")
plugins := valueAsString(iniFile.Section("paths"), "plugins", "")

12
pkg/setting/setting_unified_alerting.go
View File

@ -96,6 +96,7 @@ type UnifiedAlertingSettings struct {
ReservedLabels UnifiedAlertingReservedLabelSettings
StateHistory UnifiedAlertingStateHistorySettings
RemoteAlertmanager RemoteAlertmanagerSettings
Upgrade UnifiedAlertingUpgradeSettings
// MaxStateSaveConcurrency controls the number of goroutines (per rule) that can save alert state in parallel.
MaxStateSaveConcurrency int
}
@ -136,6 +137,11 @@ type UnifiedAlertingStateHistorySettings struct {
ExternalLabels map[string]string
}
type UnifiedAlertingUpgradeSettings struct {
// CleanUpgrade controls whether the upgrade process should clean up UA data when upgrading from legacy alerting.
CleanUpgrade bool
}
// IsEnabled returns true if UnifiedAlertingSettings.Enabled is either nil or true.
// It hides the implementation details of the Enabled and simplifies its usage.
func (u *UnifiedAlertingSettings) IsEnabled() bool {
@ -399,6 +405,12 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
uaCfg.MaxStateSaveConcurrency = ua.Key("max_state_save_concurrency").MustInt(1)
upgrade := iniFile.Section("unified_alerting.upgrade")
uaCfgUpgrade := UnifiedAlertingUpgradeSettings{
CleanUpgrade: upgrade.Key("clean_upgrade").MustBool(false),
}
uaCfg.Upgrade = uaCfgUpgrade
cfg.UnifiedAlerting = uaCfg
return nil
}