IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Storage: Customise setting dual writing modes (#87668)

Browse Source 
* Add feature toggles for mode 2 and 3 playlist dual writing
* Make current mode customised based on kind
* Check feature flags when initialising dual writer
* Fix linting
* Refactor NewDualWriter
This commit is contained in:
Arati R
2024-05-14 14:11:56 +02:00
committed by GitHub
parent d8904f3ca4
commit 6836bfe1ea
13 changed files with 92 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.golangci.toml
View File

@@ -76,6 +76,7 @@ allow = [
"github.com/grafana/grafana/pkg/apimachinery",
"github.com/grafana/grafana/pkg/apiserver",
"github.com/grafana/grafana/pkg/services/apiserver/utils",
"github.com/grafana/grafana/pkg/services/featuremgmt",
]
deny = [
{ pkg = "github.com/grafana/grafana/pkg", desc = "apiserver is not allowed to import grafana core" }

2
docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md
View File

@@ -120,6 +120,8 @@ Experimental features might be changed or removed without prior notice.
| `disableSecretsCompatibility` | Disable duplicated secret storage in legacy tables |
| `logRequestsInstrumentedAsUnknown` | Logs the path for requests that are instrumented as unknown |
| `unifiedStorage` | SQL-based k8s storage |
| `dualWritePlaylistsMode2` | Enables dual writing of playlists to both legacy and k8s storage in mode 2 |
| `dualWritePlaylistsMode3` | Enables dual writing of playlists to both legacy and k8s storage in mode 3 |
| `showDashboardValidationWarnings` | Show warnings when dashboards do not validate against the schema |
| `mysqlAnsiQuotes` | Use double quotes to escape keyword in a MySQL query |
| `alertingBacktesting` | Rule backtesting API for alerting |

2
packages/grafana-data/src/types/featureToggles.gen.ts
View File

@@ -47,6 +47,8 @@ export interface FeatureToggles {
returnToPrevious?: boolean;
grpcServer?: boolean;
unifiedStorage?: boolean;
dualWritePlaylistsMode2?: boolean;
dualWritePlaylistsMode3?: boolean;
cloudWatchCrossAccountQuerying?: boolean;
showDashboardValidationWarnings?: boolean;
mysqlAnsiQuotes?: boolean;

43
pkg/apiserver/rest/dualwriter.go
View File

@@ -80,14 +80,24 @@ const (
Mode4
)
var CurrentMode = Mode2
//TODO: make CurrentMode customisable and specific to each entity
// change DualWriter signature to get the current mode as an argument
// NewDualWriter returns a new DualWriter.
func NewDualWriter(legacy LegacyStorage, storage Storage) DualWriter {
return selectDualWriter(CurrentMode, legacy, storage)
func NewDualWriter(mode DualWriterMode, legacy LegacyStorage, storage Storage) DualWriter {
switch mode {
case Mode1:
// read and write only from legacy storage
return NewDualWriterMode1(legacy, storage)
case Mode2:
// write to both, read from storage but use legacy as backup
return NewDualWriterMode2(legacy, storage)
case Mode3:
// write to both, read from storage only
return NewDualWriterMode3(legacy, storage)
case Mode4:
// read and write only from storage
return NewDualWriterMode4(legacy, storage)
default:
return NewDualWriterMode1(legacy, storage)
}
}
type updateWrapper struct {
@@ -107,22 +117,3 @@ func (u *updateWrapper) Preconditions() *metav1.Preconditions {
func (u *updateWrapper) UpdatedObject(ctx context.Context, oldObj runtime.Object) (newObj runtime.Object, err error) {
return u.updated, nil
}
func selectDualWriter(mode DualWriterMode, legacy LegacyStorage, storage Storage) DualWriter {
switch mode {
case Mode1:
// read and write only from legacy storage
return NewDualWriterMode1(legacy, storage)
case Mode2:
// write to both, read from storage but use legacy as backup
return NewDualWriterMode2(legacy, storage)
case Mode3:
// write to both, read from storage only
return NewDualWriterMode3(legacy, storage)
case Mode4:
// read and write only from storage
return NewDualWriterMode4(legacy, storage)
default:
return NewDualWriterMode1(legacy, storage)
}
}

12
pkg/apiserver/rest/dualwriter_mode1_test.go
View File

@@ -65,7 +65,7 @@ func TestMode1_Create(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := selectDualWriter(Mode1, ls, us)
dw := NewDualWriter(Mode1, ls, us)
obj, err := dw.Create(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.CreateOptions{})
@@ -126,7 +126,7 @@ func TestMode1_Get(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := selectDualWriter(Mode1, ls, us)
dw := NewDualWriter(Mode1, ls, us)
obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{})
@@ -175,7 +175,7 @@ func TestMode1_List(t *testing.T) {
tt.setupStorageFn(m)
}
dw := selectDualWriter(Mode1, ls, us)
dw := NewDualWriter(Mode1, ls, us)
_, err := dw.List(context.Background(), &metainternalversion.ListOptions{})
@@ -228,7 +228,7 @@ func TestMode1_Delete(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := selectDualWriter(Mode1, ls, us)
dw := NewDualWriter(Mode1, ls, us)
obj, _, err := dw.Delete(context.Background(), tt.input, func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{})
@@ -285,7 +285,7 @@ func TestMode1_DeleteCollection(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := selectDualWriter(Mode1, ls, us)
dw := NewDualWriter(Mode1, ls, us)
obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, tt.input, &metainternalversion.ListOptions{})
@@ -359,7 +359,7 @@ func TestMode1_Update(t *testing.T) {
tt.setupGetFn(m, tt.input)
}
dw := selectDualWriter(Mode1, ls, us)
dw := NewDualWriter(Mode1, ls, us)
obj, _, err := dw.Update(context.Background(), tt.input, UpdatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})

12
pkg/apiserver/rest/dualwriter_mode2_test.go
View File

@@ -64,7 +64,7 @@ func TestMode2_Create(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := selectDualWriter(Mode2, ls, us)
dw := NewDualWriter(Mode2, ls, us)
obj, err := dw.Create(context.Background(), tt.input, createFn, &metav1.CreateOptions{})
@@ -138,7 +138,7 @@ func TestMode2_Get(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := selectDualWriter(Mode2, ls, us)
dw := NewDualWriter(Mode2, ls, us)
obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{})
@@ -189,7 +189,7 @@ func TestMode2_List(t *testing.T) {
tt.setupStorageFn(m)
}
dw := selectDualWriter(Mode2, ls, us)
dw := NewDualWriter(Mode2, ls, us)
obj, err := dw.List(context.Background(), &metainternalversion.ListOptions{})
@@ -281,7 +281,7 @@ func TestMode2_Delete(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := selectDualWriter(Mode2, ls, us)
dw := NewDualWriter(Mode2, ls, us)
obj, _, err := dw.Delete(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.DeleteOptions{})
@@ -351,7 +351,7 @@ func TestMode2_DeleteCollection(t *testing.T) {
tt.setupStorageFn(m)
}
dw := selectDualWriter(Mode2, ls, us)
dw := NewDualWriter(Mode2, ls, us)
obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{TypeMeta: metav1.TypeMeta{Kind: tt.input}}, &metainternalversion.ListOptions{})
@@ -458,7 +458,7 @@ func TestMode2_Update(t *testing.T) {
tt.setupStorageFn(m, tt.input)
}
dw := selectDualWriter(Mode2, ls, us)
dw := NewDualWriter(Mode2, ls, us)
obj, _, err := dw.Update(context.Background(), tt.input, UpdatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})

2
pkg/registry/apis/dashboard/register.go
View File

@@ -178,7 +178,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo(
if err := store.CompleteWithOptions(options); err != nil {
return nil, err
}
storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(legacyStore, store)
storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(grafanarest.Mode1, legacyStore, store)
}
// Summary

2
pkg/registry/apis/folders/register.go
View File

@@ -139,7 +139,7 @@ func (b *FolderAPIBuilder) GetAPIGroupInfo(
if err != nil {
return nil, err
}
storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(legacyStore, store)
storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(grafanarest.Mode1, legacyStore, store)
}
apiGroupInfo.VersionedResourcesStorageMap[v0alpha1.VERSION] = storage

11
pkg/registry/apis/playlist/register.go
View File

@@ -19,6 +19,7 @@ import (
grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
"github.com/grafana/grafana/pkg/services/apiserver/utils"
"github.com/grafana/grafana/pkg/services/featuremgmt"
playlistsvc "github.com/grafana/grafana/pkg/services/playlist"
"github.com/grafana/grafana/pkg/setting"
)
@@ -30,16 +31,19 @@ type PlaylistAPIBuilder struct {
service playlistsvc.Service
namespacer request.NamespaceMapper
gv schema.GroupVersion
features featuremgmt.FeatureToggles
}
func RegisterAPIService(p playlistsvc.Service,
apiregistration builder.APIRegistrar,
cfg *setting.Cfg,
features featuremgmt.FeatureToggles,
) *PlaylistAPIBuilder {
builder := &PlaylistAPIBuilder{
service: p,
namespacer: request.GetNamespaceMapper(cfg),
gv: playlist.PlaylistResourceInfo.GroupVersion(),
features: features,
}
apiregistration.RegisterAPI(builder)
return builder
@@ -118,7 +122,12 @@ func (b *PlaylistAPIBuilder) GetAPIGroupInfo(
if err != nil {
return nil, err
}
storage[resource.StoragePath()] = grafanarest.NewDualWriterMode2(legacyStore, store)
mode := grafanarest.Mode1
if b.features.IsEnabledGlobally(featuremgmt.FlagDualWritePlaylistsMode2) {
mode = grafanarest.Mode2
}
storage[resource.StoragePath()] = grafanarest.NewDualWriter(mode, legacyStore, store)
}
apiGroupInfo.VersionedResourcesStorageMap[playlist.VERSION] = storage

12
pkg/services/featuremgmt/registry.go
View File

@@ -228,6 +228,18 @@ var (
RequiresRestart: true, // new SQL tables created
Owner: grafanaAppPlatformSquad,
},
{
Name: "dualWritePlaylistsMode2",
Description: "Enables dual writing of playlists to both legacy and k8s storage in mode 2",
Stage: FeatureStageExperimental,
Owner: grafanaSearchAndStorageSquad,
},
{
Name: "dualWritePlaylistsMode3",
Description: "Enables dual writing of playlists to both legacy and k8s storage in mode 3",
Stage: FeatureStageExperimental,
Owner: grafanaSearchAndStorageSquad,
},
{
Name: "cloudWatchCrossAccountQuerying",
Description: "Enables cross-account querying in CloudWatch datasources",

2
pkg/services/featuremgmt/toggles_gen.csv
View File

@@ -28,6 +28,8 @@ topnav,deprecated,@grafana/grafana-frontend-platform,false,false,false
returnToPrevious,GA,@grafana/grafana-frontend-platform,false,false,true
grpcServer,preview,@grafana/grafana-app-platform-squad,false,false,false
unifiedStorage,experimental,@grafana/grafana-app-platform-squad,false,true,false
dualWritePlaylistsMode2,experimental,@grafana/search-and-storage,false,false,false
dualWritePlaylistsMode3,experimental,@grafana/search-and-storage,false,false,false
cloudWatchCrossAccountQuerying,GA,@grafana/aws-datasources,false,false,false
showDashboardValidationWarnings,experimental,@grafana/dashboards-squad,false,false,false
mysqlAnsiQuotes,experimental,@grafana/search-and-storage,false,false,false
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
28 returnToPrevious GA @grafana/grafana-frontend-platform false false true
29 grpcServer preview @grafana/grafana-app-platform-squad false false false
30 unifiedStorage experimental @grafana/grafana-app-platform-squad false true false
31 dualWritePlaylistsMode2 experimental @grafana/search-and-storage false false false
32 dualWritePlaylistsMode3 experimental @grafana/search-and-storage false false false
33 cloudWatchCrossAccountQuerying GA @grafana/aws-datasources false false false
34 showDashboardValidationWarnings experimental @grafana/dashboards-squad false false false
35 mysqlAnsiQuotes experimental @grafana/search-and-storage false false false

8
pkg/services/featuremgmt/toggles_gen.go
View File

@@ -123,6 +123,14 @@ const (
// SQL-based k8s storage
FlagUnifiedStorage = "unifiedStorage"
// FlagDualWritePlaylistsMode2
// Enables dual writing of playlists to both legacy and k8s storage in mode 2
FlagDualWritePlaylistsMode2 = "dualWritePlaylistsMode2"
// FlagDualWritePlaylistsMode3
// Enables dual writing of playlists to both legacy and k8s storage in mode 3
FlagDualWritePlaylistsMode3 = "dualWritePlaylistsMode3"
// FlagCloudWatchCrossAccountQuerying
// Enables cross-account querying in CloudWatch datasources
FlagCloudWatchCrossAccountQuerying = "cloudWatchCrossAccountQuerying"

24
pkg/services/featuremgmt/toggles_gen.json
View File

@@ -2179,6 +2179,30 @@
"stage": "experimental",
"codeowner": "@grafana/grafana-frontend-platform"
}
},
{
"metadata": {
"name": "dualWritePlaylistsMode2",
"resourceVersion": "1715369873132",
"creationTimestamp": "2024-05-10T19:37:53Z"
},
"spec": {
"description": "Enables dual writing of playlists to both legacy and k8s storage in mode 2",
"stage": "experimental",
"codeowner": "@grafana/search-and-storage"
}
},
{
"metadata": {
"name": "dualWritePlaylistsMode3",
"resourceVersion": "1715369873132",
"creationTimestamp": "2024-05-10T19:37:53Z"
},
"spec": {
"description": "Enables dual writing of playlists to both legacy and k8s storage in mode 3",
"stage": "experimental",
"codeowner": "@grafana/search-and-storage"
}
}
]
}