From d724d463b13a727cceb2748e95daa21f0dc80fab Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 10 Sep 2024 20:07:06 +0300 Subject: [PATCH] DualWriter: Add mode 5 that always uses storage (#93169) --- pkg/apiserver/rest/dualwriter.go | 10 ++++++++-- pkg/services/apiserver/builder/helper.go | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/pkg/apiserver/rest/dualwriter.go b/pkg/apiserver/rest/dualwriter.go index 2bf3c59cfd3..09eafb93da6 100644 --- a/pkg/apiserver/rest/dualwriter.go +++ b/pkg/apiserver/rest/dualwriter.go @@ -96,11 +96,17 @@ const ( // reading and writing to Storage on a best effort basis for the sake of collecting metrics. Mode1 // Mode2 is the dual writing mode that represents writing to LegacyStorage and Storage and reading from LegacyStorage. + // The objects written to storage will include any labels and annotations. + // When reading values, the results will be from Storage when they exist, otherwise from legacy storage Mode2 // Mode3 represents writing to LegacyStorage and Storage and reading from Storage. + // NOTE: Requesting mode3 will only happen when after a background sync job succeeds Mode3 // Mode4 represents writing and reading from Storage. + // NOTE: Requesting mode4 will only happen when after a background sync job succeeds Mode4 + // Mode5 uses storage regardless of the background sync state + Mode5 ) // TODO: make this function private as there should only be one public way of setting the dual writing mode @@ -126,7 +132,7 @@ func NewDualWriter( case Mode3: // write to both, read from storage only return newDualWriterMode3(legacy, storage, metrics, resource) - case Mode4: + case Mode4, Mode5: return storage default: return newDualWriterMode1(legacy, storage, metrics, resource) @@ -199,7 +205,7 @@ func SetDualWritingMode( if !valid && ok { // Only log if "ok" because initially all instances will have mode unset for playlists. - klog.Info("invalid dual writing mode for playlists mode:", m) + klog.Infof("invalid dual writing mode for %s mode: %v", entity, m) } if !valid || !ok { diff --git a/pkg/services/apiserver/builder/helper.go b/pkg/services/apiserver/builder/helper.go index 1b71e753ad3..bcee690864f 100644 --- a/pkg/services/apiserver/builder/helper.go +++ b/pkg/services/apiserver/builder/helper.go @@ -21,6 +21,7 @@ import ( utilversion "k8s.io/apiserver/pkg/util/version" k8sscheme "k8s.io/client-go/kubernetes/scheme" k8stracing "k8s.io/component-base/tracing" + "k8s.io/klog/v2" "k8s.io/kube-openapi/pkg/common" "github.com/grafana/grafana/pkg/apiserver/endpoints/filters" @@ -174,25 +175,36 @@ func InstallAPIs( mode = resourceConfig.DualWriterMode } + // Force using storage only -- regardless of internal synchronization state + if mode == grafanarest.Mode5 { + return storage, nil + } + // TODO: inherited context from main Grafana process ctx := context.Background() // Moving from one version to the next can only happen after the previous step has // successfully synchronized. - currentMode, err := grafanarest.SetDualWritingMode(ctx, kvStore, legacy, storage, key, mode, reg, serverLock, getRequestInfo(gr, namespaceMapper)) + requestInfo := getRequestInfo(gr, namespaceMapper) + currentMode, err := grafanarest.SetDualWritingMode(ctx, kvStore, legacy, storage, key, mode, reg, serverLock, requestInfo) if err != nil { return nil, err } switch currentMode { case grafanarest.Mode0: return legacy, nil - case grafanarest.Mode4: + case grafanarest.Mode4, grafanarest.Mode5: return storage, nil default: } if storageOpts.DualWriterDataSyncJobEnabled[key] { - grafanarest.StartPeriodicDataSyncer(ctx, currentMode, legacy, storage, key, reg, serverLock, getRequestInfo(gr, namespaceMapper)) + grafanarest.StartPeriodicDataSyncer(ctx, currentMode, legacy, storage, key, reg, serverLock, requestInfo) + } + + // when unable to use + if currentMode != mode { + klog.Warningf("Requested DualWrite mode: %d, but using %d for %+v", mode, currentMode, gr) } return grafanarest.NewDualWriter(currentMode, legacy, storage, reg, key), nil }