diff --git a/state/remote/state.go b/state/remote/state.go index 5ead38e82b..e73fbe8f58 100644 --- a/state/remote/state.go +++ b/state/remote/state.go @@ -28,6 +28,7 @@ type State struct { } var _ statemgr.Full = (*State)(nil) +var _ statemgr.Migrator = (*State)(nil) // statemgr.Reader impl. func (s *State) State() *states.State { @@ -37,6 +38,14 @@ func (s *State) State() *states.State { return s.state.DeepCopy() } +// StateForMigration is part of our implementation of statemgr.Migrator. +func (s *State) StateForMigration() *statefile.File { + s.mu.Lock() + defer s.mu.Unlock() + + return statefile.New(s.state.DeepCopy(), s.lineage, s.serial) +} + // statemgr.Writer impl. func (s *State) WriteState(state *states.State) error { s.mu.Lock() @@ -50,6 +59,28 @@ func (s *State) WriteState(state *states.State) error { return nil } +// WriteStateForMigration is part of our implementation of statemgr.Migrator. +func (s *State) WriteStateForMigration(f *statefile.File, force bool) error { + s.mu.Lock() + defer s.mu.Unlock() + + checkFile := statefile.New(s.state, s.lineage, s.serial) + if !force { + if err := statemgr.CheckValidImport(f, checkFile); err != nil { + return err + } + } + + // We create a deep copy of the state here, because the caller also has + // a reference to the given object and can potentially go on to mutate + // it after we return, but we want the snapshot at this point in time. + s.state = f.State.DeepCopy() + s.lineage = f.Lineage + s.serial = f.Serial + + return nil +} + // statemgr.Refresher impl. func (s *State) RefreshState() error { s.mu.Lock() diff --git a/states/statemgr/filesystem.go b/states/statemgr/filesystem.go index c9011162e6..740c23e750 100644 --- a/states/statemgr/filesystem.go +++ b/states/statemgr/filesystem.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "io/ioutil" + "log" "os" "path/filepath" "sync" @@ -62,6 +63,7 @@ type Filesystem struct { var ( _ Full = (*Filesystem)(nil) _ PersistentMeta = (*Filesystem)(nil) + _ Migrator = (*Filesystem)(nil) ) // NewFilesystem creates a filesystem-based state manager that reads and writes @@ -121,9 +123,6 @@ func (s *Filesystem) State() *states.State { // WriteState is an incorrect implementation of Writer that actually also // persists. -// WriteState for LocalState always persists the state as well. -// -// StateWriter impl. func (s *Filesystem) WriteState(state *states.State) error { // TODO: this should use a more robust method of writing state, by first // writing to a temp file on the same filesystem, and renaming the file over @@ -137,7 +136,10 @@ func (s *Filesystem) WriteState(state *states.State) error { } defer s.mutex()() + return s.writeState(state, nil) +} +func (s *Filesystem) writeState(state *states.State, meta *SnapshotMeta) error { // We'll try to write our backup first, so we can be sure we've created // it successfully before clobbering the original file it came from. if !s.writtenBackup && s.backupFile != nil && s.backupPath != "" && !statefile.StatesMarshalEqual(state, s.backupFile.State) { @@ -180,8 +182,14 @@ func (s *Filesystem) WriteState(state *states.State) error { return nil } - if s.readFile == nil || !statefile.StatesMarshalEqual(s.file.State, s.readFile.State) { - s.file.Serial++ + if meta == nil { + if s.readFile == nil || !statefile.StatesMarshalEqual(s.file.State, s.readFile.State) { + s.file.Serial++ + } + } else { + // Force new metadata + s.file.Lineage = meta.Lineage + s.file.Serial = meta.Serial } if err := statefile.Write(s.file, s.stateFileOut); err != nil { @@ -345,6 +353,51 @@ func (s *Filesystem) StateSnapshotMeta() SnapshotMeta { } } +// StateForMigration is part of our implementation of Migrator. +func (s *Filesystem) StateForMigration() *statefile.File { + return s.file.DeepCopy() +} + +// WriteStateForMigration is part of our implementation of Migrator. +func (s *Filesystem) WriteStateForMigration(f *statefile.File, force bool) error { + if s.readFile == nil { + err := s.RefreshState() + if err != nil { + return err + } + } + defer s.mutex()() + + if !force { + err := CheckValidImport(f, s.readFile) + if err != nil { + return err + } + } + + if s.readFile != nil { + log.Printf( + "[TRACE] statemgr.Filesystem: Importing snapshot with lineage %q serial %d over snapshot with lineage %q serial %d at %s", + f.Lineage, f.Serial, + s.readFile.Lineage, s.readFile.Serial, + s.path, + ) + } else { + log.Printf( + "[TRACE] statemgr.Filesystem: Importing snapshot with lineage %q serial %d as the initial state snapshot at %s", + f.Lineage, f.Serial, + s.path, + ) + } + + err := s.writeState(f.State, &SnapshotMeta{Lineage: f.Lineage, Serial: f.Serial}) + if err != nil { + return err + } + + return nil +} + // Open the state file, creating the directories and file as needed. func (s *Filesystem) createStateFiles() error { diff --git a/states/statemgr/migrate.go b/states/statemgr/migrate.go new file mode 100644 index 0000000000..8e263e07b9 --- /dev/null +++ b/states/statemgr/migrate.go @@ -0,0 +1,212 @@ +package statemgr + +import ( + "fmt" + + "github.com/hashicorp/terraform/states/statefile" +) + +// Migrator is an optional interface implemented by state managers that +// are capable of direct migration of state snapshots with their associated +// metadata unchanged. +// +// This interface is used when available by function Migrate. See that +// function for more information on how it is used. +type Migrator interface { + PersistentMeta + + // StateForMigration returns a full statefile representing the latest + // snapshot (as would be returned by Reader.State) and the associated + // snapshot metadata (as would be returned by + // PersistentMeta.StateSnapshotMeta). + // + // Just as with Reader.State, this must not fail. + StateForMigration() *statefile.File + + // WriteStateForMigration accepts a full statefile including associated + // snapshot metadata, and atomically updates the stored file (as with + // Writer.WriteState) and the metadata. + // + // If "force" is not set, the manager must call CheckValidImport with + // the given file and the current file and complete the update only if + // that function returns nil. If force is set this may override such + // checks, but some backends do not support forcing and so will act + // as if force is always true. + WriteStateForMigration(f *statefile.File, force bool) error +} + +// Migrate writes the latest transient state snapshot from src into dest, +// preserving snapshot metadata (serial and lineage) where possible. +// +// If both managers implement the optional interface Migrator then it will +// be used to copy the snapshot and its associated metadata. Otherwise, +// the normal Reader and Writer interfaces will be used instead. +// +// If the destination manager refuses the new state or fails to write it then +// its error is returned directly. +// +// For state managers that also implement Persistent, it is the caller's +// responsibility to persist the newly-written state after a successful result, +// just as with calls to Writer.WriteState. +// +// This function doesn't do any locking of its own, so if the state managers +// also implement Locker the caller should hold a lock on both managers +// for the duration of this call. +func Migrate(dst, src Transient) error { + if dstM, ok := dst.(Migrator); ok { + if srcM, ok := src.(Migrator); ok { + // Full-fidelity migration, them. + s := srcM.StateForMigration() + return dstM.WriteStateForMigration(s, true) + } + } + + // Managers to not support full-fidelity migration, so migration will not + // preserve serial/lineage. + s := src.State() + return dst.WriteState(s) +} + +// Import loads the given state snapshot into the given manager, preserving +// its metadata (serial and lineage) if the target manager supports metadata. +// +// A state manager must implement the optional interface Migrator to get +// access to the full metadata. +// +// Unless "force" is true, Import will check first that the metadata given +// in the file matches the current snapshot metadata for the manager, if the +// manager supports metadata. Some managers do not support forcing, so a +// write with an unsuitable lineage or serial may still be rejected even if +// "force" is set. "force" has no effect for managers that do not support +// snapshot metadata. +// +// For state managers that also implement Persistent, it is the caller's +// responsibility to persist the newly-written state after a successful result, +// just as with calls to Writer.WriteState. +// +// This function doesn't do any locking of its own, so if the state manager +// also implements Locker the caller should hold a lock on it for the +// duration of this call. +func Import(f *statefile.File, mgr Transient, force bool) error { + if mgrM, ok := mgr.(Migrator); ok { + return mgrM.WriteStateForMigration(f, force) + } + + // For managers that don't implement Migrator, this is just a normal write + // of the state contained in the given file. + return mgr.WriteState(f.State) +} + +// Export retrieves the latest state snapshot from the given manager, including +// its metadata (serial and lineage) where possible. +// +// A state manager must also implement either Migrator or PersistentMeta +// for the metadata to be included. Otherwise, the relevant fields will have +// zero value in the returned object. +// +// For state managers that also implement Persistent, it is the caller's +// responsibility to refresh from persistent storage first if needed. +// +// This function doesn't do any locking of its own, so if the state manager +// also implements Locker the caller should hold a lock on it for the +// duration of this call. +func Export(mgr Reader) *statefile.File { + switch mgrT := mgr.(type) { + case Migrator: + return mgrT.StateForMigration() + case PersistentMeta: + s := mgr.State() + meta := mgrT.StateSnapshotMeta() + return statefile.New(s, meta.Lineage, meta.Serial) + default: + s := mgr.State() + return statefile.New(s, "", 0) + } +} + +// SnapshotMetaRel describes a relationship between two SnapshotMeta values, +// returned from the SnapshotMeta.Compare method where the "first" value +// is the receiver of that method and the "second" is the given argument. +type SnapshotMetaRel rune + +//go:generate stringer -type=SnapshotMetaRel + +const ( + // SnapshotOlder indicates that two snapshots have a common lineage and + // that the first has a lower serial value. + SnapshotOlder SnapshotMetaRel = '<' + + // SnapshotNewer indicates that two snapshots have a common lineage and + // that the first has a higher serial value. + SnapshotNewer SnapshotMetaRel = '>' + + // SnapshotEqual indicates that two snapshots have a common lineage and + // the same serial value. + SnapshotEqual SnapshotMetaRel = '=' + + // SnapshotUnrelated indicates that two snapshots have different lineage + // and thus cannot be meaningfully compared. + SnapshotUnrelated SnapshotMetaRel = '!' + + // SnapshotLegacy indicates that one or both of the snapshots + // does not have a lineage at all, and thus no comparison is possible. + SnapshotLegacy SnapshotMetaRel = '?' +) + +// Compare determines the relationship, if any, between the given existing +// SnapshotMeta and the potential "new" SnapshotMeta that is the receiver. +func (m SnapshotMeta) Compare(existing SnapshotMeta) SnapshotMetaRel { + switch { + case m.Lineage == "" || existing.Lineage == "": + return SnapshotLegacy + case m.Lineage != existing.Lineage: + return SnapshotUnrelated + case m.Serial > existing.Serial: + return SnapshotNewer + case m.Serial < existing.Serial: + return SnapshotOlder + default: + // both serials are equal, by elimination + return SnapshotEqual + } +} + +// CheckValidImport returns nil if the "new" snapshot can be imported as a +// successor of the "existing" snapshot without forcing. +// +// If not, an error is returned describing why. +func CheckValidImport(newFile, existingFile *statefile.File) error { + if existingFile == nil || existingFile.State.Empty() { + // It's always okay to overwrite an empty state, regardless of + // its lineage/serial. + return nil + } + new := SnapshotMeta{ + Lineage: newFile.Lineage, + Serial: newFile.Serial, + } + existing := SnapshotMeta{ + Lineage: existingFile.Lineage, + Serial: existingFile.Serial, + } + rel := new.Compare(existing) + switch rel { + case SnapshotNewer: + return nil // a newer snapshot is fine + case SnapshotLegacy: + return nil // anything goes for a legacy state + case SnapshotUnrelated: + return fmt.Errorf("cannot import state with lineage %q over unrelated state with lineage %q", new.Lineage, existing.Lineage) + case SnapshotEqual: + if statefile.StatesMarshalEqual(newFile.State, existingFile.State) { + // If lineage, serial, and state all match then this is fine. + return nil + } + return fmt.Errorf("cannot overwrite existing state with serial %d with a different state that has the same serial", new.Serial) + case SnapshotOlder: + return fmt.Errorf("cannot import state with serial %d over newer state with serial %d", new.Serial, existing.Serial) + default: + // Should never happen, but we'll check to make sure for safety + return fmt.Errorf("unsupported state snapshot relationship %s", rel) + } +} diff --git a/states/statemgr/migrate_test.go b/states/statemgr/migrate_test.go new file mode 100644 index 0000000000..0cf2113a25 --- /dev/null +++ b/states/statemgr/migrate_test.go @@ -0,0 +1,102 @@ +package statemgr + +import ( + "testing" + + "github.com/zclconf/go-cty/cty" + + "github.com/hashicorp/terraform/addrs" + "github.com/hashicorp/terraform/states" + "github.com/hashicorp/terraform/states/statefile" +) + +func TestCheckValidImport(t *testing.T) { + barState := states.BuildState(func(s *states.SyncState) { + s.SetOutputValue( + addrs.OutputValue{Name: "foo"}.Absolute(addrs.RootModuleInstance), + cty.StringVal("bar"), false, + ) + }) + notBarState := states.BuildState(func(s *states.SyncState) { + s.SetOutputValue( + addrs.OutputValue{Name: "foo"}.Absolute(addrs.RootModuleInstance), + cty.StringVal("not bar"), false, + ) + }) + emptyState := states.NewState() + + tests := map[string]struct { + New *statefile.File + Existing *statefile.File + WantErr string + }{ + "exact match": { + New: statefile.New(barState, "lineage", 1), + Existing: statefile.New(barState, "lineage", 1), + WantErr: ``, + }, + "overwrite unrelated empty state": { + New: statefile.New(barState, "lineage1", 1), + Existing: statefile.New(emptyState, "lineage2", 1), + WantErr: ``, + }, + "different state with same serial": { + New: statefile.New(barState, "lineage", 1), + Existing: statefile.New(notBarState, "lineage", 1), + WantErr: `cannot overwrite existing state with serial 1 with a different state that has the same serial`, + }, + "different state with newer serial": { + New: statefile.New(barState, "lineage", 2), + Existing: statefile.New(notBarState, "lineage", 1), + WantErr: ``, + }, + "different state with older serial": { + New: statefile.New(barState, "lineage", 1), + Existing: statefile.New(notBarState, "lineage", 2), + WantErr: `cannot import state with serial 1 over newer state with serial 2`, + }, + "different lineage with same serial": { + New: statefile.New(barState, "lineage1", 2), + Existing: statefile.New(notBarState, "lineage2", 2), + WantErr: `cannot import state with lineage "lineage1" over unrelated state with lineage "lineage2"`, + }, + "different lineage with different serial": { + New: statefile.New(barState, "lineage1", 3), + Existing: statefile.New(notBarState, "lineage2", 2), + WantErr: `cannot import state with lineage "lineage1" over unrelated state with lineage "lineage2"`, + }, + "new state is legacy": { + New: statefile.New(barState, "", 2), + Existing: statefile.New(notBarState, "lineage", 2), + WantErr: ``, + }, + "old state is legacy": { + New: statefile.New(barState, "lineage", 2), + Existing: statefile.New(notBarState, "", 2), + WantErr: ``, + }, + "both states are legacy": { + New: statefile.New(barState, "", 2), + Existing: statefile.New(notBarState, "", 2), + WantErr: ``, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + gotErr := CheckValidImport(test.New, test.Existing) + + if test.WantErr == "" { + if gotErr != nil { + t.Errorf("unexpected error: %s", gotErr) + } + } else { + if gotErr == nil { + t.Errorf("succeeded, but want error: %s", test.WantErr) + } else if got, want := gotErr.Error(), test.WantErr; got != want { + t.Errorf("wrong error\ngot: %s\nwant: %s", got, want) + } + } + }) + } +} diff --git a/states/statemgr/snapshotmetarel_string.go b/states/statemgr/snapshotmetarel_string.go new file mode 100644 index 0000000000..28e1b966f9 --- /dev/null +++ b/states/statemgr/snapshotmetarel_string.go @@ -0,0 +1,26 @@ +// Code generated by "stringer -type=SnapshotMetaRel"; DO NOT EDIT. + +package statemgr + +import "strconv" + +const ( + _SnapshotMetaRel_name_0 = "SnapshotUnrelated" + _SnapshotMetaRel_name_1 = "SnapshotOlderSnapshotEqualSnapshotNewerSnapshotLegacy" +) + +var ( + _SnapshotMetaRel_index_1 = [...]uint8{0, 13, 26, 39, 53} +) + +func (i SnapshotMetaRel) String() string { + switch { + case i == 33: + return _SnapshotMetaRel_name_0 + case 60 <= i && i <= 63: + i -= 60 + return _SnapshotMetaRel_name_1[_SnapshotMetaRel_index_1[i]:_SnapshotMetaRel_index_1[i+1]] + default: + return "SnapshotMetaRel(" + strconv.FormatInt(int64(i), 10) + ")" + } +}