From dc40f044f0f8e8be76b398b4086bcf70a9a38f5b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 29 Jun 2015 12:24:13 -0700 Subject: [PATCH 1/7] command/push: prefer Atlas over local, add -set flag --- command/push.go | 16 ++++- command/push_test.go | 152 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 165 insertions(+), 3 deletions(-) diff --git a/command/push.go b/command/push.go index e3a52e63d3..3f568bd393 100644 --- a/command/push.go +++ b/command/push.go @@ -24,6 +24,7 @@ func (c *PushCommand) Run(args []string) int { var atlasAddress, atlasToken string var archiveVCS, moduleUpload bool var name string + var set []string args = c.Meta.process(args, true) cmdFlags := c.Meta.flagSet("push") cmdFlags.StringVar(&atlasAddress, "atlas-address", "", "") @@ -32,11 +33,18 @@ func (c *PushCommand) Run(args []string) int { cmdFlags.BoolVar(&moduleUpload, "upload-modules", true, "") cmdFlags.StringVar(&name, "name", "", "") cmdFlags.BoolVar(&archiveVCS, "vcs", true, "") + cmdFlags.Var((*FlagStringSlice)(&set), "set", "") cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } if err := cmdFlags.Parse(args); err != nil { return 1 } + // Make a map of the set values + setMap := make(map[string]struct{}, len(set)) + for _, v := range set { + setMap[v] = struct{}{} + } + // The pwd is used for the configuration path if one is not given pwd, err := os.Getwd() if err != nil { @@ -132,10 +140,10 @@ func (c *PushCommand) Run(args []string) int { return 1 } for k, v := range vars { - // Local variables override remote ones - if _, exists := ctx.Variables()[k]; exists { + if _, ok := setMap[k]; ok { continue } + ctx.SetVariable(k, v) } @@ -211,6 +219,10 @@ Options: -token= Access token to use to upload. If blank or unspecified, the ATLAS_TOKEN environmental variable will be used. + -set=foo Variable keys that should overwrite values in Atlas. + Otherwise, variables already set in Atlas will overwrite + local values. This flag can be repeated. + -var 'foo=bar' Set a variable in the Terraform configuration. This flag can be set multiple times. diff --git a/command/push_test.go b/command/push_test.go index 96ed37d4e0..19bce23ecd 100644 --- a/command/push_test.go +++ b/command/push_test.go @@ -179,7 +179,9 @@ func TestPush_inputPartial(t *testing.T) { } } -func TestPush_inputTfvars(t *testing.T) { +// This tests that the push command will override Atlas variables +// if requested. +func TestPush_localOverride(t *testing.T) { // Disable test mode so input would be asked and setup the // input reader/writers. test = false @@ -219,6 +221,154 @@ func TestPush_inputTfvars(t *testing.T) { client: client, } + path := testFixturePath("push-tfvars") + args := []string{ + "-var-file", path + "/terraform.tfvars", + "-vcs=false", + "-set=foo", + path, + } + if code := c.Run(args); code != 0 { + t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String()) + } + + actual := testArchiveStr(t, archivePath) + expected := []string{ + ".terraform/", + ".terraform/terraform.tfstate", + "main.tf", + "terraform.tfvars", + } + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("bad: %#v", actual) + } + + if client.UpsertOptions.Name != "foo" { + t.Fatalf("bad: %#v", client.UpsertOptions) + } + + variables := map[string]string{ + "foo": "bar", + "bar": "foo", + } + if !reflect.DeepEqual(client.UpsertOptions.Variables, variables) { + t.Fatalf("bad: %#v", client.UpsertOptions) + } +} + +// This tests that the push command prefers Atlas variables over +// local ones. +func TestPush_preferAtlas(t *testing.T) { + // Disable test mode so input would be asked and setup the + // input reader/writers. + test = false + defer func() { test = true }() + defaultInputReader = bytes.NewBufferString("nope\n") + defaultInputWriter = new(bytes.Buffer) + + tmp, cwd := testCwd(t) + defer testFixCwd(t, tmp, cwd) + + // Create remote state file, this should be pulled + conf, srv := testRemoteState(t, testState(), 200) + defer srv.Close() + + // Persist local remote state + s := terraform.NewState() + s.Serial = 5 + s.Remote = conf + testStateFileRemote(t, s) + + // Path where the archive will be "uploaded" to + archivePath := testTempFile(t) + defer os.Remove(archivePath) + + client := &mockPushClient{File: archivePath} + // Provided vars should override existing ones + client.GetResult = map[string]string{ + "foo": "old", + } + ui := new(cli.MockUi) + c := &PushCommand{ + Meta: Meta{ + ContextOpts: testCtxConfig(testProvider()), + Ui: ui, + }, + + client: client, + } + + path := testFixturePath("push-tfvars") + args := []string{ + "-var-file", path + "/terraform.tfvars", + "-vcs=false", + path, + } + if code := c.Run(args); code != 0 { + t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String()) + } + + actual := testArchiveStr(t, archivePath) + expected := []string{ + ".terraform/", + ".terraform/terraform.tfstate", + "main.tf", + "terraform.tfvars", + } + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("bad: %#v", actual) + } + + if client.UpsertOptions.Name != "foo" { + t.Fatalf("bad: %#v", client.UpsertOptions) + } + + variables := map[string]string{ + "foo": "old", + "bar": "foo", + } + if !reflect.DeepEqual(client.UpsertOptions.Variables, variables) { + t.Fatalf("bad: %#v", client.UpsertOptions) + } +} + +// This tests that the push command will send the variables in tfvars +func TestPush_tfvars(t *testing.T) { + // Disable test mode so input would be asked and setup the + // input reader/writers. + test = false + defer func() { test = true }() + defaultInputReader = bytes.NewBufferString("nope\n") + defaultInputWriter = new(bytes.Buffer) + + tmp, cwd := testCwd(t) + defer testFixCwd(t, tmp, cwd) + + // Create remote state file, this should be pulled + conf, srv := testRemoteState(t, testState(), 200) + defer srv.Close() + + // Persist local remote state + s := terraform.NewState() + s.Serial = 5 + s.Remote = conf + testStateFileRemote(t, s) + + // Path where the archive will be "uploaded" to + archivePath := testTempFile(t) + defer os.Remove(archivePath) + + client := &mockPushClient{File: archivePath} + ui := new(cli.MockUi) + c := &PushCommand{ + Meta: Meta{ + ContextOpts: testCtxConfig(testProvider()), + Ui: ui, + }, + + client: client, + } + path := testFixturePath("push-tfvars") args := []string{ "-var-file", path + "/terraform.tfvars", From 8ee3281858843299d76a4e1da122f741f4771b0a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 29 Jun 2015 13:41:07 -0700 Subject: [PATCH 2/7] command/push: UX --- command/push.go | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/command/push.go b/command/push.go index 3f568bd393..e3cfe714b5 100644 --- a/command/push.go +++ b/command/push.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "sort" "strings" "github.com/hashicorp/atlas-go/archive" @@ -133,13 +134,13 @@ func (c *PushCommand) Run(args []string) int { } // Get the variables we might already have - vars, err := c.client.Get(name) + atlasVars, err := c.client.Get(name) if err != nil { c.Ui.Error(fmt.Sprintf( "Error looking up previously pushed configuration: %s", err)) return 1 } - for k, v := range vars { + for k, v := range atlasVars { if _, ok := setMap[k]; ok { continue } @@ -177,12 +178,41 @@ func (c *PushCommand) Run(args []string) int { return 1 } + // Output to the user the variables that will be uploaded + var setVars []string + for k, _ := range ctx.Variables() { + if _, ok := setMap[k]; !ok { + if _, ok := atlasVars[k]; ok { + // Atlas variable not within override, so it came from Atlas + continue + } + } + + // This variable was set from the local value + setVars = append(setVars, k) + } + sort.Strings(setVars) + if len(setVars) > 0 { + c.Ui.Output( + "The following variables will be set or updated within Atlas from\n" + + "their local values. All other variables are already set within Atlas.\n" + + "If you want to modify the value of a variable, use the Atlas web\n" + + "interface or set it locally and use the -set flag.\n\n") + for _, v := range setVars { + c.Ui.Output(fmt.Sprintf(" * %s", v)) + } + + // Newline + c.Ui.Output("") + } + // Upsert! opts := &pushUpsertOptions{ Name: name, Archive: archiveR, Variables: ctx.Variables(), } + c.Ui.Output("Uploading Terraform configuration...") vsn, err := c.client.Upsert(opts) if err != nil { c.Ui.Error(fmt.Sprintf( From 82028a7667b9eb4e6b0a783fb8f0b27d1e8f079e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 29 Jun 2015 13:45:35 -0700 Subject: [PATCH 3/7] website: update push docs --- .../source/docs/commands/push.html.markdown | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/website/source/docs/commands/push.html.markdown b/website/source/docs/commands/push.html.markdown index 1a752e657f..ec7f073100 100644 --- a/website/source/docs/commands/push.html.markdown +++ b/website/source/docs/commands/push.html.markdown @@ -52,10 +52,20 @@ The command-line flags are all optional. The list of available flags are: * `-no-color` - Disables output with coloring + +* `-set=foo` - Marks a specific variable to be updated within Atlas. + Normally, if a variable is already set in Atlas, Terraform will not + send the local value (even if it is different). This forces it to + send the local value to Atlas. This flag can be repeated multiple times. + * `-token=` - Atlas API token to use to authorize the upload. If blank or unspecified, the `ATLAS_TOKEN` environmental variable will be used. +* `-var='foo=bar'` - Set the value of a variable for the Terraform configuration. + +* `-var-file=foo` - Set the value of variables using a variable file. + * `-vcs=true` - If true (default), then Terraform will detect if a VCS is in use, such as Git, and will only upload files that are comitted to version control. If no version control system is detected, Terraform will @@ -78,6 +88,20 @@ all the files to be safe. To exclude certain files, specify the `-exclude` flag when pushing, or specify the `exclude` parameter in the [Atlas configuration section](/docs/configuration/atlas.html). +## Terraform Variables + +When you `push`, Terraform will automatically set the local values of +your Terraform variables within Atlas. The values are only set if they +don't already exist within Atlas. If you want to force push a certain +variable value to update it, use the `-set` flag. + +All the variable values stored within Atlas are encrypted and secured +using [Vault](https://vaultproject.io). We blogged about the +[architecture of our secure storage system](https://hashicorp.com/blog/how-atlas-uses-vault-for-managing-secrets.html) if you want more detail. + +The variable values can be updated using the `-set` flag or via +the [Atlas website](https://atlas.hashicorp.com). + ## Remote State Requirement `terraform push` requires that From 9d9bcc2f6e1eb757f7eef5cbe000bbc3e347e33b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 29 Jun 2015 13:53:05 -0700 Subject: [PATCH 4/7] command/push: update flag to -overwrite, update docs --- command/push.go | 6 +++--- command/push_test.go | 2 +- .../source/docs/commands/push.html.markdown | 18 ++++++++++++++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/command/push.go b/command/push.go index e3cfe714b5..f3bfe815a8 100644 --- a/command/push.go +++ b/command/push.go @@ -34,7 +34,7 @@ func (c *PushCommand) Run(args []string) int { cmdFlags.BoolVar(&moduleUpload, "upload-modules", true, "") cmdFlags.StringVar(&name, "name", "", "") cmdFlags.BoolVar(&archiveVCS, "vcs", true, "") - cmdFlags.Var((*FlagStringSlice)(&set), "set", "") + cmdFlags.Var((*FlagStringSlice)(&set), "overwrite", "") cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } if err := cmdFlags.Parse(args); err != nil { return 1 @@ -197,7 +197,7 @@ func (c *PushCommand) Run(args []string) int { "The following variables will be set or updated within Atlas from\n" + "their local values. All other variables are already set within Atlas.\n" + "If you want to modify the value of a variable, use the Atlas web\n" + - "interface or set it locally and use the -set flag.\n\n") + "interface or set it locally and use the -overwrite flag.\n\n") for _, v := range setVars { c.Ui.Output(fmt.Sprintf(" * %s", v)) } @@ -249,7 +249,7 @@ Options: -token= Access token to use to upload. If blank or unspecified, the ATLAS_TOKEN environmental variable will be used. - -set=foo Variable keys that should overwrite values in Atlas. + -overwrite=foo Variable keys that should overwrite values in Atlas. Otherwise, variables already set in Atlas will overwrite local values. This flag can be repeated. diff --git a/command/push_test.go b/command/push_test.go index 19bce23ecd..a3d171a61c 100644 --- a/command/push_test.go +++ b/command/push_test.go @@ -225,7 +225,7 @@ func TestPush_localOverride(t *testing.T) { args := []string{ "-var-file", path + "/terraform.tfvars", "-vcs=false", - "-set=foo", + "-overwrite=foo", path, } if code := c.Run(args); code != 0 { diff --git a/website/source/docs/commands/push.html.markdown b/website/source/docs/commands/push.html.markdown index ec7f073100..3f31598386 100644 --- a/website/source/docs/commands/push.html.markdown +++ b/website/source/docs/commands/push.html.markdown @@ -53,7 +53,7 @@ The command-line flags are all optional. The list of available flags are: * `-no-color` - Disables output with coloring -* `-set=foo` - Marks a specific variable to be updated within Atlas. +* `-overwrite=foo` - Marks a specific variable to be updated within Atlas. Normally, if a variable is already set in Atlas, Terraform will not send the local value (even if it is different). This forces it to send the local value to Atlas. This flag can be repeated multiple times. @@ -93,14 +93,24 @@ flag when pushing, or specify the `exclude` parameter in the When you `push`, Terraform will automatically set the local values of your Terraform variables within Atlas. The values are only set if they don't already exist within Atlas. If you want to force push a certain -variable value to update it, use the `-set` flag. +variable value to update it, use the `-overwrite` flag. All the variable values stored within Atlas are encrypted and secured using [Vault](https://vaultproject.io). We blogged about the [architecture of our secure storage system](https://hashicorp.com/blog/how-atlas-uses-vault-for-managing-secrets.html) if you want more detail. -The variable values can be updated using the `-set` flag or via -the [Atlas website](https://atlas.hashicorp.com). +The variable values can be updated using the `-overwrite` flag or via +the [Atlas website](https://atlas.hashicorp.com). An example of updating +just a single variable `foo` is shown below: + +``` +$ terraform push -var 'foo=bar' -overwrite foo +... +``` + +Both the `-var` and `-overwrite` flag are required. The `-var` flag +sets the value locally (the exact same process as commands such as apply +or plan), and the `-overwrite` flag tells the push command to update Atlas. ## Remote State Requirement From c5d3c585c65aa2684246ebb9baab955b6a25b376 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 29 Jun 2015 13:57:58 -0700 Subject: [PATCH 5/7] command/push: update var name --- command/push.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/command/push.go b/command/push.go index f3bfe815a8..9f957b49e2 100644 --- a/command/push.go +++ b/command/push.go @@ -25,7 +25,7 @@ func (c *PushCommand) Run(args []string) int { var atlasAddress, atlasToken string var archiveVCS, moduleUpload bool var name string - var set []string + var overwrite []string args = c.Meta.process(args, true) cmdFlags := c.Meta.flagSet("push") cmdFlags.StringVar(&atlasAddress, "atlas-address", "", "") @@ -34,16 +34,16 @@ func (c *PushCommand) Run(args []string) int { cmdFlags.BoolVar(&moduleUpload, "upload-modules", true, "") cmdFlags.StringVar(&name, "name", "", "") cmdFlags.BoolVar(&archiveVCS, "vcs", true, "") - cmdFlags.Var((*FlagStringSlice)(&set), "overwrite", "") + cmdFlags.Var((*FlagStringSlice)(&overwrite), "overwrite", "") cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } if err := cmdFlags.Parse(args); err != nil { return 1 } // Make a map of the set values - setMap := make(map[string]struct{}, len(set)) - for _, v := range set { - setMap[v] = struct{}{} + overwriteMap := make(map[string]struct{}, len(overwrite)) + for _, v := range overwrite { + overwriteMap[v] = struct{}{} } // The pwd is used for the configuration path if one is not given @@ -141,7 +141,7 @@ func (c *PushCommand) Run(args []string) int { return 1 } for k, v := range atlasVars { - if _, ok := setMap[k]; ok { + if _, ok := overwriteMap[k]; ok { continue } @@ -181,7 +181,7 @@ func (c *PushCommand) Run(args []string) int { // Output to the user the variables that will be uploaded var setVars []string for k, _ := range ctx.Variables() { - if _, ok := setMap[k]; !ok { + if _, ok := overwriteMap[k]; !ok { if _, ok := atlasVars[k]; ok { // Atlas variable not within override, so it came from Atlas continue From 1f92dd5b409203fe8e0d1ad80123cc9e7d236b5e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 29 Jun 2015 13:58:54 -0700 Subject: [PATCH 6/7] command/push: update output --- command/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/push.go b/command/push.go index 9f957b49e2..7695adf212 100644 --- a/command/push.go +++ b/command/push.go @@ -194,7 +194,7 @@ func (c *PushCommand) Run(args []string) int { sort.Strings(setVars) if len(setVars) > 0 { c.Ui.Output( - "The following variables will be set or updated within Atlas from\n" + + "The following variables will be set or overwritten within Atlas from\n" + "their local values. All other variables are already set within Atlas.\n" + "If you want to modify the value of a variable, use the Atlas web\n" + "interface or set it locally and use the -overwrite flag.\n\n") From 3cc5252b9dba9bcfa702ef3c5404f1c599214de0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 29 Jun 2015 13:59:25 -0700 Subject: [PATCH 7/7] website: update copy --- website/source/docs/commands/push.html.markdown | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/source/docs/commands/push.html.markdown b/website/source/docs/commands/push.html.markdown index 3f31598386..8c5af1f8fe 100644 --- a/website/source/docs/commands/push.html.markdown +++ b/website/source/docs/commands/push.html.markdown @@ -53,7 +53,7 @@ The command-line flags are all optional. The list of available flags are: * `-no-color` - Disables output with coloring -* `-overwrite=foo` - Marks a specific variable to be updated within Atlas. +* `-overwrite=foo` - Marks a specific variable to be updated on Atlas. Normally, if a variable is already set in Atlas, Terraform will not send the local value (even if it is different). This forces it to send the local value to Atlas. This flag can be repeated multiple times. @@ -91,11 +91,11 @@ flag when pushing, or specify the `exclude` parameter in the ## Terraform Variables When you `push`, Terraform will automatically set the local values of -your Terraform variables within Atlas. The values are only set if they -don't already exist within Atlas. If you want to force push a certain +your Terraform variables on Atlas. The values are only set if they +don't already exist on Atlas. If you want to force push a certain variable value to update it, use the `-overwrite` flag. -All the variable values stored within Atlas are encrypted and secured +All the variable values stored on Atlas are encrypted and secured using [Vault](https://vaultproject.io). We blogged about the [architecture of our secure storage system](https://hashicorp.com/blog/how-atlas-uses-vault-for-managing-secrets.html) if you want more detail.