From bdab86962fdd0a2106a744d7f8f1d3d3e7bc893e Mon Sep 17 00:00:00 2001 From: Syasusu Date: Tue, 2 Apr 2024 18:32:37 +0200 Subject: [PATCH] feat: init and get command support json format output (#1453) Signed-off-by: Syasusu --- CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 4 +- internal/command/e2etest/init_test.go | 214 +++++++++++++++++------ internal/command/get.go | 16 ++ internal/command/init.go | 17 ++ internal/command/meta.go | 10 +- internal/command/meta_ui.go | 60 +++++++ internal/command/views/json_view.go | 27 ++- internal/command/views/json_view_test.go | 46 +++-- 10 files changed, 325 insertions(+), 72 deletions(-) create mode 100644 internal/command/meta_ui.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 402bd3cac2..a6213fb6e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ ENHANCEMENTS: * Dump state file when `tofu test` fails to clean up resources. ([#1243](https://github.com/opentofu/opentofu/pull/1243)) * Added aliases for `state list` (`state ls`), `state mv` (`state move`), and `state rm` (`state remove`) ([#1220](https://github.com/opentofu/opentofu/pull/1220)) * Added mechanism to introduce automatic retries for provider installations, specifically targeting transient errors ([#1233](https://github.com/opentofu/opentofu/issues/1233)) +* Added `-json` flag to `tofu init` and `tofu get` to support output in json format. ([#1453](https://github.com/opentofu/opentofu/pull/1453)) BUG FIXES: * Fix view hooks unit test flakiness by deterministically waiting for heartbeats to execute ([$1153](https://github.com/opentofu/opentofu/issues/1153)) diff --git a/go.mod b/go.mod index 5d2a667c71..2a70e96dbd 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/hashicorp/go-azure-helpers v0.43.0 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-getter v1.7.3 - github.com/hashicorp/go-hclog v1.5.0 + github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-plugin v1.4.3 github.com/hashicorp/go-retryablehttp v0.7.4 diff --git a/go.sum b/go.sum index e6fa94e2b4..20d9a0bcc3 100644 --- a/go.sum +++ b/go.sum @@ -661,8 +661,8 @@ github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= -github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= diff --git a/internal/command/e2etest/init_test.go b/internal/command/e2etest/init_test.go index 4924aceff0..fa71c26361 100644 --- a/internal/command/e2etest/init_test.go +++ b/internal/command/e2etest/init_test.go @@ -62,33 +62,67 @@ func TestInitProvidersInternal(t *testing.T) { // This test should _not_ reach out anywhere because the "terraform" // provider is internal to the core tofu binary. - fixturePath := filepath.Join("testdata", "tf-provider") - tf := e2e.NewBinary(t, tofuBin, fixturePath) + t.Run("output in human readable format", func(t *testing.T) { + fixturePath := filepath.Join("testdata", "tf-provider") + tf := e2e.NewBinary(t, tofuBin, fixturePath) - stdout, stderr, err := tf.Run("init") - if err != nil { - t.Errorf("unexpected error: %s", err) - } + stdout, stderr, err := tf.Run("init") + if err != nil { + t.Errorf("unexpected error: %s", err) + } - if stderr != "" { - t.Errorf("unexpected stderr output:\n%s", stderr) - } + if stderr != "" { + t.Errorf("unexpected stderr output:\n%s", stderr) + } - if !strings.Contains(stdout, "OpenTofu has been successfully initialized!") { - t.Errorf("success message is missing from output:\n%s", stdout) - } + if !strings.Contains(stdout, "OpenTofu has been successfully initialized!") { + t.Errorf("success message is missing from output:\n%s", stdout) + } - if strings.Contains(stdout, "Installing hashicorp/terraform") { - // Shouldn't have downloaded anything with this config, because the - // provider is built in. - t.Errorf("provider download message appeared in output:\n%s", stdout) - } + if strings.Contains(stdout, "Installing hashicorp/terraform") { + // Shouldn't have downloaded anything with this config, because the + // provider is built in. + t.Errorf("provider download message appeared in output:\n%s", stdout) + } + + if strings.Contains(stdout, "Installing terraform.io/builtin/terraform") { + // Shouldn't have downloaded anything with this config, because the + // provider is built in. + t.Errorf("provider download message appeared in output:\n%s", stdout) + } + }) + + t.Run("output in machine readable format", func(t *testing.T) { + fixturePath := filepath.Join("testdata", "tf-provider") + tf := e2e.NewBinary(t, tofuBin, fixturePath) + + stdout, stderr, err := tf.Run("init", "-json") + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if stderr != "" { + t.Errorf("unexpected stderr output:\n%s", stderr) + } + + // we can not check timestamp, so the sub string is not a valid json object + if !strings.Contains(stdout, `{"@level":"info","@message":"OpenTofu has been successfully initialized!","@module":"tofu.ui"`) { + t.Errorf("success message is missing from output:\n%s", stdout) + } + + if strings.Contains(stdout, "Installing hashicorp/terraform") { + // Shouldn't have downloaded anything with this config, because the + // provider is built in. + t.Errorf("provider download message appeared in output:\n%s", stdout) + } + + if strings.Contains(stdout, "Installing terraform.io/builtin/terraform") { + // Shouldn't have downloaded anything with this config, because the + // provider is built in. + t.Errorf("provider download message appeared in output:\n%s", stdout) + } + }) - if strings.Contains(stdout, "Installing terraform.io/builtin/terraform") { - // Shouldn't have downloaded anything with this config, because the - // provider is built in. - t.Errorf("provider download message appeared in output:\n%s", stdout) - } } func TestInitProvidersVendored(t *testing.T) { @@ -144,42 +178,85 @@ func TestInitProvidersLocalOnly(t *testing.T) { // to the host "example.com", which is the placeholder domain we use in // the test fixture.) - fixturePath := filepath.Join("testdata", "local-only-provider") - tf := e2e.NewBinary(t, tofuBin, fixturePath) - // If you run this test on a workstation with a plugin-cache directory - // configured, it will leave a bad directory behind and tofu init will - // not work until you remove it. - // - // To avoid this, we will "zero out" any existing cli config file. - tf.AddEnv("TF_CLI_CONFIG_FILE=") + t.Run("output in human readable format", func(t *testing.T) { + fixturePath := filepath.Join("testdata", "local-only-provider") + tf := e2e.NewBinary(t, tofuBin, fixturePath) + // If you run this test on a workstation with a plugin-cache directory + // configured, it will leave a bad directory behind and tofu init will + // not work until you remove it. + // + // To avoid this, we will "zero out" any existing cli config file. + tf.AddEnv("TF_CLI_CONFIG_FILE=") - // Our fixture dir has a generic os_arch dir, which we need to customize - // to the actual OS/arch where this test is running in order to get the - // desired result. - fixtMachineDir := tf.Path("terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/os_arch") - wantMachineDir := tf.Path("terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)) - err := os.Rename(fixtMachineDir, wantMachineDir) - if err != nil { - t.Fatalf("unexpected error: %s", err) - } + // Our fixture dir has a generic os_arch dir, which we need to customize + // to the actual OS/arch where this test is running in order to get the + // desired result. + fixtMachineDir := tf.Path("terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/os_arch") + wantMachineDir := tf.Path("terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)) + err := os.Rename(fixtMachineDir, wantMachineDir) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } - stdout, stderr, err := tf.Run("init") - if err != nil { - t.Errorf("unexpected error: %s", err) - } + stdout, stderr, err := tf.Run("init") + if err != nil { + t.Errorf("unexpected error: %s", err) + } - if stderr != "" { - t.Errorf("unexpected stderr output:\n%s", stderr) - } + if stderr != "" { + t.Errorf("unexpected stderr output:\n%s", stderr) + } - if !strings.Contains(stdout, "OpenTofu has been successfully initialized!") { - t.Errorf("success message is missing from output:\n%s", stdout) - } + if !strings.Contains(stdout, "OpenTofu has been successfully initialized!") { + t.Errorf("success message is missing from output:\n%s", stdout) + } + + if !strings.Contains(stdout, "- Installing example.com/awesomecorp/happycloud v1.2.0") { + t.Errorf("provider download message is missing from output:\n%s", stdout) + t.Logf("(this can happen if you have a conflicting copy of the plugin in one of the global plugin search dirs)") + } + }) + + t.Run("output in machine readable format", func(t *testing.T) { + fixturePath := filepath.Join("testdata", "local-only-provider") + tf := e2e.NewBinary(t, tofuBin, fixturePath) + // If you run this test on a workstation with a plugin-cache directory + // configured, it will leave a bad directory behind and tofu init will + // not work until you remove it. + // + // To avoid this, we will "zero out" any existing cli config file. + tf.AddEnv("TF_CLI_CONFIG_FILE=") + + // Our fixture dir has a generic os_arch dir, which we need to customize + // to the actual OS/arch where this test is running in order to get the + // desired result. + fixtMachineDir := tf.Path("terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/os_arch") + wantMachineDir := tf.Path("terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)) + err := os.Rename(fixtMachineDir, wantMachineDir) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + stdout, stderr, err := tf.Run("init", "-json") + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if stderr != "" { + t.Errorf("unexpected stderr output:\n%s", stderr) + } + + // we can not check timestamp, so the sub string is not a valid json object + if !strings.Contains(stdout, `{"@level":"info","@message":"OpenTofu has been successfully initialized!","@module":"tofu.ui"`) { + t.Errorf("success message is missing from output:\n%s", stdout) + } + + if !strings.Contains(stdout, `{"@level":"info","@message":"- Installing example.com/awesomecorp/happycloud v1.2.0...","@module":"tofu.ui"`) { + t.Errorf("provider download message is missing from output:\n%s", stdout) + t.Logf("(this can happen if you have a conflicting copy of the plugin in one of the global plugin search dirs)") + } + }) - if !strings.Contains(stdout, "- Installing example.com/awesomecorp/happycloud v1.2.0") { - t.Errorf("provider download message is missing from output:\n%s", stdout) - t.Logf("(this can happen if you have a conflicting copy of the plugin in one of the global plugin search dirs)") - } } func TestInitProvidersCustomMethod(t *testing.T) { @@ -347,9 +424,21 @@ func TestInitProviderNotFound(t *testing.T) { } }) + t.Run("registry provider not found output in json format", func(t *testing.T) { + stdout, _, err := tf.Run("init", "-no-color", "-json") + if err == nil { + t.Fatal("expected error, got success") + } + + oneLineStdout := strings.ReplaceAll(stdout, "\n", " ") + if !strings.Contains(oneLineStdout, `"diagnostic":{"severity":"error","summary":"Failed to query available provider packages","detail":"Could not retrieve the list of available versions for provider hashicorp/nonexist: provider registry registry.opentofu.org does not have a provider named registry.opentofu.org/hashicorp/nonexist\n\nAll modules should specify their required_providers so that external consumers will get the correct providers when using a module. To see which modules are currently depending on hashicorp/nonexist, run the following command:\n tofu providers\n\nIf you believe this provider is missing from the registry, please submit a issue on the OpenTofu Registry https://github.com/opentofu/registry/issues/"},"type":"diagnostic"}`) { + t.Errorf("expected error message is missing from output:\n%s", stdout) + } + }) + t.Run("local provider not found", func(t *testing.T) { // The -plugin-dir directory must exist for the provider installer to search it. - pluginDir := tf.Path("empty") + pluginDir := tf.Path("empty-for-json") if err := os.Mkdir(pluginDir, os.ModePerm); err != nil { t.Fatal(err) } @@ -364,6 +453,23 @@ func TestInitProviderNotFound(t *testing.T) { } }) + t.Run("local provider not found output in json format", func(t *testing.T) { + // The -plugin-dir directory must exist for the provider installer to search it. + pluginDir := tf.Path("empty") + if err := os.Mkdir(pluginDir, os.ModePerm); err != nil { + t.Fatal(err) + } + + stdout, _, err := tf.Run("init", "-no-color", "-plugin-dir="+pluginDir, "-json") + if err == nil { + t.Fatal("expected error, got success") + } + + if !strings.Contains(stdout, `"diagnostic":{"severity":"error","summary":"Failed to query available provider packages","detail":"Could not retrieve the list of available versions for provider hashicorp/nonexist: provider registry.opentofu.org/hashicorp/nonexist was not found in any of the search locations\n\n - `+pluginDir+`"},"type":"diagnostic"}`) { + t.Errorf("expected error message is missing from output:\n%s", stdout) + } + }) + t.Run("special characters enabled", func(t *testing.T) { _, stderr, err := tf.Run("init") if err == nil { diff --git a/internal/command/get.go b/internal/command/get.go index a3a0ce2cad..b3799a5450 100644 --- a/internal/command/get.go +++ b/internal/command/get.go @@ -10,6 +10,7 @@ import ( "fmt" "strings" + "github.com/opentofu/opentofu/internal/command/views" "github.com/opentofu/opentofu/internal/tfdiags" ) @@ -27,11 +28,22 @@ func (c *GetCommand) Run(args []string) int { cmdFlags := c.Meta.defaultFlagSet("get") cmdFlags.BoolVar(&update, "update", false, "update") cmdFlags.StringVar(&testsDirectory, "test-directory", "tests", "test-directory") + cmdFlags.BoolVar(&c.outputInJSON, "json", false, "json") cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } if err := cmdFlags.Parse(args); err != nil { c.Ui.Error(fmt.Sprintf("Error parsing command-line flags: %s\n", err.Error())) return 1 } + if c.outputInJSON { + c.Meta.color = false + c.Meta.Color = false + c.oldUi = c.Ui + c.Ui = &WrappedUi{ + cliUi: c.oldUi, + jsonView: views.NewJSONView(c.View), + outputInJSON: true, + } + } // Initialization can be aborted by interruption signals ctx, done := c.InterruptibleContext(c.CommandContext()) @@ -81,6 +93,10 @@ Options: test command will search for test files in the current directory and in the one specified by the flag. + -json Produce output in a machine-readable JSON format, + suitable for use in text editor integrations and other + automated systems. Always disables color. + ` return strings.TrimSpace(helpText) } diff --git a/internal/command/init.go b/internal/command/init.go index 857da1d40a..7756af0c8f 100644 --- a/internal/command/init.go +++ b/internal/command/init.go @@ -26,6 +26,7 @@ import ( backendInit "github.com/opentofu/opentofu/internal/backend/init" "github.com/opentofu/opentofu/internal/cloud" "github.com/opentofu/opentofu/internal/command/arguments" + "github.com/opentofu/opentofu/internal/command/views" "github.com/opentofu/opentofu/internal/configs" "github.com/opentofu/opentofu/internal/configs/configschema" "github.com/opentofu/opentofu/internal/encryption" @@ -67,11 +68,23 @@ func (c *InitCommand) Run(args []string) int { cmdFlags.StringVar(&flagLockfile, "lockfile", "", "Set a dependency lockfile mode") cmdFlags.BoolVar(&c.Meta.ignoreRemoteVersion, "ignore-remote-version", false, "continue even if remote and local OpenTofu versions are incompatible") cmdFlags.StringVar(&testsDirectory, "test-directory", "tests", "test-directory") + cmdFlags.BoolVar(&c.outputInJSON, "json", false, "json") cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } if err := cmdFlags.Parse(args); err != nil { return 1 } + if c.outputInJSON { + c.Meta.color = false + c.Meta.Color = false + c.oldUi = c.Ui + c.Ui = &WrappedUi{ + cliUi: c.oldUi, + jsonView: views.NewJSONView(c.View), + outputInJSON: true, + } + } + backendFlagSet := arguments.FlagIsSet(cmdFlags, "backend") cloudFlagSet := arguments.FlagIsSet(cmdFlags, "cloud") @@ -1233,6 +1246,10 @@ Options: test command will search for test files in the current directory and in the one specified by the flag. + -json Produce output in a machine-readable JSON format, + suitable for use in text editor integrations and other + automated systems. Always disables color. + ` return strings.TrimSpace(helpText) } diff --git a/internal/command/meta.go b/internal/command/meta.go index 808a959192..b8e99de511 100644 --- a/internal/command/meta.go +++ b/internal/command/meta.go @@ -19,7 +19,7 @@ import ( "strings" "time" - plugin "github.com/hashicorp/go-plugin" + "github.com/hashicorp/go-plugin" "github.com/hashicorp/terraform-svchost/disco" "github.com/mitchellh/cli" "github.com/mitchellh/colorstring" @@ -266,6 +266,8 @@ type Meta struct { // Used with commands which write state to allow users to write remote // state even if the remote and local OpenTofu versions don't match. ignoreRemoteVersion bool + + outputInJSON bool } type testingOverrides struct { @@ -693,6 +695,12 @@ func (m *Meta) showDiagnostics(vals ...interface{}) { return } + if m.outputInJSON { + jsonView := views.NewJSONView(m.View) + jsonView.Diagnostics(diags) + return + } + outputWidth := m.ErrorColumns() diags = diags.ConsolidateWarnings(1) diff --git a/internal/command/meta_ui.go b/internal/command/meta_ui.go new file mode 100644 index 0000000000..0c5c7c4e56 --- /dev/null +++ b/internal/command/meta_ui.go @@ -0,0 +1,60 @@ +package command + +import ( + "github.com/mitchellh/cli" + + "github.com/opentofu/opentofu/internal/command/views" +) + +// WrappedUi is a shim which adds json compatibility to those commands which +// have not yet been refactored to support output by views.View. +// +// For those not support json output command, all output is printed by cli.Ui. +// So we create WrappedUi, contains the old cli.Ui and views.JSONView, +// implement cli.Ui interface, so that we can make all command support json +// output in a short time. +type WrappedUi struct { + cliUi cli.Ui + jsonView *views.JSONView + outputInJSON bool +} + +func (m *WrappedUi) Ask(s string) (string, error) { + return m.cliUi.Ask(s) +} + +func (m *WrappedUi) AskSecret(s string) (string, error) { + return m.cliUi.AskSecret(s) +} + +func (m *WrappedUi) Output(s string) { + if m.outputInJSON { + m.jsonView.Output(s) + return + } + m.cliUi.Output(s) +} + +func (m *WrappedUi) Info(s string) { + if m.outputInJSON { + m.jsonView.Info(s) + return + } + m.cliUi.Info(s) +} + +func (m *WrappedUi) Error(s string) { + if m.outputInJSON { + m.jsonView.Error(s) + return + } + m.cliUi.Error(s) +} + +func (m *WrappedUi) Warn(s string) { + if m.outputInJSON { + m.jsonView.Warn(s) + return + } + m.cliUi.Warn(s) +} diff --git a/internal/command/views/json_view.go b/internal/command/views/json_view.go index a85d0ef692..3d17fb62d2 100644 --- a/internal/command/views/json_view.go +++ b/internal/command/views/json_view.go @@ -23,9 +23,10 @@ const JSON_UI_VERSION = "1.2" func NewJSONView(view *View) *JSONView { log := hclog.New(&hclog.LoggerOptions{ - Name: "tofu.ui", - Output: view.streams.Stdout.File, - JSONFormat: true, + Name: "tofu.ui", + Output: view.streams.Stdout.File, + JSONFormat: true, + JSONEscapeDisabled: true, }) jv := &JSONView{ log: log, @@ -128,3 +129,23 @@ func (v *JSONView) Outputs(outputs json.Outputs) { "outputs", outputs, ) } + +// Output is designed for supporting command.WrappedUi +func (v *JSONView) Output(message string) { + v.log.Info(message, "type", "output") +} + +// Info is designed for supporting command.WrappedUi +func (v *JSONView) Info(message string) { + v.log.Info(message) +} + +// Warn is designed for supporting command.WrappedUi +func (v *JSONView) Warn(message string) { + v.log.Warn(message) +} + +// Error is designed for supporting command.WrappedUi +func (v *JSONView) Error(message string) { + v.log.Error(message) +} diff --git a/internal/command/views/json_view_test.go b/internal/command/views/json_view_test.go index 7ca1a856fa..8b18cddc78 100644 --- a/internal/command/views/json_view_test.go +++ b/internal/command/views/json_view_test.go @@ -44,20 +44,44 @@ func TestNewJSONView(t *testing.T) { } func TestJSONView_Log(t *testing.T) { - streams, done := terminal.StreamsForTesting(t) - jv := NewJSONView(NewView(streams)) - - jv.Log("hello, world") - - want := []map[string]interface{}{ + testCases := []struct { + caseName string + input string + want []map[string]interface{} + }{ { - "@level": "info", - "@message": "hello, world", - "@module": "tofu.ui", - "type": "log", + "log with regular character", + "hello, world", + []map[string]interface{}{ + { + "@level": "info", + "@message": "hello, world", + "@module": "tofu.ui", + "type": "log", + }, + }, + }, + { + "log with special character", + "hello, special char, <>&", + []map[string]interface{}{ + { + "@level": "info", + "@message": "hello, special char, <>&", + "@module": "tofu.ui", + "type": "log", + }, + }, }, } - testJSONViewOutputEquals(t, done(t).Stdout(), want) + for _, tc := range testCases { + t.Run(tc.caseName, func(t *testing.T) { + streams, done := terminal.StreamsForTesting(t) + jv := NewJSONView(NewView(streams)) + jv.Log(tc.input) + testJSONViewOutputEquals(t, done(t).Stdout(), tc.want) + }) + } } // This test covers only the basics of JSON diagnostic rendering, as more