Merge pull request #32303 from mrinalirao/mr/policy-evaluation

Add policy evaluation task stage to the CLI

go.mod

@@ -39,7 +39,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-plugin v1.4.3
 	github.com/hashicorp/go-retryablehttp v0.7.1
-	github.com/hashicorp/go-tfe v1.12.0
+	github.com/hashicorp/go-tfe v1.14.0
 	github.com/hashicorp/go-uuid v1.0.3
 	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f
@@ -145,7 +145,7 @@ require (
 	github.com/hashicorp/go-msgpack v0.5.4 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-safetemp v1.0.0 // indirect
-	github.com/hashicorp/go-slug v0.10.0 // indirect
+	github.com/hashicorp/go-slug v0.10.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.1 // indirect
 	github.com/hashicorp/jsonapi v0.0.0-20210826224640-ee7dae0fb22d // indirect
 	github.com/hashicorp/serf v0.9.5 // indirect

go.sum

@@ -371,13 +371,13 @@ github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5O
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
 github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
-github.com/hashicorp/go-slug v0.10.0 h1:mh4DDkBJTh9BuEjY/cv8PTo7k9OjT4PcW8PgZnJ4jTY=
+github.com/hashicorp/go-slug v0.10.1 h1:05SCRWCBpCxOeP7stQHvMgOz0raCBCekaytu8Rg/RZ4=
-github.com/hashicorp/go-slug v0.10.0/go.mod h1:Ib+IWBYfEfJGI1ZyXMGNbu2BU+aa3Dzu41RKLH301v4=
+github.com/hashicorp/go-slug v0.10.1/go.mod h1:Ib+IWBYfEfJGI1ZyXMGNbu2BU+aa3Dzu41RKLH301v4=
 github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-tfe v1.12.0 h1:2l7emKW8rNTTbnxYHNVj6b46iJzOEp2G/3xIHfGSDnc=
+github.com/hashicorp/go-tfe v1.14.0 h1:FZKKkwlyTxw8/OE3e7NiFQLcgGXTHra9ogGhMTotxh8=
-github.com/hashicorp/go-tfe v1.12.0/go.mod h1:thYtIxtgBpDDNdf/2yYPdBJ94Fz5yT5XCNZvGtTGHAU=
+github.com/hashicorp/go-tfe v1.14.0/go.mod h1:77snluBqtTTvMrY0w/mxQA5jlHQ8NT44AqQ8UdrPf0o=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=

internal/cloud/backend_common.go

@@ -207,7 +207,7 @@ func (b *Cloud) waitTaskStage(stopCtx, cancelCtx context.Context, op *backend.Op
 		Op:            op,
 		Run:           r,
 	}
-	return b.runTasks(integration, integration.BeginOutput(outputTitle), stageID)
+	return b.runTaskStage(integration, integration.BeginOutput(outputTitle), stageID)
 }
 
 func (b *Cloud) costEstimate(stopCtx, cancelCtx context.Context, op *backend.Operation, r *tfe.Run) error {
@@ -450,7 +450,7 @@ func (b *Cloud) confirm(stopCtx context.Context, op *backend.Operation, opts *te
 
 				switch keyword {
 				case "override":
-					if r.Status != tfe.RunPolicyOverride {
+					if r.Status != tfe.RunPolicyOverride && r.Status != tfe.RunPostPlanAwaitingDecision {
 						if r.Status == tfe.RunDiscarded {
 							err = errRunDiscarded
 						} else {

internal/cloud/backend_runTasks.go

@@ -1,149 +0,0 @@
-package cloud
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"github.com/hashicorp/go-tfe"
-)
-
-type taskResultSummary struct {
-	unreachable     bool
-	pending         int
-	failed          int
-	failedMandatory int
-	passed          int
-}
-
-type taskStageReadFunc func(b *Cloud, stopCtx context.Context) (*tfe.TaskStage, error)
-
-func summarizeTaskResults(taskResults []*tfe.TaskResult) *taskResultSummary {
-	var pendingCount, errCount, errMandatoryCount, passedCount int
-	for _, task := range taskResults {
-		if task.Status == "unreachable" {
-			return &taskResultSummary{
-				unreachable: true,
-			}
-		} else if task.Status == "running" || task.Status == "pending" {
-			pendingCount++
-		} else if task.Status == "passed" {
-			passedCount++
-		} else {
-			// Everything else is a failure
-			errCount++
-			if task.WorkspaceTaskEnforcementLevel == "mandatory" {
-				errMandatoryCount++
-			}
-		}
-	}
-
-	return &taskResultSummary{
-		unreachable:     false,
-		pending:         pendingCount,
-		failed:          errCount,
-		failedMandatory: errMandatoryCount,
-		passed:          passedCount,
-	}
-}
-
-func (b *Cloud) runTasksWithTaskResults(context *IntegrationContext, output IntegrationOutputWriter, fetchTaskStage taskStageReadFunc) error {
-	return context.Poll(func(i int) (bool, error) {
-		stage, err := fetchTaskStage(b, context.StopContext)
-
-		if err != nil {
-			return false, generalError("Failed to retrieve task stage", err)
-		}
-
-		summary := summarizeTaskResults(stage.TaskResults)
-
-		if summary.unreachable {
-			output.Output("Skipping task results.")
-			output.End()
-			return false, nil
-		}
-
-		if summary.pending > 0 {
-			pendingMessage := "%d tasks still pending, %d passed, %d failed ... "
-			message := fmt.Sprintf(pendingMessage, summary.pending, summary.passed, summary.failed)
-
-			if i%4 == 0 {
-				if i > 0 {
-					output.OutputElapsed(message, len(pendingMessage)) // Up to 2 digits are allowed by the max message allocation
-				}
-			}
-			return true, nil
-		}
-
-		// No more tasks pending/running. Print all the results.
-
-		// Track the first task name that is a mandatory enforcement level breach.
-		var firstMandatoryTaskFailed *string = nil
-
-		if i == 0 {
-			output.Output(fmt.Sprintf("All tasks completed! %d passed, %d failed", summary.passed, summary.failed))
-		} else {
-			output.OutputElapsed(fmt.Sprintf("All tasks completed! %d passed, %d failed", summary.passed, summary.failed), 50)
-		}
-
-		output.Output("")
-
-		for _, t := range stage.TaskResults {
-			capitalizedStatus := string(t.Status)
-			capitalizedStatus = strings.ToUpper(capitalizedStatus[:1]) + capitalizedStatus[1:]
-
-			status := "[green]" + capitalizedStatus
-			if t.Status != "passed" {
-				level := string(t.WorkspaceTaskEnforcementLevel)
-				level = strings.ToUpper(level[:1]) + level[1:]
-				status = fmt.Sprintf("[red]%s (%s)", capitalizedStatus, level)
-
-				if t.WorkspaceTaskEnforcementLevel == "mandatory" && firstMandatoryTaskFailed == nil {
-					firstMandatoryTaskFailed = &t.TaskName
-				}
-			}
-
-			title := fmt.Sprintf(`%s ⸺   %s`, t.TaskName, status)
-			output.SubOutput(title)
-
-			if len(t.Message) > 0 {
-				output.SubOutput(fmt.Sprintf("[dim]%s", t.Message))
-			}
-			if len(t.URL) > 0 {
-				output.SubOutput(fmt.Sprintf("[dim]Details: %s", t.URL))
-			}
-			output.SubOutput("")
-		}
-
-		// If a mandatory enforcement level is breached, return an error.
-		var taskErr error = nil
-		var overall string = "[green]Passed"
-		if firstMandatoryTaskFailed != nil {
-			overall = "[red]Failed"
-			if summary.failedMandatory > 1 {
-				taskErr = fmt.Errorf("the run failed because %d mandatory tasks are required to succeed", summary.failedMandatory)
-			} else {
-				taskErr = fmt.Errorf("the run failed because the run task, %s, is required to succeed", *firstMandatoryTaskFailed)
-			}
-		} else if summary.failed > 0 { // we have failures but none of them mandatory
-			overall = "[green]Passed with advisory failures"
-		}
-
-		output.SubOutput("")
-		output.SubOutput("[bold]Overall Result: " + overall)
-
-		output.End()
-
-		return false, taskErr
-	})
-}
-
-func (b *Cloud) runTasks(ctx *IntegrationContext, output IntegrationOutputWriter, stageID string) error {
-	return b.runTasksWithTaskResults(ctx, output, func(b *Cloud, stopCtx context.Context) (*tfe.TaskStage, error) {
-		options := tfe.TaskStageReadOptions{
-			Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults},
-		}
-
-		return b.client.TaskStages.Read(ctx.StopContext, stageID, &options)
-	})
-}

internal/cloud/backend_taskStage_policyEvaluation.go

@@ -0,0 +1,157 @@
+package cloud
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/go-tfe"
+)
+
+type policyEvaluationSummary struct {
+	unreachable bool
+	pending     int
+	failed      int
+	passed      int
+}
+
+type Symbol rune
+
+const (
+	Tick          Symbol = '\u2713'
+	Cross         Symbol = '\u00d7'
+	Warning       Symbol = '\u24be'
+	Arrow         Symbol = '\u2192'
+	DownwardArrow Symbol = '\u21b3'
+)
+
+type policyEvaluationSummarizer struct {
+	finished bool
+	cloud    *Cloud
+	counter  int
+}
+
+func newPolicyEvaluationSummarizer(b *Cloud, ts *tfe.TaskStage) taskStageSummarizer {
+	if len(ts.PolicyEvaluations) == 0 {
+		return nil
+	}
+	return &policyEvaluationSummarizer{
+		finished: false,
+		cloud:    b,
+	}
+}
+
+func (pes *policyEvaluationSummarizer) Summarize(context *IntegrationContext, output IntegrationOutputWriter, ts *tfe.TaskStage) (bool, *string, error) {
+	if pes.counter == 0 {
+		output.Output("[bold]OPA Policy Evaluation\n")
+		pes.counter++
+	}
+
+	if pes.finished {
+		return false, nil, nil
+	}
+
+	counts := summarizePolicyEvaluationResults(ts.PolicyEvaluations)
+
+	if counts.pending != 0 {
+		pendingMessage := "Evaluating ... "
+		return true, &pendingMessage, nil
+	}
+
+	if counts.unreachable {
+		output.Output("Skipping policy evaluation.")
+		output.End()
+		return false, nil, nil
+	}
+
+	// Print out the summary
+	if err := pes.taskStageWithPolicyEvaluation(context, output, ts.PolicyEvaluations); err != nil {
+		return false, nil, err
+	}
+	// Mark as finished
+	pes.finished = true
+
+	return false, nil, nil
+}
+
+func summarizePolicyEvaluationResults(policyEvaluations []*tfe.PolicyEvaluation) *policyEvaluationSummary {
+	var pendingCount, errCount, passedCount int
+	for _, policyEvaluation := range policyEvaluations {
+		switch policyEvaluation.Status {
+		case "unreachable":
+			return &policyEvaluationSummary{
+				unreachable: true,
+			}
+		case "running", "pending", "queued":
+			pendingCount++
+		case "passed":
+			passedCount++
+		default:
+			// Everything else is a failure
+			errCount++
+		}
+	}
+
+	return &policyEvaluationSummary{
+		unreachable: false,
+		pending:     pendingCount,
+		failed:      errCount,
+		passed:      passedCount,
+	}
+}
+
+func (pes *policyEvaluationSummarizer) taskStageWithPolicyEvaluation(context *IntegrationContext, output IntegrationOutputWriter, policyEvaluation []*tfe.PolicyEvaluation) error {
+	var result, message string
+	// Currently only one policy evaluation supported : OPA
+	for _, polEvaluation := range policyEvaluation {
+		if polEvaluation.Status == tfe.PolicyEvaluationPassed {
+			message = "[dim] This result means that all OPA policies passed and the protected behavior is allowed"
+			result = fmt.Sprintf("[green]%s", strings.ToUpper(string(tfe.PolicyEvaluationPassed)))
+			if polEvaluation.ResultCount.AdvisoryFailed > 0 {
+				result += " (with advisory)"
+			}
+		} else {
+			message = "[dim] This result means that one or more OPA policies failed. More than likely, this was due to the discovery of violations by the main rule and other sub rules"
+			result = fmt.Sprintf("[red]%s", strings.ToUpper(string(tfe.PolicyEvaluationFailed)))
+		}
+
+		output.Output(fmt.Sprintf("[bold]%c%c Overall Result: %s", Arrow, Arrow, result))
+
+		output.Output(message)
+
+		total := getPolicyCount(polEvaluation.ResultCount)
+
+		output.Output(fmt.Sprintf("%d policies evaluated\n", total))
+
+		policyOutcomes, err := pes.cloud.client.PolicySetOutcomes.List(context.StopContext, polEvaluation.ID, nil)
+		if err != nil {
+			return err
+		}
+
+		for i, out := range policyOutcomes.Items {
+			output.Output(fmt.Sprintf("%c Policy set %d: [bold]%s (%d)", Arrow, i+1, out.PolicySetName, len(out.Outcomes)))
+			for _, outcome := range out.Outcomes {
+				output.Output(fmt.Sprintf("  %c Policy name: [bold]%s", DownwardArrow, outcome.PolicyName))
+				switch outcome.Status {
+				case "passed":
+					output.Output(fmt.Sprintf("     | [green][bold]%c Passed", Tick))
+				case "failed":
+					if outcome.EnforcementLevel == tfe.EnforcementAdvisory {
+						output.Output(fmt.Sprintf("     | [blue][bold]%c Advisory", Warning))
+					} else {
+						output.Output(fmt.Sprintf("     | [red][bold]%c Failed", Cross))
+					}
+				}
+				if outcome.Description != "" {
+					output.Output(fmt.Sprintf("     | [dim]%s", outcome.Description))
+				} else {
+					output.Output("     | [dim]No description available")
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func getPolicyCount(resultCount *tfe.PolicyResultCount) int {
+	return resultCount.AdvisoryFailed + resultCount.MandatoryFailed + resultCount.Errored + resultCount.Passed
+}

internal/cloud/backend_taskStage_policyEvaluation_test.go

@@ -0,0 +1,97 @@
+package cloud
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/hashicorp/go-tfe"
+)
+
+func TestCloud_runTaskStageWithPolicyEvaluation(t *testing.T) {
+	b, bCleanup := testBackendWithName(t)
+	defer bCleanup()
+
+	integrationContext, writer := newMockIntegrationContext(b, t)
+
+	cases := map[string]struct {
+		taskStage       func() *tfe.TaskStage
+		context         *IntegrationContext
+		writer          *testIntegrationOutput
+		expectedOutputs []string
+		isError         bool
+	}{
+		"all-succeeded": {
+			taskStage: func() *tfe.TaskStage {
+				ts := &tfe.TaskStage{}
+				ts.PolicyEvaluations = []*tfe.PolicyEvaluation{
+					{ID: "pol-pass", ResultCount: &tfe.PolicyResultCount{Passed: 1}, Status: "passed"},
+				}
+				return ts
+			},
+			writer:          writer,
+			context:         integrationContext,
+			expectedOutputs: []string{"│ [bold]OPA Policy Evaluation\n\n│ [bold]→→ Overall Result: [green]PASSED\n│ [dim] This result means that all OPA policies passed and the protected behavior is allowed\n│ 1 policies evaluated\n\n│ → Policy set 1: [bold]policy-set-that-passes (1)\n│   ↳ Policy name: [bold]policy-pass\n│      | [green][bold]✓ Passed\n│      | [dim]This policy will pass\n"},
+			isError:         false,
+		},
+		"mandatory-failed": {
+			taskStage: func() *tfe.TaskStage {
+				ts := &tfe.TaskStage{}
+				ts.PolicyEvaluations = []*tfe.PolicyEvaluation{
+					{ID: "pol-fail", ResultCount: &tfe.PolicyResultCount{MandatoryFailed: 1}, Status: "failed"},
+				}
+				return ts
+			},
+			writer:          writer,
+			context:         integrationContext,
+			expectedOutputs: []string{"│ [bold]→→ Overall Result: [red]FAILED\n│ [dim] This result means that one or more OPA policies failed. More than likely, this was due to the discovery of violations by the main rule and other sub rules\n│ 1 policies evaluated\n\n│ → Policy set 1: [bold]policy-set-that-fails (1)\n│   ↳ Policy name: [bold]policy-fail\n│      | [red][bold]× Failed\n│      | [dim]This policy will fail"},
+			isError:         true,
+		},
+		"advisory-failed": {
+			taskStage: func() *tfe.TaskStage {
+				ts := &tfe.TaskStage{}
+				ts.PolicyEvaluations = []*tfe.PolicyEvaluation{
+					{ID: "adv-fail", ResultCount: &tfe.PolicyResultCount{AdvisoryFailed: 1}, Status: "failed"},
+				}
+				return ts
+			},
+			writer:          writer,
+			context:         integrationContext,
+			expectedOutputs: []string{"│ [bold]OPA Policy Evaluation\n\n│ [bold]→→ Overall Result: [red]FAILED\n│ [dim] This result means that one or more OPA policies failed. More than likely, this was due to the discovery of violations by the main rule and other sub rules\n│ 1 policies evaluated\n\n│ → Policy set 1: [bold]policy-set-that-fails (1)\n│   ↳ Policy name: [bold]policy-fail\n│      | [blue][bold]Ⓘ Advisory\n│      | [dim]This policy will fail"},
+			isError:         false,
+		},
+		"unreachable": {
+			taskStage: func() *tfe.TaskStage {
+				ts := &tfe.TaskStage{}
+				ts.PolicyEvaluations = []*tfe.PolicyEvaluation{
+					{ID: "adv-fail", ResultCount: &tfe.PolicyResultCount{Errored: 1}, Status: "unreachable"},
+				}
+				return ts
+			},
+			writer:          writer,
+			context:         integrationContext,
+			expectedOutputs: []string{"Skipping policy evaluation."},
+			isError:         false,
+		},
+	}
+
+	for _, c := range cases {
+		c.writer.output.Reset()
+		trs := policyEvaluationSummarizer{
+			cloud: b,
+		}
+		c.context.Poll(taskStageBackoffMin, taskStageBackoffMax, func(i int) (bool, error) {
+			cont, _, _ := trs.Summarize(c.context, c.writer, c.taskStage())
+			if cont {
+				return true, nil
+			}
+
+			output := c.writer.output.String()
+			for _, expected := range c.expectedOutputs {
+				if !strings.Contains(output, expected) {
+					t.Fatalf("Expected output to contain '%s' but it was:\n\n%s", expected, output)
+				}
+			}
+			return false, nil
+		})
+	}
+}

internal/cloud/backend_taskStage_taskResults.go

@@ -0,0 +1,147 @@
+package cloud
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/go-tfe"
+)
+
+type taskResultSummary struct {
+	unreachable     bool
+	pending         int
+	failed          int
+	failedMandatory int
+	passed          int
+}
+
+type taskResultSummarizer struct {
+	finished bool
+	cloud    *Cloud
+	counter  int
+}
+
+func newTaskResultSummarizer(b *Cloud, ts *tfe.TaskStage) taskStageSummarizer {
+	if len(ts.TaskResults) == 0 {
+		return nil
+	}
+	return &taskResultSummarizer{
+		finished: false,
+		cloud:    b,
+	}
+}
+
+func (trs *taskResultSummarizer) Summarize(context *IntegrationContext, output IntegrationOutputWriter, ts *tfe.TaskStage) (bool, *string, error) {
+	if trs.finished {
+		return false, nil, nil
+	}
+	trs.counter++
+
+	counts := summarizeTaskResults(ts.TaskResults)
+
+	if counts.pending != 0 {
+		pendingMessage := "%d tasks still pending, %d passed, %d failed ... "
+		message := fmt.Sprintf(pendingMessage, counts.pending, counts.passed, counts.failed)
+		return true, &message, nil
+	}
+	if counts.unreachable {
+		output.Output("Skipping task results.")
+		output.End()
+		return false, nil, nil
+	}
+
+	// Print out the summary
+	trs.runTasksWithTaskResults(output, ts.TaskResults, counts)
+
+	// Mark as finished
+	trs.finished = true
+
+	return false, nil, nil
+}
+
+func summarizeTaskResults(taskResults []*tfe.TaskResult) *taskResultSummary {
+	var pendingCount, errCount, errMandatoryCount, passedCount int
+	for _, task := range taskResults {
+		if task.Status == tfe.TaskUnreachable {
+			return &taskResultSummary{
+				unreachable: true,
+			}
+		} else if task.Status == tfe.TaskRunning || task.Status == tfe.TaskPending {
+			pendingCount++
+		} else if task.Status == tfe.TaskPassed {
+			passedCount++
+		} else {
+			// Everything else is a failure
+			errCount++
+			if task.WorkspaceTaskEnforcementLevel == tfe.Mandatory {
+				errMandatoryCount++
+			}
+		}
+	}
+
+	return &taskResultSummary{
+		unreachable:     false,
+		pending:         pendingCount,
+		failed:          errCount,
+		failedMandatory: errMandatoryCount,
+		passed:          passedCount,
+	}
+}
+
+func (trs *taskResultSummarizer) runTasksWithTaskResults(output IntegrationOutputWriter, taskResults []*tfe.TaskResult, count *taskResultSummary) {
+	// Track the first task name that is a mandatory enforcement level breach.
+	var firstMandatoryTaskFailed *string = nil
+
+	if trs.counter == 0 {
+		output.Output(fmt.Sprintf("All tasks completed! %d passed, %d failed", count.passed, count.failed))
+	} else {
+		output.OutputElapsed(fmt.Sprintf("All tasks completed! %d passed, %d failed", count.passed, count.failed), 50)
+	}
+
+	output.Output("")
+
+	for _, t := range taskResults {
+		capitalizedStatus := string(t.Status)
+		capitalizedStatus = strings.ToUpper(capitalizedStatus[:1]) + capitalizedStatus[1:]
+
+		status := "[green]" + capitalizedStatus
+		if t.Status != "passed" {
+			level := string(t.WorkspaceTaskEnforcementLevel)
+			level = strings.ToUpper(level[:1]) + level[1:]
+			status = fmt.Sprintf("[red]%s (%s)", capitalizedStatus, level)
+
+			if t.WorkspaceTaskEnforcementLevel == "mandatory" && firstMandatoryTaskFailed == nil {
+				firstMandatoryTaskFailed = &t.TaskName
+			}
+		}
+
+		title := fmt.Sprintf(`%s ⸺   %s`, t.TaskName, status)
+		output.SubOutput(title)
+
+		if len(t.Message) > 0 {
+			output.SubOutput(fmt.Sprintf("[dim]%s", t.Message))
+		}
+		if len(t.URL) > 0 {
+			output.SubOutput(fmt.Sprintf("[dim]Details: %s", t.URL))
+		}
+		output.SubOutput("")
+	}
+
+	// If a mandatory enforcement level is breached, return an error.
+	var overall string = "[green]Passed"
+	if firstMandatoryTaskFailed != nil {
+		overall = "[red]Failed"
+		if count.failedMandatory > 1 {
+			output.Output(fmt.Sprintf("[reset][bold][red]Error:[reset][bold]the run failed because %d mandatory tasks are required to succeed", count.failedMandatory))
+		} else {
+			output.Output(fmt.Sprintf("[reset][bold][red]Error: [reset][bold]the run failed because the run task, %s, is required to succeed", *firstMandatoryTaskFailed))
+		}
+	} else if count.failed > 0 { // we have failures but none of them mandatory
+		overall = "[green]Passed with advisory failures"
+	}
+
+	output.SubOutput("")
+	output.SubOutput("[bold]Overall Result: " + overall)
+
+	output.End()
+}

internal/cloud/backend_taskStage_taskResults_test.go

@@ -82,16 +82,20 @@ func TestCloud_runTasksWithTaskResults(t *testing.T) {
 	integrationContext, writer := newMockIntegrationContext(b, t)
 
 	cases := map[string]struct {
-		taskResults     []*tfe.TaskResult
+		taskStage       func() *tfe.TaskStage
 		context         *IntegrationContext
 		writer          *testIntegrationOutput
 		expectedOutputs []string
 		isError         bool
 	}{
 		"all-succeeded": {
-			taskResults: []*tfe.TaskResult{
+			taskStage: func() *tfe.TaskStage {
-				{ID: "1", TaskName: "Mandatory", Message: "A-OK", Status: "passed", WorkspaceTaskEnforcementLevel: "mandatory"},
+				ts := &tfe.TaskStage{}
-				{ID: "2", TaskName: "Advisory", Message: "A-OK", Status: "passed", WorkspaceTaskEnforcementLevel: "advisory"},
+				ts.TaskResults = []*tfe.TaskResult{
+					{ID: "1", TaskName: "Mandatory", Message: "A-OK", Status: "passed", WorkspaceTaskEnforcementLevel: "mandatory"},
+					{ID: "2", TaskName: "Advisory", Message: "A-OK", Status: "passed", WorkspaceTaskEnforcementLevel: "advisory"},
+				}
+				return ts
 			},
 			writer:          writer,
 			context:         integrationContext,
@@ -99,9 +103,13 @@ func TestCloud_runTasksWithTaskResults(t *testing.T) {
 			isError:         false,
 		},
 		"mandatory-failed": {
-			taskResults: []*tfe.TaskResult{
+			taskStage: func() *tfe.TaskStage {
-				{ID: "1", TaskName: "Mandatory", Message: "500 Error", Status: "failed", WorkspaceTaskEnforcementLevel: "mandatory"},
+				ts := &tfe.TaskStage{}
-				{ID: "2", TaskName: "Advisory", Message: "A-OK", Status: "passed", WorkspaceTaskEnforcementLevel: "advisory"},
+				ts.TaskResults = []*tfe.TaskResult{
+					{ID: "1", TaskName: "Mandatory", Message: "500 Error", Status: "failed", WorkspaceTaskEnforcementLevel: "mandatory"},
+					{ID: "2", TaskName: "Advisory", Message: "A-OK", Status: "passed", WorkspaceTaskEnforcementLevel: "advisory"},
+				}
+				return ts
 			},
 			writer:          writer,
 			context:         integrationContext,
@@ -109,9 +117,13 @@ func TestCloud_runTasksWithTaskResults(t *testing.T) {
 			isError:         true,
 		},
 		"advisory-failed": {
-			taskResults: []*tfe.TaskResult{
+			taskStage: func() *tfe.TaskStage {
-				{ID: "1", TaskName: "Mandatory", Message: "A-OK", Status: "passed", WorkspaceTaskEnforcementLevel: "mandatory"},
+				ts := &tfe.TaskStage{}
-				{ID: "2", TaskName: "Advisory", Message: "500 Error", Status: "failed", WorkspaceTaskEnforcementLevel: "advisory"},
+				ts.TaskResults = []*tfe.TaskResult{
+					{ID: "1", TaskName: "Mandatory", Message: "A-OK", Status: "passed", WorkspaceTaskEnforcementLevel: "mandatory"},
+					{ID: "2", TaskName: "Advisory", Message: "500 Error", Status: "failed", WorkspaceTaskEnforcementLevel: "advisory"},
+				}
+				return ts
 			},
 			writer:          writer,
 			context:         integrationContext,
@@ -119,9 +131,13 @@ func TestCloud_runTasksWithTaskResults(t *testing.T) {
 			isError:         false,
 		},
 		"unreachable": {
-			taskResults: []*tfe.TaskResult{
+			taskStage: func() *tfe.TaskStage {
-				{ID: "1", TaskName: "Mandatory", Message: "", Status: "unreachable", WorkspaceTaskEnforcementLevel: "mandatory"},
+				ts := &tfe.TaskStage{}
-				{ID: "2", TaskName: "Advisory", Message: "", Status: "unreachable", WorkspaceTaskEnforcementLevel: "advisory"},
+				ts.TaskResults = []*tfe.TaskResult{
+					{ID: "1", TaskName: "Mandatory", Message: "", Status: "unreachable", WorkspaceTaskEnforcementLevel: "mandatory"},
+					{ID: "2", TaskName: "Advisory", Message: "", Status: "unreachable", WorkspaceTaskEnforcementLevel: "advisory"},
+				}
+				return ts
 			},
 			writer:          writer,
 			context:         integrationContext,
@@ -130,27 +146,24 @@ func TestCloud_runTasksWithTaskResults(t *testing.T) {
 		},
 	}
 
-	for caseName, c := range cases {
+	for _, c := range cases {
 		c.writer.output.Reset()
-		err := b.runTasksWithTaskResults(c.context, writer, func(b *Cloud, stopCtx context.Context) (*tfe.TaskStage, error) {
+		trs := taskResultSummarizer{
-			return &tfe.TaskStage{
+			cloud: b,
-				TaskResults: c.taskResults,
-			}, nil
-		})
-
-		if c.isError && err == nil {
-			t.Fatalf("Expected %s to be error", caseName)
 		}
-
+		c.context.Poll(taskStageBackoffMin, taskStageBackoffMax, func(i int) (bool, error) {
-		if !c.isError && err != nil {
+			cont, _, _ := trs.Summarize(c.context, c.writer, c.taskStage())
-			t.Errorf("Expected %s to not be error but received %s", caseName, err)
+			if cont {
-		}
+				return true, nil
-
-		output := c.writer.output.String()
-		for _, expected := range c.expectedOutputs {
-			if !strings.Contains(output, expected) {
-				t.Fatalf("Expected output to contain '%s' but it was:\n\n%s", expected, output)
 			}
-		}
+
+			output := c.writer.output.String()
+			for _, expected := range c.expectedOutputs {
+				if !strings.Contains(output, expected) {
+					t.Fatalf("Expected output to contain '%s' but it was:\n\n%s", expected, output)
+				}
+			}
+			return false, nil
+		})
 	}
 }

internal/cloud/backend_taskStages.go

@@ -2,13 +2,36 @@ package cloud
 
 import (
 	"context"
+	"fmt"
 	"strings"
 
+	"github.com/hashicorp/go-multierror"
 	tfe "github.com/hashicorp/go-tfe"
+	"github.com/hashicorp/terraform/internal/terraform"
 )
 
 type taskStages map[tfe.Stage]*tfe.TaskStage
 
+const (
+	taskStageBackoffMin = 4000.0
+	taskStageBackoffMax = 12000.0
+)
+
+const taskStageHeader = `
+To view this run in a browser, visit:
+https://%s/app/%s/%s/runs/%s
+`
+
+type taskStageSummarizer interface {
+	// Summarize takes an IntegrationContext, IntegrationOutputWriter for
+	// writing output and a pointer to a tfe.TaskStage object as arguments.
+	// This function summarizes and outputs the results of the task stage.
+	// It returns a boolean which signifies whether we should continue polling
+	// for results, an optional message string to print while it is polling
+	// and an error if any.
+	Summarize(*IntegrationContext, IntegrationOutputWriter, *tfe.TaskStage) (bool, *string, error)
+}
+
 func (b *Cloud) runTaskStages(ctx context.Context, client *tfe.Client, runId string) (taskStages, error) {
 	taskStages := make(taskStages, 0)
 	result, err := client.Runs.ReadWithOptions(ctx, runId, &tfe.RunReadOptions{
@@ -30,3 +53,132 @@ func (b *Cloud) runTaskStages(ctx context.Context, client *tfe.Client, runId str
 
 	return taskStages, nil
 }
+
+func (b *Cloud) getTaskStageWithAllOptions(ctx *IntegrationContext, stageID string) (*tfe.TaskStage, error) {
+	options := tfe.TaskStageReadOptions{
+		Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults, tfe.PolicyEvaluationsTaskResults},
+	}
+	stage, err := b.client.TaskStages.Read(ctx.StopContext, stageID, &options)
+	if err != nil {
+		return nil, generalError("Failed to retrieve task stage", err)
+	} else {
+		return stage, nil
+	}
+}
+
+func (b *Cloud) runTaskStage(ctx *IntegrationContext, output IntegrationOutputWriter, stageID string) error {
+	var errs *multierror.Error
+
+	// Create our summarizers
+	summarizers := make([]taskStageSummarizer, 0)
+	ts, err := b.getTaskStageWithAllOptions(ctx, stageID)
+	if err != nil {
+		return err
+	}
+
+	if s := newTaskResultSummarizer(b, ts); s != nil {
+		summarizers = append(summarizers, s)
+	}
+
+	if s := newPolicyEvaluationSummarizer(b, ts); s != nil {
+		summarizers = append(summarizers, s)
+	}
+
+	return ctx.Poll(taskStageBackoffMin, taskStageBackoffMax, func(i int) (bool, error) {
+		options := tfe.TaskStageReadOptions{
+			Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults, tfe.PolicyEvaluationsTaskResults},
+		}
+		stage, err := b.client.TaskStages.Read(ctx.StopContext, stageID, &options)
+		if err != nil {
+			return false, generalError("Failed to retrieve task stage", err)
+		}
+
+		switch stage.Status {
+		case tfe.TaskStagePending:
+			// Waiting for it to start
+			return true, nil
+		// Note: Terminal statuses need to print out one last time just in case
+		case tfe.TaskStageRunning, tfe.TaskStagePassed:
+			ok, e := processSummarizers(ctx, output, stage, summarizers, errs)
+			if e != nil {
+				errs = e
+			}
+			if ok {
+				return true, nil
+			}
+		case tfe.TaskStageCanceled, tfe.TaskStageErrored, tfe.TaskStageFailed:
+			ok, e := processSummarizers(ctx, output, stage, summarizers, errs)
+			if e != nil {
+				errs = e
+			}
+			if ok {
+				return true, nil
+			}
+			return false, fmt.Errorf("Task Stage %s.", stage.Status)
+		case tfe.TaskStageAwaitingOverride:
+			ok, e := processSummarizers(ctx, output, stage, summarizers, errs)
+			if e != nil {
+				errs = e
+			}
+			if ok {
+				return true, nil
+			}
+			cont, err := b.processStageOverrides(ctx, output, stage.ID)
+			if err != nil {
+				errs = multierror.Append(errs, err)
+			} else {
+				return cont, nil
+			}
+		case tfe.TaskStageUnreachable:
+			return false, nil
+		default:
+			return false, fmt.Errorf("Invalid Task stage status: %s ", stage.Status)
+		}
+		return false, errs.ErrorOrNil()
+	})
+}
+
+func processSummarizers(ctx *IntegrationContext, output IntegrationOutputWriter, stage *tfe.TaskStage, summarizers []taskStageSummarizer, errs *multierror.Error) (bool, *multierror.Error) {
+	for _, s := range summarizers {
+		cont, msg, err := s.Summarize(ctx, output, stage)
+		if err != nil {
+			errs = multierror.Append(errs, err)
+			break
+		}
+
+		if !cont {
+			continue
+		}
+
+		// cont is true and we must continue to poll
+		if msg != nil {
+			output.OutputElapsed(*msg, len(*msg)) // Up to 2 digits are allowed by the max message allocation
+		}
+		return true, nil
+	}
+	return false, errs
+}
+
+func (b *Cloud) processStageOverrides(context *IntegrationContext, output IntegrationOutputWriter, taskStageID string) (bool, error) {
+	opts := &terraform.InputOpts{
+		Id:          fmt.Sprintf("%c%c [bold]Override", Arrow, Arrow),
+		Query:       "\nDo you want to override the failed policy check?",
+		Description: "Only 'override' will be accepted to override.",
+	}
+	runUrl := fmt.Sprintf(taskStageHeader, b.hostname, b.organization, context.Op.Workspace, context.Run.ID)
+	err := b.confirm(context.StopContext, context.Op, opts, context.Run, "override")
+	if err != nil && err != errRunOverridden {
+		return false, fmt.Errorf(
+			fmt.Sprintf("Failed to override: %s\n%s\n", err.Error(), runUrl),
+		)
+	}
+
+	if err != errRunOverridden {
+		if _, err = b.client.TaskStages.Override(context.StopContext, taskStageID, tfe.TaskStageOverrideOptions{}); err != nil {
+			return false, generalError(fmt.Sprintf("Failed to override policy check.\n%s", runUrl), err)
+		}
+	} else {
+		output.Output(fmt.Sprintf("The run needs to be manually overridden or discarded.\n%s\n", runUrl))
+	}
+	return false, nil
+}

internal/cloud/backend_taskStages_test.go

@@ -3,6 +3,7 @@ package cloud
 import (
 	"context"
 	"errors"
+	"strings"
 	"testing"
 
 	"github.com/golang/mock/gomock"
@@ -205,3 +206,63 @@ func TestTaskStagesWithErrors(t *testing.T) {
 		t.Error("Expected to error but did not")
 	}
 }
+
+func TestTaskStageOverride(t *testing.T) {
+	b, bCleanup := testBackendWithName(t)
+	defer bCleanup()
+
+	integrationContext, writer := newMockIntegrationContext(b, t)
+
+	integrationContext.Op.UIOut = b.CLI
+
+	cases := map[string]struct {
+		taskStageID string
+		isError     bool
+		errMsg      string
+		input       *mockInput
+	}{
+		"override-pass": {
+			taskStageID: "ts-pass",
+			isError:     false,
+			input: testInput(t, map[string]string{
+				"→→ [bold]Override": "override",
+			}),
+			errMsg: "",
+		},
+		"override-fail": {
+			taskStageID: "ts-err",
+			isError:     true,
+			input: testInput(t, map[string]string{
+				"→→ [bold]Override": "override",
+			}),
+			errMsg: "",
+		},
+		"skip-override": {
+			taskStageID: "ts-err",
+			isError:     true,
+			errMsg:      "Failed to override: Apply discarded.",
+			input: testInput(t, map[string]string{
+				"→→ [bold]Override": "no",
+			}),
+		},
+	}
+	for _, c := range cases {
+		integrationContext.Op.UIIn = c.input
+		_, err := b.processStageOverrides(integrationContext, writer, c.taskStageID)
+		if c.isError {
+			if err == nil {
+				t.Fatalf("Expected to fail with some error")
+			}
+			if c.errMsg != "" {
+				if !strings.Contains(err.Error(), c.errMsg) {
+					t.Fatalf("Expected: %s, got: %s", c.errMsg, err.Error())
+				}
+			}
+
+		} else {
+			if err != nil {
+				t.Fatalf("Expected to pass, got err: %s", err)
+			}
+		}
+	}
+}

internal/cloud/cloud_integration.go

@@ -38,14 +38,14 @@ type integrationCLIOutput struct {
 
 var _ IntegrationOutputWriter = (*integrationCLIOutput)(nil) // Compile time check
 
-func (s *IntegrationContext) Poll(every func(i int) (bool, error)) error {
+func (s *IntegrationContext) Poll(backoffMinInterval float64, backoffMaxInterval float64, every func(i int) (bool, error)) error {
 	for i := 0; ; i++ {
 		select {
 		case <-s.StopContext.Done():
 			return s.StopContext.Err()
 		case <-s.CancelContext.Done():
 			return s.CancelContext.Err()
-		case <-time.After(backoff(backoffMin, backoffMax, i)):
+		case <-time.After(backoff(backoffMinInterval, backoffMaxInterval, i)):
 			// blocks for a time between min and max
 		}
 

internal/cloud/testing.go

@@ -209,6 +209,8 @@ func testBackend(t *testing.T, obj cty.Value) (*Cloud, func()) {
 	b.client.CostEstimates = mc.CostEstimates
 	b.client.Organizations = mc.Organizations
 	b.client.Plans = mc.Plans
+	b.client.TaskStages = mc.TaskStages
+	b.client.PolicySetOutcomes = mc.PolicySetOutcomes
 	b.client.PolicyChecks = mc.PolicyChecks
 	b.client.Runs = mc.Runs
 	b.client.StateVersions = mc.StateVersions
@@ -269,6 +271,7 @@ func testUnconfiguredBackend(t *testing.T) (*Cloud, func()) {
 	b.client.CostEstimates = mc.CostEstimates
 	b.client.Organizations = mc.Organizations
 	b.client.Plans = mc.Plans
+	b.client.PolicySetOutcomes = mc.PolicySetOutcomes
 	b.client.PolicyChecks = mc.PolicyChecks
 	b.client.Runs = mc.Runs
 	b.client.StateVersions = mc.StateVersions

internal/cloud/tfe_client_mock.go

@@ -27,6 +27,8 @@ type MockClient struct {
 	CostEstimates         *MockCostEstimates
 	Organizations         *MockOrganizations
 	Plans                 *MockPlans
+	PolicySetOutcomes     *MockPolicySetOutcomes
+	TaskStages            *MockTaskStages
 	PolicyChecks          *MockPolicyChecks
 	Runs                  *MockRuns
 	StateVersions         *MockStateVersions
@@ -42,6 +44,8 @@ func NewMockClient() *MockClient {
 	c.CostEstimates = newMockCostEstimates(c)
 	c.Organizations = newMockOrganizations(c)
 	c.Plans = newMockPlans(c)
+	c.TaskStages = newMockTaskStages(c)
+	c.PolicySetOutcomes = newMockPolicySetOutcomes(c)
 	c.PolicyChecks = newMockPolicyChecks(c)
 	c.Runs = newMockRuns(c)
 	c.StateVersions = newMockStateVersions(c)
@@ -545,6 +549,164 @@ func (m *MockPlans) ReadJSONOutput(ctx context.Context, planID string) ([]byte,
 	return []byte(planOutput), nil
 }
 
+type MockTaskStages struct {
+	client *MockClient
+}
+
+func newMockTaskStages(client *MockClient) *MockTaskStages {
+	return &MockTaskStages{
+		client: client,
+	}
+}
+
+func (m *MockTaskStages) Override(ctx context.Context, taskStageID string, options tfe.TaskStageOverrideOptions) (*tfe.TaskStage, error) {
+	switch taskStageID {
+	case "ts-err":
+		return nil, errors.New("test error")
+
+	default:
+		return nil, nil
+	}
+}
+
+func (m *MockTaskStages) Read(ctx context.Context, taskStageID string, options *tfe.TaskStageReadOptions) (*tfe.TaskStage, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (m *MockTaskStages) List(ctx context.Context, runID string, options *tfe.TaskStageListOptions) (*tfe.TaskStageList, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+type MockPolicySetOutcomes struct {
+	client *MockClient
+}
+
+func newMockPolicySetOutcomes(client *MockClient) *MockPolicySetOutcomes {
+	return &MockPolicySetOutcomes{
+		client: client,
+	}
+}
+
+func (m *MockPolicySetOutcomes) List(ctx context.Context, policyEvaluationID string, options *tfe.PolicySetOutcomeListOptions) (*tfe.PolicySetOutcomeList, error) {
+	switch policyEvaluationID {
+	case "pol-pass":
+		return &tfe.PolicySetOutcomeList{
+			Items: []*tfe.PolicySetOutcome{
+				{
+					ID: policyEvaluationID,
+					Outcomes: []tfe.Outcome{
+						{
+							EnforcementLevel: "mandatory",
+							Query:            "data.example.rule",
+							Status:           "passed",
+							PolicyName:       "policy-pass",
+							Description:      "This policy will pass",
+						},
+					},
+					Overridable:          tfe.Bool(true),
+					Error:                "",
+					PolicySetName:        "policy-set-that-passes",
+					PolicySetDescription: "This policy set will always pass",
+					ResultCount: tfe.PolicyResultCount{
+						AdvisoryFailed:  0,
+						MandatoryFailed: 0,
+						Passed:          1,
+						Errored:         0,
+					},
+				},
+			},
+		}, nil
+	case "pol-fail":
+		return &tfe.PolicySetOutcomeList{
+			Items: []*tfe.PolicySetOutcome{
+				{
+					ID: policyEvaluationID,
+					Outcomes: []tfe.Outcome{
+						{
+							EnforcementLevel: "mandatory",
+							Query:            "data.example.rule",
+							Status:           "failed",
+							PolicyName:       "policy-fail",
+							Description:      "This policy will fail",
+						},
+					},
+					Overridable:          tfe.Bool(true),
+					Error:                "",
+					PolicySetName:        "policy-set-that-fails",
+					PolicySetDescription: "This policy set will always fail",
+					ResultCount: tfe.PolicyResultCount{
+						AdvisoryFailed:  0,
+						MandatoryFailed: 1,
+						Passed:          0,
+						Errored:         0,
+					},
+				},
+			},
+		}, nil
+
+	case "adv-fail":
+		return &tfe.PolicySetOutcomeList{
+			Items: []*tfe.PolicySetOutcome{
+				{
+					ID: policyEvaluationID,
+					Outcomes: []tfe.Outcome{
+						{
+							EnforcementLevel: "advisory",
+							Query:            "data.example.rule",
+							Status:           "failed",
+							PolicyName:       "policy-fail",
+							Description:      "This policy will fail",
+						},
+					},
+					Overridable:          tfe.Bool(true),
+					Error:                "",
+					PolicySetName:        "policy-set-that-fails",
+					PolicySetDescription: "This policy set will always fail",
+					ResultCount: tfe.PolicyResultCount{
+						AdvisoryFailed:  1,
+						MandatoryFailed: 0,
+						Passed:          0,
+						Errored:         0,
+					},
+				},
+			},
+		}, nil
+	default:
+		return &tfe.PolicySetOutcomeList{
+			Items: []*tfe.PolicySetOutcome{
+				{
+					ID: policyEvaluationID,
+					Outcomes: []tfe.Outcome{
+						{
+							EnforcementLevel: "mandatory",
+							Query:            "data.example.rule",
+							Status:           "passed",
+							PolicyName:       "policy-pass",
+							Description:      "This policy will pass",
+						},
+					},
+					Overridable:          tfe.Bool(true),
+					Error:                "",
+					PolicySetName:        "policy-set-that-passes",
+					PolicySetDescription: "This policy set will always pass",
+					ResultCount: tfe.PolicyResultCount{
+						AdvisoryFailed:  0,
+						MandatoryFailed: 0,
+						Passed:          1,
+						Errored:         0,
+					},
+				},
+			},
+		}, nil
+	}
+}
+
+func (m *MockPolicySetOutcomes) Read(ctx context.Context, policySetOutcomeID string) (*tfe.PolicySetOutcome, error) {
+	return nil, nil
+}
+
 type MockPolicyChecks struct {
 	client *MockClient
 	checks map[string]*tfe.PolicyCheck