add policy evaluation task stage

This commit is contained in:
mrinalirao 2022-11-29 15:10:23 +11:00
parent d7c7f3689c
commit 2be890a37c
5 changed files with 412 additions and 3 deletions

View File

@ -0,0 +1,157 @@
package cloud
import (
"fmt"
"strings"
"github.com/hashicorp/go-tfe"
)
type policyEvaluationSummary struct {
unreachable bool
pending int
failed int
passed int
}
type Symbol rune
const (
Tick Symbol = '\u2713'
Cross Symbol = '\u00d7'
Warning Symbol = '\u24be'
Arrow Symbol = '\u2192'
DownwardArrow Symbol = '\u21b3'
)
type policyEvaluationSummarizer struct {
finished bool
cloud *Cloud
counter int
}
func newPolicyEvaluationSummarizer(b *Cloud, ts *tfe.TaskStage) taskStageSummarizer {
if len(ts.PolicyEvaluations) == 0 {
return nil
}
return &policyEvaluationSummarizer{
finished: false,
cloud: b,
}
}
func (pes *policyEvaluationSummarizer) Summarize(context *IntegrationContext, output IntegrationOutputWriter, ts *tfe.TaskStage) (bool, *string, error) {
if pes.counter == 0 {
output.Output("[bold]OPA Policy Evaluation\n")
pes.counter++
}
if pes.finished {
return false, nil, nil
}
counts := summarizePolicyEvaluationResults(ts.PolicyEvaluations)
if counts.pending != 0 {
pendingMessage := "Evaluating ... "
return true, &pendingMessage, nil
}
if counts.unreachable {
output.Output("Skipping policy evaluation.")
output.End()
return false, nil, nil
}
// Print out the summary
if err := pes.taskStageWithPolicyEvaluation(context, output, ts.PolicyEvaluations); err != nil {
return false, nil, err
}
// Mark as finished
pes.finished = true
return false, nil, nil
}
func summarizePolicyEvaluationResults(policyEvaluations []*tfe.PolicyEvaluation) *policyEvaluationSummary {
var pendingCount, errCount, passedCount int
for _, policyEvaluation := range policyEvaluations {
switch policyEvaluation.Status {
case "unreachable":
return &policyEvaluationSummary{
unreachable: true,
}
case "running", "pending", "queued":
pendingCount++
case "passed":
passedCount++
default:
// Everything else is a failure
errCount++
}
}
return &policyEvaluationSummary{
unreachable: false,
pending: pendingCount,
failed: errCount,
passed: passedCount,
}
}
func (pes *policyEvaluationSummarizer) taskStageWithPolicyEvaluation(context *IntegrationContext, output IntegrationOutputWriter, policyEvaluation []*tfe.PolicyEvaluation) error {
var result, message string
// Currently only one policy evaluation supported : OPA
for _, polEvaluation := range policyEvaluation {
if polEvaluation.Status == tfe.PolicyEvaluationPassed {
message = "[dim] This result means that all OPA policies passed and the protected behaviour is allowed"
result = fmt.Sprintf("[green]%s", strings.ToUpper(string(tfe.PolicyEvaluationPassed)))
if polEvaluation.ResultCount.AdvisoryFailed > 0 {
result += " (with advisory)"
}
} else {
message = "[dim] This result means that one or more OPA policies failed. More than likely, this was due to the discovery of violations by the main rule and other sub rules"
result = fmt.Sprintf("[red]%s", strings.ToUpper(string(tfe.PolicyEvaluationFailed)))
}
output.Output(fmt.Sprintf("[bold]%c%c Overall Result: %s", Arrow, Arrow, result))
output.Output(message)
total := getPolicyCount(polEvaluation.ResultCount)
output.Output(fmt.Sprintf("%d policies evaluated\n", total))
policyOutcomes, err := pes.cloud.client.PolicySetOutcomes.List(context.StopContext, polEvaluation.ID, nil)
if err != nil {
return err
}
for i, out := range policyOutcomes.Items {
output.Output(fmt.Sprintf("%c Policy set %d: [bold]%s (%d)", Arrow, i+1, out.PolicySetName, len(out.Outcomes)))
for _, outcome := range out.Outcomes {
output.Output(fmt.Sprintf(" %c Policy name: [bold]%s", DownwardArrow, outcome.PolicyName))
switch outcome.Status {
case "passed":
output.Output(fmt.Sprintf(" | [green][bold]%c Passed", Tick))
case "failed":
if outcome.EnforcementLevel == tfe.EnforcementAdvisory {
output.Output(fmt.Sprintf(" | [blue][bold]%c Advisory", Warning))
} else {
output.Output(fmt.Sprintf(" | [red][bold]%c Failed", Cross))
}
}
if outcome.Description != "" {
output.Output(fmt.Sprintf(" | [dim]%s", outcome.Description))
} else {
output.Output(" | [dim]No description available")
}
}
}
}
return nil
}
func getPolicyCount(resultCount *tfe.PolicyResultCount) int {
return resultCount.AdvisoryFailed + resultCount.MandatoryFailed + resultCount.Errored + resultCount.Passed
}

View File

@ -0,0 +1,97 @@
package cloud
import (
"strings"
"testing"
"github.com/hashicorp/go-tfe"
)
func TestCloud_runTaskStageWithPolicyEvaluation(t *testing.T) {
b, bCleanup := testBackendWithName(t)
defer bCleanup()
integrationContext, writer := newMockIntegrationContext(b, t)
cases := map[string]struct {
taskStage func() *tfe.TaskStage
context *IntegrationContext
writer *testIntegrationOutput
expectedOutputs []string
isError bool
}{
"all-succeeded": {
taskStage: func() *tfe.TaskStage {
ts := &tfe.TaskStage{}
ts.PolicyEvaluations = []*tfe.PolicyEvaluation{
{ID: "pol-pass", ResultCount: &tfe.PolicyResultCount{Passed: 1}, Status: "passed"},
}
return ts
},
writer: writer,
context: integrationContext,
expectedOutputs: []string{"│ [bold]OPA Policy Evaluation\n\n│ [bold]→→ Overall Result: [green]PASSED\n│ [dim] This result means that all OPA policies passed and the protected behaviour is allowed\n│ 1 policies evaluated\n\n│ → Policy set 1: [bold]policy-set-that-passes (1)\n│ ↳ Policy name: [bold]policy-pass\n│ | [green][bold]✓ Passed\n│ | [dim]This policy will pass\n"},
isError: false,
},
"mandatory-failed": {
taskStage: func() *tfe.TaskStage {
ts := &tfe.TaskStage{}
ts.PolicyEvaluations = []*tfe.PolicyEvaluation{
{ID: "pol-fail", ResultCount: &tfe.PolicyResultCount{MandatoryFailed: 1}, Status: "failed"},
}
return ts
},
writer: writer,
context: integrationContext,
expectedOutputs: []string{"│ [bold]→→ Overall Result: [red]FAILED\n│ [dim] This result means that one or more OPA policies failed. More than likely, this was due to the discovery of violations by the main rule and other sub rules\n│ 1 policies evaluated\n\n│ → Policy set 1: [bold]policy-set-that-fails (1)\n│ ↳ Policy name: [bold]policy-fail\n│ | [red][bold]× Failed\n│ | [dim]This policy will fail"},
isError: true,
},
"advisory-failed": {
taskStage: func() *tfe.TaskStage {
ts := &tfe.TaskStage{}
ts.PolicyEvaluations = []*tfe.PolicyEvaluation{
{ID: "adv-fail", ResultCount: &tfe.PolicyResultCount{AdvisoryFailed: 1}, Status: "failed"},
}
return ts
},
writer: writer,
context: integrationContext,
expectedOutputs: []string{"│ [bold]OPA Policy Evaluation\n\n│ [bold]→→ Overall Result: [red]FAILED\n│ [dim] This result means that one or more OPA policies failed. More than likely, this was due to the discovery of violations by the main rule and other sub rules\n│ 1 policies evaluated\n\n│ → Policy set 1: [bold]policy-set-that-fails (1)\n│ ↳ Policy name: [bold]policy-fail\n│ | [blue][bold]Ⓘ Advisory\n│ | [dim]This policy will fail"},
isError: false,
},
"unreachable": {
taskStage: func() *tfe.TaskStage {
ts := &tfe.TaskStage{}
ts.PolicyEvaluations = []*tfe.PolicyEvaluation{
{ID: "adv-fail", ResultCount: &tfe.PolicyResultCount{Errored: 1}, Status: "unreachable"},
}
return ts
},
writer: writer,
context: integrationContext,
expectedOutputs: []string{"Skipping policy evaluation."},
isError: false,
},
}
for _, c := range cases {
c.writer.output.Reset()
trs := policyEvaluationSummarizer{
cloud: b,
}
c.context.Poll(func(i int) (bool, error) {
cont, _, _ := trs.Summarize(c.context, c.writer, c.taskStage())
if cont {
return true, nil
}
output := c.writer.output.String()
for _, expected := range c.expectedOutputs {
if !strings.Contains(output, expected) {
t.Fatalf("Expected output to contain '%s' but it was:\n\n%s", expected, output)
}
}
return false, nil
})
}
}

View File

@ -44,7 +44,7 @@ func (b *Cloud) runTaskStages(ctx context.Context, client *tfe.Client, runId str
func (b *Cloud) getTaskStageWithAllOptions(ctx *IntegrationContext, stageID string) (*tfe.TaskStage, error) {
options := tfe.TaskStageReadOptions{
Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults},
Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults, tfe.PolicyEvaluationsTaskResults},
}
stage, err := b.client.TaskStages.Read(ctx.StopContext, stageID, &options)
if err != nil {
@ -63,13 +63,18 @@ func (b *Cloud) runTaskStage(ctx *IntegrationContext, output IntegrationOutputWr
if err != nil {
return err
}
if s := newTaskResultSummarizer(b, ts); s != nil {
summarizers = append(summarizers, s)
}
if s := newPolicyEvaluationSummarizer(b, ts); s != nil {
summarizers = append(summarizers, s)
}
return ctx.Poll(func(i int) (bool, error) {
options := tfe.TaskStageReadOptions{
Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults},
Include: []tfe.TaskStageIncludeOpt{tfe.TaskStageTaskResults, tfe.PolicyEvaluationsTaskResults},
}
stage, err := b.client.TaskStages.Read(ctx.StopContext, stageID, &options)
if err != nil {
@ -98,7 +103,25 @@ func (b *Cloud) runTaskStage(ctx *IntegrationContext, output IntegrationOutputWr
errs.Append(err)
}
}
case "unreachable":
case tfe.TaskStageAwaitingOverride:
// TODO: Add override functionality
for _, s := range summarizers {
cont, msg, err := s.Summarize(ctx, output, stage)
if cont {
if msg != nil {
if i%4 == 0 {
if i > 0 {
output.OutputElapsed(*msg, len(*msg)) // Up to 2 digits are allowed by the max message allocation
}
}
}
return true, nil
}
if err != nil {
errs.Append(err)
}
}
case tfe.TaskStageUnreachable:
return false, nil
default:
return false, fmt.Errorf("Invalid Task stage status: %s ", stage.Status)

View File

@ -209,6 +209,7 @@ func testBackend(t *testing.T, obj cty.Value) (*Cloud, func()) {
b.client.CostEstimates = mc.CostEstimates
b.client.Organizations = mc.Organizations
b.client.Plans = mc.Plans
b.client.PolicySetOutcomes = mc.PolicySetOutcomes
b.client.PolicyChecks = mc.PolicyChecks
b.client.Runs = mc.Runs
b.client.StateVersions = mc.StateVersions
@ -269,6 +270,7 @@ func testUnconfiguredBackend(t *testing.T) (*Cloud, func()) {
b.client.CostEstimates = mc.CostEstimates
b.client.Organizations = mc.Organizations
b.client.Plans = mc.Plans
b.client.PolicySetOutcomes = mc.PolicySetOutcomes
b.client.PolicyChecks = mc.PolicyChecks
b.client.Runs = mc.Runs
b.client.StateVersions = mc.StateVersions

View File

@ -27,6 +27,7 @@ type MockClient struct {
CostEstimates *MockCostEstimates
Organizations *MockOrganizations
Plans *MockPlans
PolicySetOutcomes *MockPolicySetOutcomes
PolicyChecks *MockPolicyChecks
Runs *MockRuns
StateVersions *MockStateVersions
@ -42,6 +43,7 @@ func NewMockClient() *MockClient {
c.CostEstimates = newMockCostEstimates(c)
c.Organizations = newMockOrganizations(c)
c.Plans = newMockPlans(c)
c.PolicySetOutcomes = newMockPolicySetOutcomes(c)
c.PolicyChecks = newMockPolicyChecks(c)
c.Runs = newMockRuns(c)
c.StateVersions = newMockStateVersions(c)
@ -545,6 +547,134 @@ func (m *MockPlans) ReadJSONOutput(ctx context.Context, planID string) ([]byte,
return []byte(planOutput), nil
}
type MockPolicySetOutcomes struct {
client *MockClient
}
func newMockPolicySetOutcomes(client *MockClient) *MockPolicySetOutcomes {
return &MockPolicySetOutcomes{
client: client,
}
}
func (m *MockPolicySetOutcomes) List(ctx context.Context, policyEvaluationID string, options *tfe.PolicySetOutcomeListOptions) (*tfe.PolicySetOutcomeList, error) {
switch policyEvaluationID {
case "pol-pass":
return &tfe.PolicySetOutcomeList{
Items: []*tfe.PolicySetOutcome{
{
ID: policyEvaluationID,
Outcomes: []tfe.Outcome{
{
EnforcementLevel: "mandatory",
Query: "data.example.rule",
Status: "passed",
PolicyName: "policy-pass",
Description: "This policy will pass",
},
},
Overridable: tfe.Bool(true),
Error: "",
PolicySetName: "policy-set-that-passes",
PolicySetDescription: "This policy set will always pass",
ResultCount: tfe.PolicyResultCount{
AdvisoryFailed: 0,
MandatoryFailed: 0,
Passed: 1,
Errored: 0,
},
},
},
}, nil
case "pol-fail":
return &tfe.PolicySetOutcomeList{
Items: []*tfe.PolicySetOutcome{
{
ID: policyEvaluationID,
Outcomes: []tfe.Outcome{
{
EnforcementLevel: "mandatory",
Query: "data.example.rule",
Status: "failed",
PolicyName: "policy-fail",
Description: "This policy will fail",
},
},
Overridable: tfe.Bool(true),
Error: "",
PolicySetName: "policy-set-that-fails",
PolicySetDescription: "This policy set will always fail",
ResultCount: tfe.PolicyResultCount{
AdvisoryFailed: 0,
MandatoryFailed: 1,
Passed: 0,
Errored: 0,
},
},
},
}, nil
case "adv-fail":
return &tfe.PolicySetOutcomeList{
Items: []*tfe.PolicySetOutcome{
{
ID: policyEvaluationID,
Outcomes: []tfe.Outcome{
{
EnforcementLevel: "advisory",
Query: "data.example.rule",
Status: "failed",
PolicyName: "policy-fail",
Description: "This policy will fail",
},
},
Overridable: tfe.Bool(true),
Error: "",
PolicySetName: "policy-set-that-fails",
PolicySetDescription: "This policy set will always fail",
ResultCount: tfe.PolicyResultCount{
AdvisoryFailed: 1,
MandatoryFailed: 0,
Passed: 0,
Errored: 0,
},
},
},
}, nil
default:
return &tfe.PolicySetOutcomeList{
Items: []*tfe.PolicySetOutcome{
{
ID: policyEvaluationID,
Outcomes: []tfe.Outcome{
{
EnforcementLevel: "mandatory",
Query: "data.example.rule",
Status: "passed",
PolicyName: "policy-pass",
Description: "This policy will pass",
},
},
Overridable: tfe.Bool(true),
Error: "",
PolicySetName: "policy-set-that-passes",
PolicySetDescription: "This policy set will always pass",
ResultCount: tfe.PolicyResultCount{
AdvisoryFailed: 0,
MandatoryFailed: 0,
Passed: 1,
Errored: 0,
},
},
},
}, nil
}
}
func (m *MockPolicySetOutcomes) Read(ctx context.Context, policySetOutcomeID string) (*tfe.PolicySetOutcome, error) {
return nil, nil
}
type MockPolicyChecks struct {
client *MockClient
checks map[string]*tfe.PolicyCheck