Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Adjust test report of error cases #2

Merged
merged 2 commits into from
Sep 21, 2024
Merged

fix: Adjust test report of error cases #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d6a01ad
fix: Adjust testResult structs for better result report
hrk091 Sep 21, 2024
1d3cee8
refactor: Extract summaryLine func
hrk091 Sep 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
199 changes: 115 additions & 84 deletions internal/tester/result.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,28 @@ type testResult interface {
String(verbose bool) string
}

var _ testResult = &policyEvalResult{}
func summaryLine(pass bool, policy string, testCase TestCase, result string) string {
var summary string
if pass {
summary = "PASS"
} else {
summary = "FAIL"
}

summary += fmt.Sprintf(": %s", policy)
if testCase.Object.IsValid() && testCase.OldObject.IsValid() { //nolint:gocritic
summary += fmt.Sprintf(" - (UPDATE) %s -> %s", testCase.OldObject.String(), testCase.Object.NamespacedName.String())
} else if testCase.Object.IsValid() {
summary += fmt.Sprintf(" - (CREATE) %s", testCase.Object.String())
} else if testCase.OldObject.IsValid() {
summary += fmt.Sprintf(" - (DELETE) %s", testCase.OldObject.String())
}
if testCase.Param.IsValid() {
summary += fmt.Sprintf(" (Param: %s)", testCase.Param.String())
}
summary += fmt.Sprintf(" - %s ==> %s", strings.ToUpper(string(testCase.Expect)), result)
return summary
}

type policyEvalResult struct {
Policy string
Expand All @@ -40,6 +61,8 @@ type policyEvalResult struct {
Result validating.PolicyDecisionEvaluation
}

var _ testResult = &policyEvalResult{}

func newPolicyEvalResult(policy string, tc TestCase, decisions []validating.PolicyDecision) *policyEvalResult {
result := validating.EvalAdmit
for _, d := range decisions {
Expand All @@ -64,41 +87,70 @@ func (r *policyEvalResult) Pass() bool {
}

func (r *policyEvalResult) String(verbose bool) string {
var summary string
if r.Pass() {
summary = "PASS"
} else {
summary = "FAIL"
summary := summaryLine(r.Pass(), r.Policy, r.TestCase, string(r.Result))
out := []string{summary}
if !r.Pass() || verbose {
for _, d := range r.Decisions {
// Workaround to handle the case where the evaluation is not set
// TODO remove this workaround after htcps://github.com/kubernetes/kubernetes/pull/126867 is released
if d.Evaluation == "" {
d.Evaluation = validating.EvalDeny
}
if d.Evaluation == validating.EvalDeny {
out = append(out, fmt.Sprintf("--- DENY: reason %q, message %q", d.Reason, d.Message))
} else if d.Evaluation == validating.EvalError {
out = append(out, fmt.Sprintf("--- ERROR: reason %q, message %q", d.Reason, d.Message))
}
}
}
return strings.Join(out, "\n")
}

type policyNotFoundResult struct {
Policy string
}

var _ testResult = &policyNotFoundResult{}

summary += fmt.Sprintf(": %s", r.Policy)
if r.TestCase.Object.IsValid() && r.TestCase.OldObject.IsValid() { //nolint:gocritic
summary += fmt.Sprintf(" - (UPDATE) %s -> %s", r.TestCase.OldObject.String(), r.TestCase.Object.NamespacedName.String())
} else if r.TestCase.Object.IsValid() {
summary += fmt.Sprintf(" - (CREATE) %s", r.TestCase.Object.String())
} else if r.TestCase.OldObject.IsValid() {
summary += fmt.Sprintf(" - (DELETE) %s", r.TestCase.OldObject.String())
func newPolicyNotFoundResult(policy string) *policyNotFoundResult {
return &policyNotFoundResult{
Policy: policy,
}
if r.TestCase.Param.IsValid() {
summary += fmt.Sprintf(" (Param: %s)", r.TestCase.Param.String())
}

func (r *policyNotFoundResult) Pass() bool {
return false
}

func (r *policyNotFoundResult) String(verbose bool) string {
return fmt.Sprintf("FAIL: %s ==> POLICY NOT FOUND", r.Policy)
}

type setupErrorResult struct {
Policy string
TestCase TestCase
Errors []error
}

var _ testResult = &setupErrorResult{}

func newSetupErrorResult(policy string, tc TestCase, errs []error) *setupErrorResult {
return &setupErrorResult{
Policy: policy,
TestCase: tc,
Errors: errs,
}
summary += fmt.Sprintf(" - %s ==> %s", strings.ToUpper(string(r.TestCase.Expect)), strings.ToUpper(string(r.Result)))
}

func (r *setupErrorResult) Pass() bool {
return false
}

func (r *setupErrorResult) String(verbose bool) string {
summary := summaryLine(r.Pass(), r.Policy, r.TestCase, "SETUP ERROR")
out := []string{summary}
for _, d := range r.Decisions {
if r.Pass() && !verbose {
continue
}
// Workaround to handle the case where the evaluation is not set
// TODO remove this workaround after htcps://github.com/kubernetes/kubernetes/pull/126867 is released
if d.Evaluation == "" {
d.Evaluation = validating.EvalDeny
}
if d.Evaluation == validating.EvalDeny {
out = append(out, fmt.Sprintf("--- DENY: reason %q, message %q", d.Reason, d.Message))
} else if d.Evaluation == validating.EvalError {
out = append(out, fmt.Sprintf("--- ERROR: reason %q, message %q", d.Reason, d.Message))
}
for _, err := range r.Errors {
out = append(out, fmt.Sprintf("--- ERROR: %v", err))
}
return strings.Join(out, "\n")
}
Expand All @@ -124,90 +176,69 @@ func (r *policyNotMatchConditionResult) Pass() bool {
}

func (r *policyNotMatchConditionResult) String(verbose bool) string {
var summary string
if r.Pass() {
summary = "PASS"
} else {
summary = "FAIL"
}

summary += fmt.Sprintf(": %s", r.Policy)
if r.TestCase.Object.IsValid() && r.TestCase.OldObject.IsValid() { //nolint:gocritic
summary += fmt.Sprintf(" - (UPDATE) %s -> %s", r.TestCase.OldObject.String(), r.TestCase.Object.NamespacedName.String())
} else if r.TestCase.Object.IsValid() {
summary += fmt.Sprintf(" - (CREATE) %s", r.TestCase.Object.String())
} else if r.TestCase.OldObject.IsValid() {
summary += fmt.Sprintf(" - (DELETE) %s", r.TestCase.OldObject.String())
}
if r.TestCase.Param.IsValid() {
summary += fmt.Sprintf(" (Param: %s)", r.TestCase.Param.String())
}
if r.Pass() {
summary += fmt.Sprintf(" - %s ==> %s", "SKIP", "SKIP")
} else {
summary += fmt.Sprintf(" - %s ==> %s", strings.ToUpper(string(r.TestCase.Expect)), "NOT MATCH")
}

summary := summaryLine(r.Pass(), r.Policy, r.TestCase, "SKIP")
out := []string{summary}
out = append(out, fmt.Sprintf("--- NOT MATCH: condition-name %q", r.FailedConditionName))
if !r.Pass() || verbose {
out = append(out, fmt.Sprintf("--- NOT MATCH: condition-name %q", r.FailedConditionName))
}

return strings.Join(out, "\n")
}

type policyNotFoundResult struct {
Policy string
type policyEvalErrorResult struct {
Policy string
TestCase TestCase
Errors []error
}

var _ testResult = &policyNotFoundResult{}
var _ testResult = &policyEvalErrorResult{}

func newPolicyNotFoundResult(policy string) *policyNotFoundResult {
return &policyNotFoundResult{
Policy: policy,
func newPolicyEvalErrorResult(policy string, tc TestCase, errs []error) *policyEvalErrorResult {
return &policyEvalErrorResult{
Policy: policy,
TestCase: tc,
Errors: errs,
}
}

func (r *policyNotFoundResult) Pass() bool {
return false
func (r *policyEvalErrorResult) Pass() bool {
return r.TestCase.Expect == Error
}

func (r *policyNotFoundResult) String(verbose bool) string {
return fmt.Sprintf("FAIL: %s ==> POLICY NOT FOUND", r.Policy)
func (r *policyEvalErrorResult) String(verbose bool) string {
summary := summaryLine(r.Pass(), r.Policy, r.TestCase, "ERROR")
out := []string{summary}
if !r.Pass() || verbose {
for _, err := range r.Errors {
out = append(out, fmt.Sprintf("--- ERROR: %v", err))
}
}

return strings.Join(out, "\n")
}

type policyEvalErrorResult struct {
type policyEvalFatalErrorResult struct {
Policy string
TestCase TestCase
Errors []error
}

var _ testResult = &policyEvalErrorResult{}
var _ testResult = &policyEvalFatalErrorResult{}

func newPolicyEvalErrorResult(policy string, tc TestCase, errs []error) *policyEvalErrorResult {
return &policyEvalErrorResult{
func newPolicyEvalFatalErrorResult(policy string, tc TestCase, errs []error) *policyEvalFatalErrorResult {
return &policyEvalFatalErrorResult{
Policy: policy,
TestCase: tc,
Errors: errs,
}
}

func (r *policyEvalErrorResult) Pass() bool {
func (r *policyEvalFatalErrorResult) Pass() bool {
return false
}

func (r *policyEvalErrorResult) String(verbose bool) string {
summary := fmt.Sprintf("FAIL: %s", r.Policy)
if r.TestCase.Object.IsValid() && r.TestCase.OldObject.IsValid() { //nolint:gocritic
summary += fmt.Sprintf(" - %s -> %s ", r.TestCase.Object.String(), r.TestCase.OldObject.NamespacedName.String())
} else if r.TestCase.Object.IsValid() {
summary += fmt.Sprintf(" - %s", r.TestCase.Object.String())
} else if r.TestCase.OldObject.IsValid() {
summary += fmt.Sprintf(" - %s", r.TestCase.OldObject.String())
}
if r.TestCase.Param.IsValid() {
summary += fmt.Sprintf(" (Param: %s)", r.TestCase.Param.String())
}
summary += fmt.Sprintf(" - %s ==> %s", strings.ToUpper(string(r.TestCase.Expect)), "SETUP ERROR")

func (r *policyEvalFatalErrorResult) String(verbose bool) string {
summary := summaryLine(r.Pass(), r.Policy, r.TestCase, "FATAL ERROR")
out := []string{summary}
for _, err := range r.Errors {
out = append(out, fmt.Sprintf("--- ERROR: %v", err))
Expand Down
4 changes: 2 additions & 2 deletions internal/tester/tester.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ func runEach(cfg CmdConfig, manifestPath string) testResultSummary {
// Setup params for validation
given, errs := newValidationParams(vap, tc, loader)
if len(errs) > 0 {
results = append(results, newPolicyEvalErrorResult(tt.Policy, tc, errs))
results = append(results, newSetupErrorResult(tt.Policy, tc, errs))
continue
}

Expand All @@ -145,7 +145,7 @@ func runEach(cfg CmdConfig, manifestPath string) testResultSummary {
slog.Debug("RUN: ", "policy", tt.Policy, "expect", tc.Expect, "object", tc.Object.String(), "oldObject", tc.OldObject.String(), "param", tc.Param.String())
validationResult, err := validator.Validate(given)
if err != nil {
results = append(results, newPolicyEvalErrorResult(tt.Policy, tc, []error{err}))
results = append(results, newPolicyEvalFatalErrorResult(tt.Policy, tc, []error{err}))
continue
}

Expand Down