test-level report granularity (#581)

Signed-off-by: Marcin Owsiany <[email protected]>

docs/testing/reference.md

@@ -35,6 +35,7 @@ artifactsDir      | string           | The directory to output artifacts to (cur
 commands          | list of [Commands](#commands) | Commands to run prior to running the tests.                                   | []
 kindContainers    | list of strings  | List of Docker images to load into the KIND cluster once it is started.                  | []
 reportFormat      | string           | Determines the report format. If empty, no report is generated. One of: JSON, XML.       |
+reportGranularity | string           | What granularity to report failures at. One of: `step`, `test`.                          | `step`
 reportName        | string           | The name of report to create. This field is not used unless reportFormat is set.         | "kuttl-test"
 namespace         | string           | The namespace to use for tests. This namespace will be created if it does not exist and removed if it was created (unless `skipDelete` is set). If no namespace is set, one will be auto-generated. |
 suppress          | list of strings  | Suppresses log collection of the specified types. Currently only `events` is supported.  |

pkg/apis/testharness/v1beta1/test_types.go

@@ -84,6 +84,10 @@ type TestSuite struct {
 
 	// ReportName defines the name of report to create.  It defaults to "kuttl-report" and is not used unless ReportFormat is defined.
 	ReportName string `json:"reportName"`
+
+	// ReportGranularity defines the granularity at which failures are reported. It defaults to "step".
+	ReportGranularity string `json:"reportGranularity"`
+
 	// Namespace defines the namespace to use for tests
 	// The value "" means to auto-generate tests namespaces, these namespaces will be created and removed for each test
 	// Any other value is the name of the namespace to use.  This namespace will be created if it does not exist and will

pkg/kuttlctl/cmd/test.go

@@ -59,6 +59,7 @@ func newTestCmd() *cobra.Command { //nolint:gocyclo
 	timeout := 30
 	reportFormat := ""
 	reportName := "kuttl-report"
+	reportGranularity := "kuttl-report"
 	namespace := ""
 	suppress := []string{}
 	var runLabels labelSetValue
@@ -183,6 +184,13 @@ For more detailed documentation, visit: https://kuttl.dev`,
 				options.ReportName = reportName
 			}
 
+			if isSet(flags, "report-granularity") {
+				if reportGranularity != "step" && reportGranularity != "test" {
+					return fmt.Errorf("unrecognized report granularity %q", reportGranularity)
+				}
+				options.ReportGranularity = reportGranularity
+			}
+
 			if isSet(flags, "artifacts-dir") {
 				options.ArtifactsDir = artifactsDir
 			}
@@ -257,6 +265,7 @@ For more detailed documentation, visit: https://kuttl.dev`,
 	testCmd.Flags().IntVar(&timeout, "timeout", 30, "The timeout to use as default for TestSuite configuration.")
 	testCmd.Flags().StringVar(&reportFormat, "report", "", "Specify JSON|XML for report.  Report location determined by --artifacts-dir.")
 	testCmd.Flags().StringVar(&reportName, "report-name", "kuttl-report", "Name for the report.  Report location determined by --artifacts-dir and report file type determined by --report.")
+	testCmd.Flags().StringVar(&reportGranularity, "report-granularity", "step", "Report granularity. Can be 'step' (default) or 'test'.")
 	testCmd.Flags().StringVarP(&namespace, "namespace", "n", "", "Namespace to use for tests. Provided namespaces must exist prior to running tests.")
 	testCmd.Flags().StringSliceVar(&suppress, "suppress-log", []string{}, "Suppress logging for these kinds of logs (events).")
 	testCmd.Flags().Var(&runLabels, "test-run-labels", "Labels to use for this test run.")

pkg/report/report.go

@@ -87,8 +87,9 @@ type Testsuite struct {
 	// Testcases is a collection of test cases.
 	Testcases []*Testcase `xml:"testcase" json:"testcase,omitempty"`
 	// SubSuites is a collection of child test suites.
-	SubSuites []*Testsuite `xml:"testsuite" json:"testsuite,omitempty"`
-	lock      sync.Mutex
+	SubSuites         []*Testsuite `xml:"testsuite" json:"testsuite,omitempty"`
+	lock              sync.Mutex
+	reportGranularity string
 }
 
 // Testsuites is a collection of Testsuite and defines the rollup summary of all stats.
@@ -135,9 +136,9 @@ func NewSuiteCollection(name string) *Testsuites {
 }
 
 // NewSuite returns the address of a newly created TestSuite
-func NewSuite(name string) *Testsuite {
+func NewSuite(name string, reportGranularity string) *Testsuite {
 	start := time.Now()
-	return &Testsuite{Name: name, Timestamp: start}
+	return &Testsuite{Name: name, Timestamp: start, reportGranularity: reportGranularity}
 }
 
 // NewCase returns the address of a newly create Testcase
@@ -190,7 +191,7 @@ func (ts *Testsuite) AddProperty(property Property) {
 
 // NewSubSuite creates a new child suite and returns it.
 func (ts *Testsuite) NewSubSuite(name string) *Testsuite {
-	s := NewSuite(name)
+	s := NewSuite(name, "")
 	ts.lock.Lock()
 	defer ts.lock.Unlock()
 	ts.SubSuites = append(ts.SubSuites, s)
@@ -223,9 +224,15 @@ func (ts *Testsuite) summarize() time.Time {
 	return end
 }
 
-func (ts *Testsuite) NewTest(name string) TestReporter {
-	subSuite := ts.NewSubSuite(name)
-	return &testReporter{suite: subSuite}
+func (ts *Testsuite) NewTestReporter(name string) TestReporter {
+	switch ts.reportGranularity {
+	case "test":
+		tc := NewCase(name)
+		return &testReporter{testCase: tc, suite: ts}
+	default:
+		subSuite := ts.NewSubSuite(name)
+		return &testReporter{suite: subSuite}
+	}
 }
 
 type stepReport struct {
@@ -246,9 +253,17 @@ func (s *stepReport) AddAssertions(i int) {
 	s.assertions += i
 }
 
+func (s *stepReport) populate(testCase *Testcase) {
+	if s.failed {
+		testCase.Failure = NewFailure(s.failureMsg, s.errors)
+	}
+	testCase.Assertions += s.assertions
+}
+
 type testReporter struct {
 	suite       *Testsuite
 	stepReports []*stepReport
+	testCase    *Testcase
 }
 
 func (r *testReporter) Step(stepName string) StepReporter {
@@ -258,12 +273,18 @@ func (r *testReporter) Step(stepName string) StepReporter {
 }
 
 func (r *testReporter) Done() {
+	if r.testCase != nil {
+		// Reporting with test granularity.
+		for _, report := range r.stepReports {
+			report.populate(r.testCase)
+		}
+		r.suite.AddTestcase(r.testCase)
+		return
+	}
+	// Reporting with step granularity.
 	for _, report := range r.stepReports {
 		testCase := NewCase(report.name)
-		if report.failed {
-			testCase.Failure = NewFailure(report.failureMsg, report.errors)
-		}
-		testCase.Assertions += report.assertions
+		report.populate(testCase)
 		r.suite.AddTestcase(testCase)
 	}
 }
@@ -346,13 +367,6 @@ func ensureDir(dir string) error {
 	return err
 }
 
-// NewSuite creates and assigns a TestSuite to the TestSuites (then returns the suite)
-func (ts *Testsuites) NewSuite(name string) *Testsuite {
-	suite := NewSuite(name)
-	ts.AddTestSuite(suite)
-	return suite
-}
-
 // SetFailure adds a failure to the TestSuites collection for startup failures in the test harness
 func (ts *Testsuites) SetFailure(message string) {
 	ts.Failure = &Failure{

pkg/test/harness.go

@@ -379,7 +379,7 @@ func (h *Harness) RunTests() {
 
 	h.T.Run("harness", func(t *testing.T) {
 		for testDir, tests := range realTestSuite {
-			suiteReport := h.report.NewSuite(testDir)
+			suiteReport := h.NewSuiteReport(testDir)
 			for _, test := range tests {
 				test := test
 
@@ -397,8 +397,7 @@ func (h *Harness) RunTests() {
 						t.Fatal(err)
 					}
 
-					testReporter := suiteReport.NewTest(test.Name)
-					test.Run(t, testReporter)
+					test.Run(t, suiteReport.NewTestReporter(test.Name))
 				})
 			}
 		}
@@ -612,6 +611,13 @@ func (h *Harness) Report() {
 	}
 }
 
+// NewSuiteReport creates and assigns a TestSuite to the TestSuites (then returns the suite),
+func (h *Harness) NewSuiteReport(name string) *report.Testsuite {
+	suite := report.NewSuite(name, h.TestSuite.ReportGranularity)
+	h.report.AddTestSuite(suite)
+	return suite
+}
+
 // reportName returns the configured ReportName.
 func (h *Harness) reportName() string {
 	if h.TestSuite.ReportName != "" {

test/junit/.gitignore

@@ -1,6 +1,12 @@
 /kuttl-ouput-step-json.txt
 /kuttl-ouput-step-xml.txt
+/kuttl-ouput-test-json.txt
+/kuttl-ouput-test-xml.txt
 /kuttl-report-step.json
 /kuttl-report-step.json.normalized
 /kuttl-report-step.xml
 /kuttl-report-step.xml.normalized
+/kuttl-report-test.json
+/kuttl-report-test.json.normalized
+/kuttl-report-test.xml
+/kuttl-report-test.xml.normalized

test/junit/Makefile

@@ -2,18 +2,28 @@
 test:
 	$(MAKE) -C ../../ cli
 	rm -f kuttl-report-step.xml.normalized kuttl-report-step.json.normalized
-	../../bin/kubectl-kuttl test --config /dev/null --start-control-plane --report-name kuttl-report-step --timeout 10 --report xml suite1 suite2 > kuttl-ouput-step-xml.txt 2>&1 || true # this is meant to fail
+	rm -f kuttl-report-test.xml.normalized kuttl-report-test.json.normalized
+	../../bin/kubectl-kuttl test --config /dev/null --start-control-plane --report-name kuttl-report-step --report-granularity step --timeout 10 --report xml suite1 suite2 > kuttl-ouput-step-xml.txt 2>&1 || true # this is meant to fail
 	if [ ! -e kuttl-report-step.xml ]; then cat kuttl-output-step-xml.txt; exit 1; fi
-	../../bin/kubectl-kuttl test --config /dev/null --start-control-plane --report-name kuttl-report-step --timeout 10 --report json suite1 suite2 > kuttl-ouput-step-json.txt 2>&1 || true # this is meant to fail
+	../../bin/kubectl-kuttl test --config /dev/null --start-control-plane --report-name kuttl-report-step --report-granularity step --timeout 10 --report json suite1 suite2 > kuttl-ouput-step-json.txt 2>&1 || true # this is meant to fail
 	if [ ! -e kuttl-report-step.json ]; then cat kuttl-output-step-json.txt; exit 1; fi
+	../../bin/kubectl-kuttl test --config /dev/null --start-control-plane --report-name kuttl-report-test --report-granularity test --timeout 10 --report xml suite1 suite2 > kuttl-ouput-test-xml.txt 2>&1 || true # this is meant to fail
+	if [ ! -e kuttl-report-test.xml ]; then cat kuttl-output-test-xml.txt; exit 1; fi
+	../../bin/kubectl-kuttl test --config /dev/null --start-control-plane --report-name kuttl-report-test --report-granularity test --timeout 10 --report json suite1 suite2 > kuttl-ouput-test-json.txt 2>&1 || true # this is meant to fail
+	if [ ! -e kuttl-report-test.json ]; then cat kuttl-output-test-json.txt; exit 1; fi
 	$(MAKE) kuttl-report-step.xml.normalized kuttl-report-step.json.normalized
+	$(MAKE) kuttl-report-test.xml.normalized kuttl-report-test.json.normalized
 	diff -u kuttl-report-step.xml.golden kuttl-report-step.xml.normalized
 	diff -u kuttl-report-step.json.golden kuttl-report-step.json.normalized
+	diff -u kuttl-report-test.xml.golden kuttl-report-test.xml.normalized
+	diff -u kuttl-report-test.json.golden kuttl-report-test.json.normalized
 
 .PHONY: update-golden
 update-golden:
 	cp kuttl-report-step.json.normalized kuttl-report-step.json.golden
 	cp kuttl-report-step.xml.normalized kuttl-report-step.xml.golden
+	cp kuttl-report-test.json.normalized kuttl-report-test.json.golden
+	cp kuttl-report-test.xml.normalized kuttl-report-test.xml.golden
 
 # The following targets replace all timestamps and durations with dummy values to make comparisons easy.
 

test/junit/kuttl-report-test.json.golden

@@ -0,0 +1,78 @@
+{
+   "name": "",
+   "tests": 6,
+   "failures": 4,
+   "time": "1.0",
+   "testsuite": [
+     {
+       "tests": 3,
+       "failures": 2,
+       "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+       "time": "1.0",
+       "name": "suite1",
+       "testcase": [
+         {
+           "classname": "suite1",
+           "name": "test0",
+           "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+           "time": "1.0"
+         },
+         {
+           "classname": "suite1",
+           "name": "test1",
+           "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+           "time": "1.0",
+           "failure": {
+             "text": "command \"echo step stdout\\\\n echo \u003e\u00262 step stderr\\\\n false\" failed, exit status 1",
+             "message": "failed in step 2-run"
+           }
+         },
+         {
+           "classname": "suite1",
+           "name": "test2",
+           "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+           "time": "1.0",
+           "failure": {
+             "text": "command \"echo assert stdout\\\\n echo \u003e\u00262 assert stderr\\\\n false\" failed, exit status 1",
+             "message": "failed in step 1-run"
+           }
+         }
+       ]
+     },
+     {
+       "tests": 3,
+       "failures": 2,
+       "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+       "time": "1.0",
+       "name": "suite2",
+       "testcase": [
+         {
+           "classname": "suite2",
+           "name": "test0",
+           "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+           "time": "1.0"
+         },
+         {
+           "classname": "suite2",
+           "name": "test1",
+           "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+           "time": "1.0",
+           "failure": {
+             "text": "command \"echo step stdout\\\\n echo \u003e\u00262 step stderr\\\\n false\" failed, exit status 1",
+             "message": "failed in step 2-run"
+           }
+         },
+         {
+           "classname": "suite2",
+           "name": "test2",
+           "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+           "time": "1.0",
+           "failure": {
+             "text": "command \"echo assert stdout\\\\n echo \u003e\u00262 assert stderr\\\\n false\" failed, exit status 1",
+             "message": "failed in step 1-run"
+           }
+         }
+       ]
+     }
+   ]
+ }

test/junit/kuttl-report-test.xml.golden

@@ -0,0 +1,20 @@
+ <testsuites name="" tests="6" failures="4" time="1.0">
+   <testsuite tests="3" failures="2" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" name="suite1">
+     <testcase classname="suite1" name="test0" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+     <testcase classname="suite1" name="test1" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0">
+       <failure message="failed in step 2-run" type="">command &#34;echo step stdout\\n echo &gt;&amp;2 step stderr\\n false&#34; failed, exit status 1</failure>
+     </testcase>
+     <testcase classname="suite1" name="test2" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0">
+       <failure message="failed in step 1-run" type="">command &#34;echo assert stdout\\n echo &gt;&amp;2 assert stderr\\n false&#34; failed, exit status 1</failure>
+     </testcase>
+   </testsuite>
+   <testsuite tests="3" failures="2" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" name="suite2">
+     <testcase classname="suite2" name="test0" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+     <testcase classname="suite2" name="test1" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0">
+       <failure message="failed in step 2-run" type="">command &#34;echo step stdout\\n echo &gt;&amp;2 step stderr\\n false&#34; failed, exit status 1</failure>
+     </testcase>
+     <testcase classname="suite2" name="test2" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0">
+       <failure message="failed in step 1-run" type="">command &#34;echo assert stdout\\n echo &gt;&amp;2 assert stderr\\n false&#34; failed, exit status 1</failure>
+     </testcase>
+   </testsuite>
+ </testsuites>