fix: Exclude unsupported assertion results from failure count in repo…

…rt headings; use separate "unsupported" metric (#1265) * Update Test Results table headings to count 'MAY' assertion results separately
w3c · Nov 5, 2024 · 9c54e2d · 9c54e2d
1 parent 475230a
commit 9c54e2d
Show file tree

Hide file tree

Showing 7 changed files with 176 additions and 65 deletions.
diff --git a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx
@@ -507,16 +507,23 @@ const CandidateTestPlanRun = () => {
             const testResult =
               testPlanReport.finalizedTestResults[currentTestIndex];
 
-            const { assertionsPassedCount, assertionsFailedCount } = getMetrics(
-              { testResult }
-            );
+            const {
+              assertionsPassedCount,
+              mustAssertionsFailedCount,
+              shouldAssertionsFailedCount,
+              mayAssertionsFailedCount
+            } = getMetrics({ testResult });
+
+            const mustShouldAssertionsFailedCount =
+              mustAssertionsFailedCount + shouldAssertionsFailedCount;
 
             return (
               <>
                 <h2 className="test-results-header">
                   Test Results&nbsp;(
                   {assertionsPassedCount} passed,&nbsp;
-                  {assertionsFailedCount} failed)
+                  {mustShouldAssertionsFailedCount} failed,&nbsp;
+                  {mayAssertionsFailedCount} unsupported)
                 </h2>
                 <TestPlanResultsTable
                   key={`${testPlanReport.id} + ${testResult.id}`}

diff --git a/client/components/Reports/SummarizeTestPlanReport.jsx b/client/components/Reports/SummarizeTestPlanReport.jsx
@@ -272,17 +272,26 @@ const SummarizeTestPlanReport = ({ testPlanVersion, testPlanReports }) => {
           'https://aria-at.netlify.app'
         );
 
-        const { assertionsPassedCount, assertionsFailedCount } = getMetrics({
+        const {
+          assertionsPassedCount,
+          mustAssertionsFailedCount,
+          shouldAssertionsFailedCount,
+          mayAssertionsFailedCount
+        } = getMetrics({
           testResult
         });
 
+        const mustShouldAssertionsFailedCount =
+          mustAssertionsFailedCount + shouldAssertionsFailedCount;
+
         return (
           <Fragment key={testResult.id}>
             <div className="test-result-heading">
               <h2 id={`result-${testResult.id}`} tabIndex="-1">
                 Test {index + 1}: {test.title}&nbsp;(
-                {assertionsPassedCount}
-                &nbsp;passed, {assertionsFailedCount} failed)
+                {assertionsPassedCount} passed,&nbsp;
+                {mustShouldAssertionsFailedCount} failed,&nbsp;
+                {mayAssertionsFailedCount} unsupported)
                 <DisclaimerInfo phase={testPlanVersion.phase} />
               </h2>
               <div className="test-result-buttons">

diff --git a/client/components/TestRenderer/index.jsx b/client/components/TestRenderer/index.jsx
@@ -476,17 +476,26 @@ const TestRenderer = ({
     const { results } = submitResult;
     const { header } = results;
 
-    const { assertionsPassedCount, assertionsFailedCount } = getMetrics({
+    const {
+      assertionsPassedCount,
+      mustAssertionsFailedCount,
+      shouldAssertionsFailedCount,
+      mayAssertionsFailedCount
+    } = getMetrics({
       testResult
     });
 
+    const mustShouldAssertionsFailedCount =
+      mustAssertionsFailedCount + shouldAssertionsFailedCount;
+
     return (
       <>
         <HeadingText>{header}</HeadingText>
         <SubHeadingText id="overallstatus">
           Test Results&nbsp;(
           {assertionsPassedCount} passed,&nbsp;
-          {assertionsFailedCount} failed)
+          {mustShouldAssertionsFailedCount} failed,&nbsp;
+          {mayAssertionsFailedCount} unsupported)
         </SubHeadingText>
         <TestPlanResultsTable
           test={{ title: header, at }}

diff --git a/client/components/common/TestPlanResultsTable/index.jsx b/client/components/common/TestPlanResultsTable/index.jsx
@@ -44,11 +44,16 @@ const TestPlanResultsTable = ({
       {testResult.scenarioResults.map((scenarioResult, index) => {
         const {
           assertionsPassedCount,
-          assertionsFailedCount,
+          mustAssertionsFailedCount,
+          shouldAssertionsFailedCount,
+          mayAssertionsFailedCount,
           severeImpactPassedAssertionCount,
           moderateImpactPassedAssertionCount
         } = getMetrics({ scenarioResult });
 
+        const mustShouldAssertionsFailedCount =
+          mustAssertionsFailedCount + shouldAssertionsFailedCount;
+
         const hasNoSevereUnexpectedBehavior =
           severeImpactPassedAssertionCount > 0;
         const hasNoModerateUnexpectedBehavior =
@@ -119,7 +124,8 @@ const TestPlanResultsTable = ({
             <CommandHeading>
               {commandsString}&nbsp;Results:&nbsp;
               {assertionsPassedCount} passed,&nbsp;
-              {assertionsFailedCount} failed
+              {mustShouldAssertionsFailedCount} failed,&nbsp;
+              {mayAssertionsFailedCount} unsupported
             </CommandHeading>
             <p className="test-plan-results-response-p">
               {test.at?.name} Response:

diff --git a/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1.html b/client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1.html
@@ -501,10 +501,12 @@ <h1>
                               aria-labelledby="disclosure-btn-test-instructions-and-results-Test Results for Chrome"
                               class="css-19fsyrg">
                               <h2 class="test-results-header">
-                                Test Results&nbsp;(12 passed,&nbsp;0 failed)
+                                Test Results&nbsp;(12 passed,&nbsp;0
+                                failed,&nbsp;0 unsupported)
                               </h2>
                               <h3>
-                                Space&nbsp;Results:&nbsp;6 passed,&nbsp;0 failed
+                                Space&nbsp;Results:&nbsp;6 passed,&nbsp;0
+                                failed,&nbsp;0 unsupported
                               </h3>
                               <p class="test-plan-results-response-p">
                                 JAWS Response:
@@ -576,7 +578,8 @@ <h3>
                               Other behaviors that create negative impact: None
                               <hr aria-hidden="true" />
                               <h3>
-                                Enter&nbsp;Results:&nbsp;6 passed,&nbsp;0 failed
+                                Enter&nbsp;Results:&nbsp;6 passed,&nbsp;0
+                                failed,&nbsp;0 unsupported
                               </h3>
                               <p class="test-plan-results-response-p">
                                 JAWS Response: