diff mbox series

[30/41] oeqa/core/runner: Report the status of tests with failing subtests

Message ID 20261115084845.701136-30-richard.purdie@linuxfoundation.org
State New
Headers show
Series [01/41] oeqa/loader: Drop code for pre python 3.5 | expand

Commit Message

Richard Purdie Nov. 15, 2026, 8:48 a.m. UTC 
unittest records subtest failures against a subtest object rather than
the test case, so the result lookups never matched them. Tests with
failing subtests were reported as UNKNOWN in the results and their
failure messages weren't shown as the test finished. Match subtests to
their test case and add a test.

AI-Generated: Uses Claude Opus 5.5
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/lib/oeqa/core/runner.py              |  9 +++++++--
 meta/lib/oeqa/core/tests/cases/subtest.py | 19 +++++++++++++++++++
 meta/lib/oeqa/core/tests/test_runner.py   | 13 +++++++++++++
 3 files changed, 39 insertions(+), 2 deletions(-)
 create mode 100644 meta/lib/oeqa/core/tests/cases/subtest.py
diff mbox series

Patch

diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
index 9164ee6bcca..a9b416f7cd7 100644
--- a/meta/lib/oeqa/core/runner.py
+++ b/meta/lib/oeqa/core/runner.py
@@ -15,6 +15,11 @@  import sys
 from unittest import TextTestResult as _TestResult
 from unittest import TextTestRunner as _TestRunner
 
+def _case_id(case):
+    # Failures in subtests are recorded against a subtest object, use the
+    # id of the test case it belongs to
+    return getattr(case, 'test_case', case).id()
+
 class OEStreamLogger:
     def __init__(self, logger):
         self.logger = logger
@@ -73,7 +78,7 @@  class OETestResult(_TestResult):
         # to wait until selftest finishes to see them.
         for t in ['failures', 'errors', 'skipped', 'expectedFailures']:
             for (scase, msg) in getattr(self, t):
-                if test.id() == scase.id():
+                if test.id() == _case_id(scase):
                     self.tc.logger.info(str(msg))
                     self.shownmsg.append(test.id())
                     break
@@ -109,7 +114,7 @@  class OETestResult(_TestResult):
                     scase, msg = resultclass
                 else:
                     scase, msg = resultclass, None
-                if case.id() == scase.id():
+                if case.id() == _case_id(scase):
                     found = True
                     break
                 scase_str = str(scase.id())
diff --git a/meta/lib/oeqa/core/tests/cases/subtest.py b/meta/lib/oeqa/core/tests/cases/subtest.py
new file mode 100644
index 00000000000..d24135f7c98
--- /dev/null
+++ b/meta/lib/oeqa/core/tests/cases/subtest.py
@@ -0,0 +1,19 @@ 
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+from oeqa.core.case import OETestCase
+
+class SubTestTest(OETestCase):
+
+    def testSubTestFail(self):
+        for i in range(2):
+            with self.subTest(i=i):
+                self.assertEqual(i, 0)
+
+    def testSubTestPass(self):
+        for i in range(2):
+            with self.subTest(i=i):
+                self.assertTrue(True, msg='How is this possible?')
diff --git a/meta/lib/oeqa/core/tests/test_runner.py b/meta/lib/oeqa/core/tests/test_runner.py
index 205464cfae9..ce7be7b14e3 100755
--- a/meta/lib/oeqa/core/tests/test_runner.py
+++ b/meta/lib/oeqa/core/tests/test_runner.py
@@ -36,5 +36,18 @@  class TestRunner(TestBase):
 
         fp.close()
 
+class TestResultStatus(TestBase):
+    def _statuses(self, modules):
+        tc = self._testLoader(modules=modules)
+        result = tc.runTests()
+        return {case_id: result._getTestResultDetails(case)[0]
+                for case_id, case in tc._registry['cases'].items()}
+
+    def test_subtest_status(self):
+        self.assertEqual(self._statuses(['subtest']), {
+            'subtest.SubTestTest.testSubTestFail': 'FAILED',
+            'subtest.SubTestTest.testSubTestPass': 'PASSED',
+        })
+
 if __name__ == '__main__':
     unittest.main()