new file mode 100644
@@ -0,0 +1,29 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+from oeqa.core.case import OETestCase
+
+class FailuresATest(OETestCase):
+
+ def testFailuresA1(self):
+ self.fail("Failed on purpose")
+
+ def testFailuresA2(self):
+ pass
+
+ def testFailuresA3(self):
+ raise Exception("Error on purpose")
+
+class FailuresBTest(OETestCase):
+
+ def testFailuresB1(self):
+ pass
+
+ def testFailuresB2(self):
+ self.fail("Failed on purpose")
+
+ def testFailuresB3(self):
+ self.fail("Failed on purpose")
@@ -6,6 +6,7 @@
#
import os
+import re
import sys
import unittest
from unittest import mock
@@ -64,5 +65,20 @@ class TestConcurrency(TestBase):
self.assertIs(sys.stdout, stdout)
self.assertIs(sys.stderr, stderr)
+ def test_concurrent_failed_count(self):
+ # The failed count in the progress shown for each test must be the
+ # number of tests which failed so far in all processes
+ tc, result = self._run(['failures'], processes=2)
+ failed = {test.id() for test, _ in result.failures + result.errors}
+ self.assertEqual(len(failed), 4)
+ # progressinfo is in the order the results were received
+ seen = set()
+ for test_id, progress in result.progressinfo.items():
+ if test_id in failed:
+ seen.add(test_id)
+ count = int(re.search(r"\((\d+) failed\)", progress).group(1))
+ self.assertEqual(count, len(seen), msg=progress)
+ self.assertEqual(seen, failed)
+
if __name__ == '__main__':
unittest.main()
@@ -70,6 +70,11 @@ class BBThreadsafeForwardingResult(ThreadsafeForwardingResult):
self.semaphore.acquire()
try:
if self._test_start:
+ # Count failing tests across all the processes. This is called
+ # once per result, unlike the add* methods of the target which
+ # testtools may call twice.
+ if method.__name__ in ('addError', 'addFailure'):
+ self.finalresult.failedtests.add(test.id())
self.result.starttime[test.id()] = self._test_start.timestamp()
self.result.threadprogress[self.threadnum].append(test.id())
totalprogress = sum(len(x) for x in self.result.threadprogress.values())
@@ -80,7 +85,7 @@ class BBThreadsafeForwardingResult(ThreadsafeForwardingResult):
totalprogress,
self.totaltests,
"{0:.2f}".format(time.time()-self._test_start.timestamp()),
- self.target.failed_tests,
+ len(self.finalresult.failedtests),
test.id())
finally:
self.semaphore.release()
@@ -223,6 +228,7 @@ class ConcurrentTestSuite(unittest.TestSuite):
queue = Queue()
semaphore = threading.Semaphore(1)
result.threadprogress = {}
+ result.failedtests = set()
for i, (testserver, testnum, output) in enumerate(testservers):
result.threadprogress[i] = []
process_result = BBThreadsafeForwardingResult(
The "(N failed)" count shown in the progress line for each test when running tests concurrently came from the result proxy for the process which ran the test. It was only that process' count, was read before the current test's result was counted, and counted each failure twice since testtools first passes a failure with details, which OETestResult rejects, and then retries without. Count the failing tests across all processes on the final result in _add_result_with_semaphore(), which is called once per result under the semaphore shared by the reader threads, before the progress line is formatted. Add a test which checks the count in each progress line. AI-Generated: Uses Claude Opus 5.5 Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> --- meta/lib/oeqa/core/tests/cases/failures.py | 29 ++++++++++++++++++++ meta/lib/oeqa/core/tests/test_concurrency.py | 16 +++++++++++ meta/lib/oeqa/core/utils/concurrencytest.py | 8 +++++- 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 meta/lib/oeqa/core/tests/cases/failures.py