diff mbox series

oeqa/selftest/gcc: Introduce SSH multiplexing for parallel tests

Message ID 20261112073756.1119386-1-Harish.Sadineni@windriver.com
State New
Headers show
Series oeqa/selftest/gcc: Introduce SSH multiplexing for parallel tests | expand

Commit Message

Sadineni, Harish Nov. 12, 2026, 7:37 a.m. UTC 
From: Harish Sadineni <Harish.Sadineni@windriver.com>

The removal of init-system-helpers-service from initscripts in
commit 674355759fdb[1] caused the service command to become
unavailable. As a result, restarting sshd failed, and the
configured MaxStartups setting was not applied. This caused
system-emulated GCC tests to fail with UNRESOLVED results.

Although this issue was addressed, intermittent SSH connection
failures were still observed when running tests in parallel
across builds which is addresed by ssh connection multiplexing.

Introduced SSH connection multiplexing to allow parallel GCC
test shards to share a master SSH connection, reducing
connection overhead and avoiding connection startup races
between ssh and scp.

Increase sshd MaxSessions to support the parallel sessions
required by GCC tests. Use a per-build ControlPath to avoid
conflicts between builds and clean up stale control sockets
before starting a new test run.

With SSH multiplexing, the GCC test build time was reduced
from approximately 8 hours to 3.5 hours on a local machine.

Test results:
+--------------------------------+---------+---------+---------+
| Metric                         |  Before |   After |   Diff  |
+--------------------------------+---------+---------+---------+
| # of expected passes           | 181458  | 210555  | +29097  |
| # of unexpected failures       |   3771  |     56  |  -3715  |
| # of unexpected successes      |      2  |      2  |      0  |
| # of expected failures         |   1512  |   1550  |    +38  |
| # of unresolved testcases      |  21501  |      0  | -21501  |
| # of unsupported tests         |   4102  |   3317  |   -785  |
+--------------------------------+---------+---------+---------+

Autobuilder results comparison with random old build and
new build with ssh multiplexing change.
+----------------+----------------------+---------------------+----------+
| Test           | w/o SSH multiplexing | w/ SSH multiplexing | Diff     |
+----------------+----------------------+---------------------+----------+
| test_cross_gcc | 40m 13s              | 21m 07s             | -19m 06s |
| test_libstdcxx | 30m 38s              | 28m 32s             | -2m 06s  |
| test_cross_gxx | 20m 49s              | 13m 24s             | -7m 25s  |
| test_libgomp   | 32m 36s              | 9m 09s              | -23m 27s |
| test_libatomic | 6m 52s               | 6m 36s              | -16s     |
| test_libitm    | 6m 04s               | 6m 29s              | +25s     |
+----------------+----------------------+---------------------+----------+
| cumulative gcc | ~2h 17m              | ~1h 25m             | -52m     |
+----------------+----------------------+---------------------+----------+

[1] https://git.openembedded.org/openembedded-core/commit/?id=674355759fdb07f3d17a053de18cddb7ce758c62

Signed-off-by: Harish Sadineni <Harish.Sadineni@windriver.com>
---
 meta/lib/oeqa/selftest/cases/gcc.py         | 22 ++++++++++++++++++---
 meta/recipes-devtools/gcc/gcc-testsuite.inc | 16 +++++++++++++--
 2 files changed, 33 insertions(+), 5 deletions(-)
diff mbox series

Patch

diff --git a/meta/lib/oeqa/selftest/cases/gcc.py b/meta/lib/oeqa/selftest/cases/gcc.py
index d01a791cb5..efae5bedf4 100644
--- a/meta/lib/oeqa/selftest/cases/gcc.py
+++ b/meta/lib/oeqa/selftest/cases/gcc.py
@@ -3,6 +3,8 @@ 
 #
 # SPDX-License-Identifier: MIT
 #
+import hashlib
+import subprocess
 import os
 import time
 from oeqa.core.decorator import OETestTag
@@ -84,10 +86,24 @@  class GccSelfTestBase(OESelftestTestCase, OEPTestResultTestCase):
             # validate that SSH is working
             status, _ = qemu.run("uname")
             self.assertEqual(status, 0)
-            qemu.run('echo "MaxStartups 75:30:100" >> /etc/ssh/sshd_config')
-            qemu.run('service sshd restart')
 
-            return self.run_check(*args, ssh=qemu.ip, **kwargs)
+            # Increase MaxSessions to accommodate concurrent GCC test shards sharing
+            # the SSH master connection.
+            qemu.run('echo "MaxSessions 200" >> /etc/ssh/sshd_config')
+            status, output = qemu.run('[ ! -f /var/run/sshd.pid ] || kill -HUP $(cat /var/run/sshd.pid)')
+            if status != 0:
+                raise Exception(f"sshd reload failed (rc={status}): {output}")
+
+            # With ControlPersist=yes, never times out on its own, so close it
+            # explicitly once all test shards have finished.
+            controlpath = "/tmp/oe-gcc-ssh-" + hashlib.sha1(get_bb_var("B", "gcc-runtime").encode()).hexdigest()[:12]
+            try:
+                return self.run_check(*args, ssh=qemu.ip, **kwargs)
+            finally:
+                subprocess.run(["ssh", "-O", "exit",
+                                "-o", "ControlPath={0}".format(controlpath),
+                                "root@{0}".format(qemu.ip)],
+                               stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 
 @OETestTag("toolchain-user")
 class GccCrossSelfTest(GccSelfTestBase):
diff --git a/meta/recipes-devtools/gcc/gcc-testsuite.inc b/meta/recipes-devtools/gcc/gcc-testsuite.inc
index 4859d6d84c..db08a01f3d 100644
--- a/meta/recipes-devtools/gcc/gcc-testsuite.inc
+++ b/meta/recipes-devtools/gcc/gcc-testsuite.inc
@@ -80,6 +80,7 @@  python check_prepare() {
         return "\n".join(content)
 
     def generate_remote_ssh_linux_config(d):
+        import hashlib, os
         content = []
         content.append('load_generic_config "unix"')
         content.append('process_multilib_options ""')
@@ -87,8 +88,19 @@  python check_prepare() {
         content.append("set_board_info username {0}".format(d.getVar("TOOLCHAIN_TEST_HOST_USER")))
 
         port = d.getVar("TOOLCHAIN_TEST_HOST_PORT")
-        content.append("set_board_info rsh_prog \"/usr/bin/ssh -p {0} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR \"".format(port))
-        content.append("set_board_info rcp_prog \"/usr/bin/scp -P {0} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR \"".format(port))
+
+        # Use SSH multiplexing to share one master connection across GCC tests.
+        # Keep the master alive until the selftest explicitly closes it.
+        # Use a per-build ControlPath to avoid conflicts between builds.
+        controlpath = "/tmp/oe-gcc-ssh-" + hashlib.sha1(d.getVar("B").encode()).hexdigest()[:12]
+        # remove stale socket left by a previous stopped run of this build
+        if os.path.exists(controlpath):
+            os.remove(controlpath)
+        rsh_mux = " -o ControlMaster=auto -o ControlPath={0} -o ControlPersist=yes".format(controlpath)
+        rcp_mux = " -o ControlMaster=no -o ControlPath={0}".format(controlpath)
+
+        content.append("set_board_info rsh_prog \"/usr/bin/ssh -p {0} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR{1} \"".format(port, rsh_mux))
+        content.append("set_board_info rcp_prog \"/usr/bin/scp -P {0} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR{1} \"".format(port, rcp_mux))
 
         return "\n".join(content)