diff mbox series

[auh,2/3] modules/steps.py: refactor compile() to use parallel per-machine builds

Message ID 20260414053202.3869771-2-Qi.Chen@windriver.com
State New
Headers show
Series [auh,1/3] upgrade-helper.py: remove gcc-runtime pre-build step from upgrade loop | expand

Commit Message

ChenQi April 14, 2026, 5:32 a.m. UTC
From: Chen Qi <Qi.Chen@windriver.com>

Reasons:

1. Building recipes one by one consumes little build resource. In practice,
   when building one recipe machine by machine in a serialized way, the
   machine load is very very low. This means AUH is not making use of
   the machines' computing resources.
2. Yocto project has good and mature technique to control the build to
   not exhaust machines' resource, e.g., BB_NUMBER_THREADS, BB_PRESSURE_MAX_XXX.
   So we really don't need to worry about AUH exhausting server resources.

Technical Details:

1. Each machine gets its own build directory ($BUILDDIR-{machine}) with
   BUILDDIR and BBPATH env vars set accordingly, and conf/ copied from the
   original build directory. All machines compile in parallel via
   multiprocessing.Pool, with errors collected and reported after all
   workers complete.

2. Use original build directory for the first machine in parallel build.
   Skip creating a separate build directory for the first machine and
   build directly in the original BUILDDIR. Only the remaining machines
   get their own $BUILDDIR-{machine} copies. The purpose is that if testimage
   is enabled, then the build can just the previous results.

3. Always refresh per-machine conf/ directory before compilation. This
   ensure anything new in the main build directory will get into new
   build directory.

4. Add --serial-build option to use sequential per-machine compilation.
   When --serial-build is passed, compile each machine serially using the
   original build logic instead of the parallel multiprocessing approach.

Note:

After this change, AUH will use parallel build by default. The '--serial-build'
is there if users really need to use serial build for some reason.

Signed-off-by: Chen Qi <Qi.Chen@windriver.com>
---
 modules/steps.py  | 96 +++++++++++++++++++++++++++++++++++++----------
 upgrade-helper.py |  3 ++
 2 files changed, 80 insertions(+), 19 deletions(-)
diff mbox series

Patch

diff --git a/modules/steps.py b/modules/steps.py
index a6ec341..300ae72 100644
--- a/modules/steps.py
+++ b/modules/steps.py
@@ -22,10 +22,12 @@ 
 import os
 import shutil
 import re
+import multiprocessing
 
 from logging import debug as D
 from logging import info as I
 from logging import warning as W
+from logging import error as E
 
 from errors import Error, DevtoolError, CompilationError
 from buildhistory import BuildHistory
@@ -104,31 +106,87 @@  def devtool_upgrade(devtool, bb, git, opts, group):
     for p in group['pkgs']:
         _devtool_upgrade(devtool, bb, git, opts, p)
 
-def _compile(bb, pkg, machine, workdir):
-        try:
-            bb.complete(pkg, machine)
-        except Error as e:
-            with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
-                f.write(e.stdout + e.stderr)
-            for line in e.stdout.split("\n") + e.stderr.split("\n"):
-                # version going backwards is not a real error
-                if re.match(".* went backwards which would break package feeds .*", line):
-                    break
-                # 'not in COMPATIBLE_HOST/MACHINE is not a real error
-                if re.match(".*not in COMPATIBLE.*", line):
-                    break
-            else:
-                raise CompilationError()
+def _compile_worker(args):
+    """Worker function for multiprocess compilation. Runs in a separate process."""
+    pkg, machine, workdir, orig_builddir, use_orig = args
+    from utils.bitbake import Bitbake
+
+    if use_orig:
+        builddir = orig_builddir
+    else:
+        builddir = "{}-{}".format(orig_builddir, machine)
+        os.makedirs(builddir, exist_ok=True)
+
+        new_conf = os.path.join(builddir, "conf")
+        if os.path.isdir(new_conf):
+            shutil.rmtree(new_conf)
+        shutil.copytree(os.path.join(orig_builddir, "conf"), new_conf)
+
+    os.environ["BUILDDIR"] = builddir
+    os.environ["BBPATH"] = builddir
+
+    bb = Bitbake(builddir)
+    try:
+        bb.complete(pkg, machine)
+        return (machine, True, None)
+    except Error as e:
+        with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
+            f.write(e.stdout + e.stderr)
+        for line in e.stdout.split("\n") + e.stderr.split("\n"):
+            if re.match(".* went backwards which would break package feeds .*", line):
+                return (machine, True, None)
+            if re.match(".*not in COMPATIBLE.*", line):
+                return (machine, True, None)
+        return (machine, False, e.stdout + e.stderr)
+
+def _compile_serial(bb, pkg, machine, workdir):
+    """Compile a single machine serially (original behaviour)."""
+    try:
+        bb.complete(pkg, machine)
+    except Error as e:
+        with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
+            f.write(e.stdout + e.stderr)
+        for line in e.stdout.split("\n") + e.stderr.split("\n"):
+            if re.match(".* went backwards which would break package feeds .*", line):
+                break
+            if re.match(".*not in COMPATIBLE.*", line):
+                break
+        else:
+            raise CompilationError()
 
 def compile(devtool, bb, git, opts, group):
     if opts['skip_compilation']:
         W(" %s: Compilation was skipped by user choice!" % group['name'])
         return
 
-    for machine in opts['machines']:
-        I(" %s: compiling upgraded version for %s ..." % (group['name'], machine))
-        _compile(bb, " ".join([pkg_ctx['PN'] for pkg_ctx in group['pkgs']]), machine, group['workdir'])
-        if opts['buildhistory'] and machine == opts['machines'][0]:
+    pkg = " ".join([pkg_ctx['PN'] for pkg_ctx in group['pkgs']])
+    machines = opts['machines']
+
+    if opts.get('serial_build'):
+        for machine in machines:
+            I(" %s: compiling upgraded version for %s ..." % (group['name'], machine))
+            _compile_serial(bb, pkg, machine, group['workdir'])
+            if opts['buildhistory'] and machine == machines[0]:
+                I(" %s: Checking buildhistory ..." % group['name'])
+                group['buildhistory'].diff()
+    else:
+        orig_builddir = os.environ.get("BUILDDIR", "")
+        worker_args = [(pkg, m, group['workdir'], orig_builddir, i == 0) for i, m in enumerate(machines)]
+
+        I(" %s: compiling upgraded version for %s in parallel ..." % (group['name'], machines))
+        with multiprocessing.Pool(processes=len(machines)) as pool:
+            results = pool.map(_compile_worker, worker_args)
+
+        os.environ["BUILDDIR"] = orig_builddir
+        os.environ["BBPATH"] = orig_builddir
+
+        failed = [(m, err) for m, ok, err in results if not ok]
+        if failed:
+            for m, err in failed:
+                E(" %s: compilation failed for %s" % (group['name'], m))
+            raise CompilationError()
+
+        if opts['buildhistory'] and machines:
             I(" %s: Checking buildhistory ..." % group['name'])
             group['buildhistory'].diff()
 
diff --git a/upgrade-helper.py b/upgrade-helper.py
index 2ceb722..1c05333 100755
--- a/upgrade-helper.py
+++ b/upgrade-helper.py
@@ -108,6 +108,8 @@  def parse_cmdline():
                         help="layers to include in the upgrade research")
     parser.add_argument("--layer-dir", action="store", default='',
                         help="the layers root directory")
+    parser.add_argument("--serial-build", action="store_true", default=False,
+                        help="compile for each machine serially instead of in parallel")
     return parser.parse_args()
 
 def parse_config_file(config_file):
@@ -193,6 +195,7 @@  class Updater(object):
         self.opts['author'] = "Upgrade Helper <%s>" % \
                 settings.get('from', 'uh@not.set')
         self.opts['skip_compilation'] = self.args.skip_compilation
+        self.opts['serial_build'] = self.args.serial_build
         self.opts['buildhistory'] = self._buildhistory_is_enabled()
         self.opts['testimage'] = self._testimage_is_enabled()