@@ -22,10 +22,12 @@
import os
import shutil
import re
+import multiprocessing
from logging import debug as D
from logging import info as I
from logging import warning as W
+from logging import error as E
from errors import Error, DevtoolError, CompilationError
from buildhistory import BuildHistory
@@ -104,31 +106,87 @@ def devtool_upgrade(devtool, bb, git, opts, group):
for p in group['pkgs']:
_devtool_upgrade(devtool, bb, git, opts, p)
-def _compile(bb, pkg, machine, workdir):
- try:
- bb.complete(pkg, machine)
- except Error as e:
- with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
- f.write(e.stdout + e.stderr)
- for line in e.stdout.split("\n") + e.stderr.split("\n"):
- # version going backwards is not a real error
- if re.match(".* went backwards which would break package feeds .*", line):
- break
- # 'not in COMPATIBLE_HOST/MACHINE is not a real error
- if re.match(".*not in COMPATIBLE.*", line):
- break
- else:
- raise CompilationError()
+def _compile_worker(args):
+ """Worker function for multiprocess compilation. Runs in a separate process."""
+ pkg, machine, workdir, orig_builddir, use_orig = args
+ from utils.bitbake import Bitbake
+
+ if use_orig:
+ builddir = orig_builddir
+ else:
+ builddir = "{}-{}".format(orig_builddir, machine)
+ os.makedirs(builddir, exist_ok=True)
+
+ new_conf = os.path.join(builddir, "conf")
+ if os.path.isdir(new_conf):
+ shutil.rmtree(new_conf)
+ shutil.copytree(os.path.join(orig_builddir, "conf"), new_conf)
+
+ os.environ["BUILDDIR"] = builddir
+ os.environ["BBPATH"] = builddir
+
+ bb = Bitbake(builddir)
+ try:
+ bb.complete(pkg, machine)
+ return (machine, True, None)
+ except Error as e:
+ with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
+ f.write(e.stdout + e.stderr)
+ for line in e.stdout.split("\n") + e.stderr.split("\n"):
+ if re.match(".* went backwards which would break package feeds .*", line):
+ return (machine, True, None)
+ if re.match(".*not in COMPATIBLE.*", line):
+ return (machine, True, None)
+ return (machine, False, e.stdout + e.stderr)
+
+def _compile_serial(bb, pkg, machine, workdir):
+ """Compile a single machine serially (original behaviour)."""
+ try:
+ bb.complete(pkg, machine)
+ except Error as e:
+ with open("{}/bitbake-output-{}.txt".format(workdir, machine), 'w') as f:
+ f.write(e.stdout + e.stderr)
+ for line in e.stdout.split("\n") + e.stderr.split("\n"):
+ if re.match(".* went backwards which would break package feeds .*", line):
+ break
+ if re.match(".*not in COMPATIBLE.*", line):
+ break
+ else:
+ raise CompilationError()
def compile(devtool, bb, git, opts, group):
if opts['skip_compilation']:
W(" %s: Compilation was skipped by user choice!" % group['name'])
return
- for machine in opts['machines']:
- I(" %s: compiling upgraded version for %s ..." % (group['name'], machine))
- _compile(bb, " ".join([pkg_ctx['PN'] for pkg_ctx in group['pkgs']]), machine, group['workdir'])
- if opts['buildhistory'] and machine == opts['machines'][0]:
+ pkg = " ".join([pkg_ctx['PN'] for pkg_ctx in group['pkgs']])
+ machines = opts['machines']
+
+ if opts.get('serial_build'):
+ for machine in machines:
+ I(" %s: compiling upgraded version for %s ..." % (group['name'], machine))
+ _compile_serial(bb, pkg, machine, group['workdir'])
+ if opts['buildhistory'] and machine == machines[0]:
+ I(" %s: Checking buildhistory ..." % group['name'])
+ group['buildhistory'].diff()
+ else:
+ orig_builddir = os.environ.get("BUILDDIR", "")
+ worker_args = [(pkg, m, group['workdir'], orig_builddir, i == 0) for i, m in enumerate(machines)]
+
+ I(" %s: compiling upgraded version for %s in parallel ..." % (group['name'], machines))
+ with multiprocessing.Pool(processes=len(machines)) as pool:
+ results = pool.map(_compile_worker, worker_args)
+
+ os.environ["BUILDDIR"] = orig_builddir
+ os.environ["BBPATH"] = orig_builddir
+
+ failed = [(m, err) for m, ok, err in results if not ok]
+ if failed:
+ for m, err in failed:
+ E(" %s: compilation failed for %s" % (group['name'], m))
+ raise CompilationError()
+
+ if opts['buildhistory'] and machines:
I(" %s: Checking buildhistory ..." % group['name'])
group['buildhistory'].diff()
@@ -108,6 +108,8 @@ def parse_cmdline():
help="layers to include in the upgrade research")
parser.add_argument("--layer-dir", action="store", default='',
help="the layers root directory")
+ parser.add_argument("--serial-build", action="store_true", default=False,
+ help="compile for each machine serially instead of in parallel")
return parser.parse_args()
def parse_config_file(config_file):
@@ -193,6 +195,7 @@ class Updater(object):
self.opts['author'] = "Upgrade Helper <%s>" % \
settings.get('from', 'uh@not.set')
self.opts['skip_compilation'] = self.args.skip_compilation
+ self.opts['serial_build'] = self.args.serial_build
self.opts['buildhistory'] = self._buildhistory_is_enabled()
self.opts['testimage'] = self._testimage_is_enabled()