@@ -145,7 +145,16 @@ def sigterm_handler(signum, frame):
os.killpg(0, signal.SIGTERM)
sys.exit()
-def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
+def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
+
+ fn = runtask['fn']
+ task = runtask['task']
+ taskname = runtask['taskname']
+ taskhash = runtask['taskhash']
+ unihash = runtask['unihash']
+ appends = runtask['appends']
+ taskdepdata = runtask['taskdepdata']
+ quieterrors = runtask['quieterrors']
# We need to setup the environment BEFORE the fork, since
# a fork() or exec*() activates PSEUDO...
@@ -157,8 +166,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
uid = os.getuid()
gid = os.getgid()
-
- taskdep = workerdata["taskdeps"][fn]
+ taskdep = runtask['taskdep']
if 'umask' in taskdep and taskname in taskdep['umask']:
umask = taskdep['umask'][taskname]
elif workerdata["umask"]:
@@ -170,24 +178,24 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
except TypeError:
pass
- dry_run = cfg.dry_run or dry_run_exec
+ dry_run = cfg.dry_run or runtask['dry_run']
# We can't use the fakeroot environment in a dry run as it possibly hasn't been built
if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
fakeroot = True
- envvars = (workerdata["fakerootenv"][fn] or "").split()
+ envvars = (runtask['fakerootenv'] or "").split()
for key, value in (var.split('=') for var in envvars):
envbackup[key] = os.environ.get(key)
os.environ[key] = value
fakeenv[key] = value
- fakedirs = (workerdata["fakerootdirs"][fn] or "").split()
+ fakedirs = (runtask['fakerootdirs'] or "").split()
for p in fakedirs:
bb.utils.mkdirhier(p)
logger.debug2('Running %s:%s under fakeroot, fakedirs: %s' %
(fn, taskname, ', '.join(fakedirs)))
else:
- envvars = (workerdata["fakerootnoenv"][fn] or "").split()
+ envvars = (runtask['fakerootnoenv'] or "").split()
for key, value in (var.split('=') for var in envvars):
envbackup[key] = os.environ.get(key)
os.environ[key] = value
@@ -474,11 +482,15 @@ class BitbakeWorker(object):
sys.exit(0)
def handle_runtask(self, data):
- fn, task, taskname, taskhash, unihash, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
- workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
+ runtask = pickle.loads(data)
- pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
+ fn = runtask['fn']
+ task = runtask['task']
+ taskname = runtask['taskname']
+
+ workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
+ pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, self.extraconfigdata, runtask)
self.build_pids[pid] = task
self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
@@ -1311,10 +1311,6 @@ class RunQueue:
workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec, fakerootlogs=fakerootlogs)
workerdata = {
- "taskdeps" : self.rqdata.dataCaches[mc].task_deps,
- "fakerootenv" : self.rqdata.dataCaches[mc].fakerootenv,
- "fakerootdirs" : self.rqdata.dataCaches[mc].fakerootdirs,
- "fakerootnoenv" : self.rqdata.dataCaches[mc].fakerootnoenv,
"sigdata" : bb.parse.siggen.get_taskdata(),
"logdefaultlevel" : bb.msg.loggerDefaultLogLevel,
"build_verbose_shell" : self.cooker.configuration.build_verbose_shell,
@@ -2139,18 +2135,30 @@ class RunQueueExecute:
startevent = sceneQueueTaskStarted(task, self.stats, self.rq)
bb.event.fire(startevent, self.cfgData)
- taskdepdata = self.sq_build_taskdepdata(task)
-
taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
- taskhash = self.rqdata.get_task_hash(task)
- unihash = self.rqdata.get_task_unihash(task)
+ runtask = {
+ 'fn' : taskfn,
+ 'task' : task,
+ 'taskname' : taskname,
+ 'taskhash' : self.rqdata.get_task_hash(task),
+ 'unihash' : self.rqdata.get_task_unihash(task),
+ 'quieterrors' : True,
+ 'appends' : self.cooker.collections[mc].get_file_appends(taskfn),
+ 'taskdepdata' : self.sq_build_taskdepdata(task),
+ 'dry_run' : False,
+ 'taskdep': taskdep,
+ 'fakerootenv' : self.rqdata.dataCaches[mc].fakerootenv[taskfn],
+ 'fakerootdirs' : self.rqdata.dataCaches[mc].fakerootdirs[taskfn],
+ 'fakerootnoenv' : self.rqdata.dataCaches[mc].fakerootnoenv[taskfn]
+ }
+
if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
if not mc in self.rq.fakeworker:
self.rq.start_fakeworker(self, mc)
- self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+ self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps(runtask) + b"</runtask>")
self.rq.fakeworker[mc].process.stdin.flush()
else:
- self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+ self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps(runtask) + b"</runtask>")
self.rq.worker[mc].process.stdin.flush()
self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -2220,11 +2228,23 @@ class RunQueueExecute:
startevent = runQueueTaskStarted(task, self.stats, self.rq)
bb.event.fire(startevent, self.cfgData)
- taskdepdata = self.build_taskdepdata(task)
-
taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
- taskhash = self.rqdata.get_task_hash(task)
- unihash = self.rqdata.get_task_unihash(task)
+ runtask = {
+ 'fn' : taskfn,
+ 'task' : task,
+ 'taskname' : taskname,
+ 'taskhash' : self.rqdata.get_task_hash(task),
+ 'unihash' : self.rqdata.get_task_unihash(task),
+ 'quieterrors' : False,
+ 'appends' : self.cooker.collections[mc].get_file_appends(taskfn),
+ 'taskdepdata' : self.build_taskdepdata(task),
+ 'dry_run' : self.rqdata.setscene_enforce,
+ 'taskdep': taskdep,
+ 'fakerootenv' : self.rqdata.dataCaches[mc].fakerootenv[taskfn],
+ 'fakerootdirs' : self.rqdata.dataCaches[mc].fakerootdirs[taskfn],
+ 'fakerootnoenv' : self.rqdata.dataCaches[mc].fakerootnoenv[taskfn]
+ }
+
if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
if not mc in self.rq.fakeworker:
try:
@@ -2234,10 +2254,10 @@ class RunQueueExecute:
self.rq.state = runQueueFailed
self.stats.taskFailed()
return True
- self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+ self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps(runtask) + b"</runtask>")
self.rq.fakeworker[mc].process.stdin.flush()
else:
- self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+ self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps(runtask) + b"</runtask>")
self.rq.worker[mc].process.stdin.flush()
self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
When setting up the worker we were transfering large amounts of data which aren't needed until task execution time. Defer the fakeroot and taskdeps data until they're needed for a specific task. This will duplicate some information when executing different tasks for a given recipe but as is is spread over the build run, it shouldn't be an issue overall. Also take the opportunity to clean up the silly length argument lists that were being passed around at the expense of extra dictionary keys. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> --- bin/bitbake-worker | 32 +++++++++++++++++++--------- lib/bb/runqueue.py | 52 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 58 insertions(+), 26 deletions(-)