@@ -171,6 +171,10 @@ class RunQueueScheduler(object):
self.stamps[tid] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
if tid in self.rq.runq_buildable:
self.buildable.add(tid)
+ if taskname not in self.skip_maxthread:
+ self.skip_maxthread[taskname] = self.rq.cfgData.getVarFlag(taskname, "number_threads")
+ # Only count the running tasks per name if some task has a limit
+ self.has_maxthread = any(self.skip_maxthread.values())
self.rev_prio_map = None
self.prev_cpu_pressure = 0.0
@@ -259,9 +263,11 @@ class RunQueueScheduler(object):
"""
# Once tasks are running we don't need to worry about them again
self.buildable.difference_update(self.rq.runq_running)
- buildable = set(self.buildable)
- buildable.difference_update(self.rq.holdoff_tasks)
- buildable.intersection_update(self.rq.tasks_covered | self.rq.tasks_notcovered)
+ holdoff = self.rq.holdoff_tasks
+ covered = self.rq.tasks_covered
+ notcovered = self.rq.tasks_notcovered
+ buildable = {t for t in self.buildable
+ if t not in holdoff and (t in covered or t in notcovered)}
if not buildable:
return None
@@ -273,16 +279,15 @@ class RunQueueScheduler(object):
# Filter out tasks that have a max number of threads that have been exceeded
skip_buildable = {}
- for running in self.rq.runq_running.difference(self.rq.runq_complete):
- rtaskname = taskname_from_tid(running)
- if rtaskname not in self.skip_maxthread:
- self.skip_maxthread[rtaskname] = self.rq.cfgData.getVarFlag(rtaskname, "number_threads")
- if not self.skip_maxthread[rtaskname]:
- continue
- if rtaskname in skip_buildable:
- skip_buildable[rtaskname] += 1
- else:
- skip_buildable[rtaskname] = 1
+ if self.has_maxthread:
+ for running in self.rq.runq_running.difference(self.rq.runq_complete):
+ rtaskname = taskname_from_tid(running)
+ if not self.skip_maxthread[rtaskname]:
+ continue
+ if rtaskname in skip_buildable:
+ skip_buildable[rtaskname] += 1
+ else:
+ skip_buildable[rtaskname] = 1
if len(buildable) == 1:
tid = buildable.pop()
@@ -72,6 +72,21 @@ class RunQueueTests(unittest.TestCase):
self.shutdown(tempdir)
+ def test_number_threads(self):
+ with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+ postconf = os.path.join(tempdir, "number_threads.conf")
+ with open(postconf, "w") as f:
+ f.write('do_fetch[number_threads] = "1"\n')
+ cmd = ["bitbake", "-R", postconf, "a1", "b1"]
+ # a1:do_fetch is scheduled first, so the quick b1:do_fetch can
+ # only finish after the slow a1:do_fetch if the limit is applied
+ tasks = self.run_bitbakecmd(cmd, tempdir, slowtasks="a1:fetch")
+ expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
+ self.assertEqual(set(tasks), set(expected))
+ self.assertLess(tasks.index('a1:fetch'), tasks.index('b1:fetch'))
+
+ self.shutdown(tempdir)
+
def test_single_setscenevalid(self):
with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
cmd = ["bitbake", "a1"]
next_buildable_task() runs at least once for every task in the build, and again on every worker wakeup while a slot is free. Each call builds tasks_covered | tasks_notcovered to filter the buildable tasks, and when a task is buildable it also computes runq_running - runq_complete. These four sets grow to the size of the whole runqueue, although only a few tasks are buildable at any time. Filter the buildable tasks by membership instead. The difference is only needed for number_threads limits, so fill the existing skip_maxthread cache once when the scheduler is created, and only count the running tasks when some task has a limit. A no-op 'bitbake core-image-minimal' with OE-core goes from 7.64s to 6.61s, and a no-op image build with 18 layers from 46.22s to 41.44s (median of 5 runs each). The runqueue task order of dry runs and of a single-threaded build from sstate is unchanged. Add a selftest checking that a number_threads limit is applied, which wasn't covered before. AI-Generated: Uses Claude Code (Claude Opus 5.5) Signed-off-by: Ricardo Salveti <ricardo.salveti@oss.qualcomm.com> --- Changes in v2: - Drop the runq_running_not_complete set, so nothing has to be kept in sync with runq_running and runq_complete. Fill the existing number_threads flag cache once when the scheduler is created instead, and only compute the difference when some task has a limit. - Add a selftest checking that a number_threads limit is applied. lib/bb/runqueue.py | 31 ++++++++++++++++++------------- lib/bb/tests/runqueue.py | 15 +++++++++++++++ 2 files changed, 33 insertions(+), 13 deletions(-)