From patchwork Sat Oct 24 12:33:32 2026 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mathieu Dubois-Briand X-Patchwork-Id: 101889 Received: from smtpout-02.galae.net (smtpout-02.galae.net [185.246.84.56]) by mx.groups.io with SMTP id smtpd.msgproc01-g2.5388.1792845244692549176 for ; Sat, 24 Oct 2026 05:34:05 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=dkim header.b=AJ1NFBvv; spf=pass (domain: bootlin.com, ip: 185.246.84.56, mailfrom: mathieu.dubois-briand@bootlin.com) Received: from smtpout-01.galae.net (smtpout-01.galae.net [212.83.139.233]) by smtpout-02.galae.net (Postfix) with ESMTPS id A74321A1315 for ; Sat, 24 Oct 2026 12:34:02 +0000 (UTC) Received: from mail.galae.net (mail.galae.net [212.83.136.155]) by smtpout-01.galae.net (Postfix) with ESMTPS id 7C5E660769 for ; Sat, 24 Oct 2026 12:34:02 +0000 (UTC) Received: from [127.0.0.1] (localhost [127.0.0.1]) by localhost (Mailerdaemon) with ESMTPSA id AE26B10328EDB; Sat, 24 Oct 2026 14:34:01 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=dkim; t=1792845242; h=from:subject:date:message-id:to:cc:mime-version:content-type: content-transfer-encoding:in-reply-to:references; bh=BPI1boXRsi+aaNwyctXWaDwj3Y5vNjyk2HPL4DgItFE=; b=AJ1NFBvv8EhIgHd6Rm8P03/kKEMErjg3Lu5STZeKIAVhIwYX12h9+JE6r3ID0N/28AdKHr 3u0RJRrnv90Bbod/dM7jS8qB0LMcp06RgYAkJArGUoTVfERAjtv1ewMZl3B9KC+qMYA5iF dYGZnDG8+xBMSvvLlXbpl+46UhCiNH+gPBEHwA+9XL8D1AAVN5JTDhiGyHrwshOgX8t8rM sj8LQeJtpGcUTAoXwjh0ATxCw/QsuhYA4kHEAhjTHSab8VyVPKJ5tlo0TfPyg22N8wYm2L /zQCfVgxI7lZRRdeV8D6Eqfo2QNHRPHx1qkaUNbay896zhAxPTUf3rwpy9T+FA== From: Mathieu Dubois-Briand Date: Sat, 24 Oct 2026 14:33:32 +0200 Subject: [PATCH RFC 1/3] utils: Extend LogCatcher to record raw messages and levels MIME-Version: 1.0 Message-Id: <20261024-mathieu-debug-replay-v4-v1-1-3f7f52a3b8e6@bootlin.com> References: <20261024-mathieu-debug-replay-v4-v1-0-3f7f52a3b8e6@bootlin.com> In-Reply-To: <20261024-mathieu-debug-replay-v4-v1-0-3f7f52a3b8e6@bootlin.com> To: bitbake-devel@lists.openembedded.org Cc: Thomas Petazzoni , Mathieu Dubois-Briand X-Mailer: b4 0.14.3 X-Developer-Signature: v=1; a=ed25519-sha256; t=1792845239; l=3537; i=mathieu.dubois-briand@bootlin.com; s=20241219; h=from:subject:message-id; bh=zn4aA6JzVFz39faIO4Fd9j6OZcfvz5Am6gIM0CJdChI=; b=DASbHu5cJhgIAOCnPkDip0m5VfnITObfOuCBfBq+hHjLwFRGh4t6nX+bFAwU3B9Mk2pvCOCQP xMTWwbsnxuLCpvkheiPRldPbGHit5aKAfvMpZL2kLa24MuDBONQcGOw X-Developer-Key: i=mathieu.dubois-briand@bootlin.com; a=ed25519; pk=1PVTmzPXfKvDwcPUzG0aqdGoKZJA3b9s+3DqRlm0Lww= X-Last-TLS-Session-Version: TLSv1.3 List-Id: X-Webhook-Received: from 45-33-107-173.ip.linodeusercontent.com [45.33.107.173] by lin-tor-korg-lkml-alma10-1 with HTTPS for ; Sat, 24 Oct 2026 12:34:13 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/20376 LogCatcher stores only formatted messages, which strips the severity. Also record each log record's raw message and level, so callers can later replay them with their original severity. Lower the handler minimum level to WARNONCE so warnonce and erroronce messages are captured too. Add unit tests covering the captured levels and raw messages, the formatted output and the level filtering. AI-Generated: Uses opencode (DeepSeek V4 Flash) Signed-off-by: Mathieu Dubois-Briand --- lib/bb/tests/utils.py | 42 ++++++++++++++++++++++++++++++++++++++++++ lib/bb/utils.py | 4 +++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/lib/bb/tests/utils.py b/lib/bb/tests/utils.py index 52b7bf85bf47..a34f6d698225 100644 --- a/lib/bb/tests/utils.py +++ b/lib/bb/tests/utils.py @@ -8,9 +8,11 @@ import unittest import bb +import bb.build import os import tempfile import re +import logging class VerCmpString(unittest.TestCase): @@ -693,6 +695,46 @@ class EnvironmentTests(unittest.TestCase): self.assertEqual(os.environ["A"], "this is A") self.assertNotIn("B", os.environ) +class LogCatcherTests(unittest.TestCase): + def setUp(self): + self.logger = logging.getLogger("BitBake") + self.oldlevel = self.logger.level + self.logger.setLevel(logging.DEBUG) + self.catcher = bb.utils.LogCatcher() + self.logger.addHandler(self.catcher) + + def tearDown(self): + self.logger.removeHandler(self.catcher) + self.logger.setLevel(self.oldlevel) + + def test_records_capture_level_and_raw_message(self): + bb.warn("a warning") + bb.warnonce("a warn once") + bb.error("an error") + bb.erroronce("an error once") + + self.assertEqual(self.catcher.records, [ + (logging.WARNING, "a warning"), + (bb.msg.BBLogFormatter.WARNONCE, "a warn once"), + (logging.ERROR, "an error"), + (bb.msg.BBLogFormatter.ERRORONCE, "an error once"), + ]) + + def test_messages_are_formatted(self): + bb.warn("a warning") + self.assertEqual(self.catcher.messages, ["WARNING: a warning"]) + self.assertTrue(self.catcher.contains("WARNING: a warning")) + + def test_below_minimum_level_is_ignored(self): + bb.debug(1, "a debug") + bb.plain("a plain") + bb.note("a note") + bb.verbnote("a verb note") + + self.assertEqual(self.catcher.records, []) + self.assertEqual(self.catcher.messages, []) + + class FilemodeTests(unittest.TestCase): def test_filemode_convert(self): self.assertEqual(0o775, bb.utils.to_filemode("0o775")) diff --git a/lib/bb/utils.py b/lib/bb/utils.py index ec6eb2181aa8..4a4e911b0f8a 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -2154,9 +2154,11 @@ class LogCatcher(logging.Handler): """Logging handler for collecting logged messages so you can check them later""" def __init__(self): self.messages = [] - logging.Handler.__init__(self, logging.WARNING) + self.records = [] + logging.Handler.__init__(self, bb.msg.BBLogFormatter.WARNONCE) def emit(self, record): self.messages.append(bb.build.logformatter.format(record)) + self.records.append((record.levelno, record.getMessage())) def contains(self, message): return (message in self.messages) From patchwork Sat Oct 24 12:33:33 2026 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mathieu Dubois-Briand X-Patchwork-Id: 101891 Received: from smtpout-02.galae.net (smtpout-02.galae.net [185.246.84.56]) by mx.groups.io with SMTP id smtpd.msgproc02-g2.5423.1792845244953617374 for ; Sat, 24 Oct 2026 05:34:05 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=dkim header.b=E8xT6Zke; spf=pass (domain: bootlin.com, ip: 185.246.84.56, mailfrom: mathieu.dubois-briand@bootlin.com) Received: from smtpout-01.galae.net (smtpout-01.galae.net [212.83.139.233]) by smtpout-02.galae.net (Postfix) with ESMTPS id 5579B1A1316 for ; Sat, 24 Oct 2026 12:34:03 +0000 (UTC) Received: from mail.galae.net (mail.galae.net [212.83.136.155]) by smtpout-01.galae.net (Postfix) with ESMTPS id 236F960769 for ; Sat, 24 Oct 2026 12:34:03 +0000 (UTC) Received: from [127.0.0.1] (localhost [127.0.0.1]) by localhost (Mailerdaemon) with ESMTPSA id 6346910328FE8; Sat, 24 Oct 2026 14:34:02 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=dkim; t=1792845242; h=from:subject:date:message-id:to:cc:mime-version:content-type: content-transfer-encoding:in-reply-to:references; bh=ZKu7CIfHGAHP0hs1KMEesIl1hWsQUuTfFK04XanqYqQ=; b=E8xT6Zke394oZC/gG6iKFBAqSm/QVXrNSM838JXwQwVmlOzr2MfVva5UL5oMdvjOURkyWZ FAPqSOSBBZiwNM0tQCIMWSmSlZgjmAIZBKWrs52YXExzDGe6FKdu5AJDasJ38iqv3IFUPJ f38tvptFf0kb54bSuF1D77IknrukSaBFoVW9mRluJxJMmpXtM2+bH+Xb2r27MJ+yzwVwU9 iAKZUv5IS0+JxYEiO1l5j07ZJ3H5KE9EUoRvut9LlP+zEkKO01NO9SSgW5Wvk7FQ6uc1ZB dooU4dtlTaFOTtMAtB9Dqht66wfP3psYsl7fwjAPcj3Y+qHNCZLYHMBWoPVC9w== From: Mathieu Dubois-Briand Date: Sat, 24 Oct 2026 14:33:33 +0200 Subject: [PATCH RFC 2/3] build: Attach a log catcher to tasks and save their records MIME-Version: 1.0 Message-Id: <20261024-mathieu-debug-replay-v4-v1-2-3f7f52a3b8e6@bootlin.com> References: <20261024-mathieu-debug-replay-v4-v1-0-3f7f52a3b8e6@bootlin.com> In-Reply-To: <20261024-mathieu-debug-replay-v4-v1-0-3f7f52a3b8e6@bootlin.com> To: bitbake-devel@lists.openembedded.org Cc: Thomas Petazzoni , Mathieu Dubois-Briand X-Mailer: b4 0.14.3 X-Developer-Signature: v=1; a=ed25519-sha256; t=1792845239; l=2632; i=mathieu.dubois-briand@bootlin.com; s=20241219; h=from:subject:message-id; bh=bwu9ErDIi26KsDaJGWrD240rDAdMinoHL0OlD+mO1jk=; b=D3WM5hto+s+2xsIPdae4PSaEYS3BBoZUW9hpuZ12xK1pNwMi+Jj+Lgm72+72Q/0/7i1NQVSfH DnLVXXfLP1FAK9pXtF0dmKHmkwOec1LH1EmE69HEFY6EYyUgnXSQzRG X-Developer-Key: i=mathieu.dubois-briand@bootlin.com; a=ed25519; pk=1PVTmzPXfKvDwcPUzG0aqdGoKZJA3b9s+3DqRlm0Lww= X-Last-TLS-Session-Version: TLSv1.3 List-Id: X-Webhook-Received: from 45-33-107-173.ip.linodeusercontent.com [45.33.107.173] by lin-tor-korg-lkml-alma10-1 with HTTPS for ; Sat, 24 Oct 2026 12:34:14 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/20375 Every task gets a LogCatcher which records its warnings and errors during execution. Right after the task body, and before the postfuncs, the records are written to ${T}/logrecords..json. The sstate packaging postfunc reads the current task's file to bundle the messages for replay when the task is later restored from the sstate cache. Writing the file before the postfuncs makes both the current task's own record and the files of already finished dependency tasks readable from the same place. AI-Generated: Uses opencode (DeepSeek V4 Flash) Signed-off-by: Mathieu Dubois-Briand --- lib/bb/build.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/bb/build.py b/lib/bb/build.py index 8b49721d60f8..033e5b1c3f30 100644 --- a/lib/bb/build.py +++ b/lib/bb/build.py @@ -21,6 +21,7 @@ import time import re import stat import datetime +import json import bb import bb.msg import bb.process @@ -656,6 +657,11 @@ def _exec_task(fn, task, d, quieterr): except OSError: pass + # Task log records, for sstate log replay. Written right after the task + # body, before the postfuncs, so the sstate packaging postfunc can read them. + logrecords = os.path.join(tempdir, 'logrecords.{0}.json'.format(task)) + bb.utils.remove(logrecords) + prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) @@ -698,6 +704,9 @@ def _exec_task(fn, task, d, quieterr): errchk = ErrorCheckHandler() bblogger.addHandler(errchk) + tasklogcatcher = bb.utils.LogCatcher() + bblogger.addHandler(tasklogcatcher) + localdata.setVar('BB_LOGFILE', logfn) localdata.setVar('BB_RUNTASK', task) localdata.setVar('BB_TASK_LOGGER', bblogger) @@ -711,6 +720,9 @@ def _exec_task(fn, task, d, quieterr): for func in (prefuncs or '').split(): exec_func(func, localdata) exec_func(task, localdata) + if tasklogcatcher.records: + with open(logrecords, 'w') as f: + json.dump(tasklogcatcher.records, f) for func in (postfuncs or '').split(): exec_func(func, localdata) finally: @@ -720,6 +732,7 @@ def _exec_task(fn, task, d, quieterr): sys.stderr.flush() bblogger.removeHandler(handler) + bblogger.removeHandler(tasklogcatcher) # Restore the backup fds os.dup2(osi[0], osi[1]) From patchwork Sat Oct 24 12:33:34 2026 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mathieu Dubois-Briand X-Patchwork-Id: 101890 Received: from smtpout-02.galae.net (smtpout-02.galae.net [185.246.84.56]) by mx.groups.io with SMTP id smtpd.msgproc02-g2.5424.1792845245570256248 for ; Sat, 24 Oct 2026 05:34:05 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=dkim header.b=weI9x5dH; spf=pass (domain: bootlin.com, ip: 185.246.84.56, mailfrom: mathieu.dubois-briand@bootlin.com) Received: from smtpout-01.galae.net (smtpout-01.galae.net [212.83.139.233]) by smtpout-02.galae.net (Postfix) with ESMTPS id E24FF1A1317 for ; Sat, 24 Oct 2026 12:34:03 +0000 (UTC) Received: from mail.galae.net (mail.galae.net [212.83.136.155]) by smtpout-01.galae.net (Postfix) with ESMTPS id B80C560769 for ; Sat, 24 Oct 2026 12:34:03 +0000 (UTC) Received: from [127.0.0.1] (localhost [127.0.0.1]) by localhost (Mailerdaemon) with ESMTPSA id 0347A1032902C; Sat, 24 Oct 2026 14:34:02 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=dkim; t=1792845243; h=from:subject:date:message-id:to:cc:mime-version:content-type: content-transfer-encoding:in-reply-to:references; bh=+KLpa4IXHQLvBptLW9hu9SJ9MUs9VGgXXsywQeuQD2Y=; b=weI9x5dHrQulfjp62dtQT3FnwQEjqdD9wqXd523uPVVyL8Jrx1AlTQLIwnSInN9p0csNop vmcZxJEplMs2ajoMcZ9Lmgbu03xOjH2fNlxFb7k9bS9PQkoEIE8ix5RwaoyNDUSiRX1RI3 wGVxwauGBswNgYWpQnZGpBf1ZruNReKYWu03KOwPtprnCVoAsxFuUQPh1+KZcl1XXhGNfk ZY6VCncVRx39u1ubVMesuE8A6Eo9WEurMAlMt+XRaM2PuyAUK3OX79+s7r21+JfFQMVQiH yiFL9njipps2UYwtqdBB/K2gkUU4AG+rrA9IOpuG7Pq6GfU+WcamlVzbxD2Pcw== From: Mathieu Dubois-Briand Date: Sat, 24 Oct 2026 14:33:34 +0200 Subject: [PATCH RFC 3/3] runqueue: Hand each covered task's log to a single setscene task MIME-Version: 1.0 Message-Id: <20261024-mathieu-debug-replay-v4-v1-3-3f7f52a3b8e6@bootlin.com> References: <20261024-mathieu-debug-replay-v4-v1-0-3f7f52a3b8e6@bootlin.com> In-Reply-To: <20261024-mathieu-debug-replay-v4-v1-0-3f7f52a3b8e6@bootlin.com> To: bitbake-devel@lists.openembedded.org Cc: Thomas Petazzoni , Mathieu Dubois-Briand X-Mailer: b4 0.14.3 X-Developer-Signature: v=1; a=ed25519-sha256; t=1792845239; l=3079; i=mathieu.dubois-briand@bootlin.com; s=20241219; h=from:subject:message-id; bh=/jp3DSDXTwmpUy5ISVhbFiy/CGC2vA5VmpD6YQqIWhI=; b=Av1SO2FqyxJy21AlCCBmY/I1rNC5j5ATKX6V8zkwJUR42DGDMMrUu4thuPd85i7Q5YsKn4Lw2 H/hQmnP6GNxAy6is1VZkIgZmfk7m9rK5YsXU6GVE169TFIzcpGu0LEI X-Developer-Key: i=mathieu.dubois-briand@bootlin.com; a=ed25519; pk=1PVTmzPXfKvDwcPUzG0aqdGoKZJA3b9s+3DqRlm0Lww= X-Last-TLS-Session-Version: TLSv1.3 List-Id: X-Webhook-Received: from 45-33-107-173.ip.linodeusercontent.com [45.33.107.173] by lin-tor-korg-lkml-alma10-1 with HTTPS for ; Sat, 24 Oct 2026 12:34:14 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/20374 When a setscene task starts, hand it the covered tasks of the same recipe which were not yet handed to another setscene task, and pass their names to the task through BB_SETSCENE_REPLAY_TASKS. Several setscene tasks can cover the same task (e.g. do_populate_sysroot and do_package both cover do_compile), so this lets metadata replay the log of each covered task exactly once when restoring it from the sstate cache. AI-Generated: Uses opencode (DeepSeek V4 Flash) Signed-off-by: Mathieu Dubois-Briand --- bin/bitbake-worker | 1 + lib/bb/runqueue.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/bin/bitbake-worker b/bin/bitbake-worker index 61484a5c2522..0a46eea3ccb5 100755 --- a/bin/bitbake-worker +++ b/bin/bitbake-worker @@ -266,6 +266,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask): the_data = databuilder.mcdata[mc] the_data.setVar("BB_WORKERCONTEXT", "1") the_data.setVar("BB_TASKDEPDATA", taskdepdata) + the_data.setVar("BB_SETSCENE_REPLAY_TASKS", " ".join(runtask.get('replaytasks', []))) the_data.setVar('BB_CURRENTTASK', taskname.replace("do_", "")) if cfg.limited_deps: the_data.setVar("BB_LIMITEDDEPS", "1") diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index ef95eb46bf95..8bb1cb1a6e72 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -1939,6 +1939,8 @@ class RunQueueExecute: self.scenequeue_notcovered = set() self.tasks_notcovered = set() self.scenequeue_notneeded = set() + # Covered tasks already handed to a setscene task for log replay + self.sq_replay_assigned = set() self.setscene_tids_generator = None @@ -2297,6 +2299,12 @@ class RunQueueExecute: startevent = sceneQueueTaskStarted(task, self.stats, self.rq) bb.event.fire(startevent, self.cfgData) + # Each covered task of this recipe is assigned to exactly one + # setscene task, so its log is replayed only once. + replaytasks = {t for t in self.sqdata.sq_covered_tasks[task] + if fn_from_tid(t) == taskfn} - self.sq_replay_assigned + self.sq_replay_assigned |= replaytasks + taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] realfn = bb.cache.virtualfn2realfn(taskfn)[0] runtask = { @@ -2305,6 +2313,7 @@ class RunQueueExecute: 'taskname' : taskname, 'taskhash' : self.rqdata.get_task_hash(task), 'unihash' : self.rqdata.get_task_unihash(task), + 'replaytasks' : sorted(taskname_from_tid(t) for t in replaytasks), 'quieterrors' : True, 'appends' : self.cooker.collections[mc].get_file_appends(taskfn), 'layername' : self.cooker.collections[mc].calc_bbfile_priority(realfn)[2],