From patchwork Wed Jul 16 15:22:46 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 66986 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id DE853C83F1B for ; Wed, 16 Jul 2025 15:22:56 +0000 (UTC) Received: from mail-wr1-f50.google.com (mail-wr1-f50.google.com [209.85.221.50]) by mx.groups.io with SMTP id smtpd.web11.25141.1752679373053446695 for ; Wed, 16 Jul 2025 08:22:53 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=C3yKbyQe; spf=pass (domain: linuxfoundation.org, ip: 209.85.221.50, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wr1-f50.google.com with SMTP id ffacd0b85a97d-3a507e88b0aso5036882f8f.1 for ; Wed, 16 Jul 2025 08:22:52 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1752679371; x=1753284171; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:message-id:date:subject:to :from:from:to:cc:subject:date:message-id:reply-to; bh=uD5hyyLm5eid0GU+7/tUcpRXpde53WAclPTtYt/Bdow=; b=C3yKbyQeAcPNC5qGWoqsvvXinVToD0J07bIMWcj4P7my0VOz0D5MQZJSVt2csLxdE3 7hIGUL7wxIAtsxQpe8b/wyOwV2ib4tj6yc2BHLYEZWX0d5njfJj2myrbhlrWDsrl+sZb rbfdlYDFBVuORAJKVpPXvDq2Q0niZn2GXqFms= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1752679371; x=1753284171; h=content-transfer-encoding:mime-version:message-id:date:subject:to :from:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=uD5hyyLm5eid0GU+7/tUcpRXpde53WAclPTtYt/Bdow=; b=lO6w4kIyyCsbI9ggz47DBFsy780O15kNVDZJfPyqvGiHho7VR3F2rLg8YC794l8WJ/ r5GAC+5W4LeH9t8v3ScQ0kXL/t0ZHjBbogi9KxggFJV+Z+33y/RrkAGp67HS98DDBTw2 JqzumKIQcfilVFuZar/csBgxKD8bnjLyRNOUiUg3wdfKQTVeIvKMHO8/oM2BdRi46Z7S wJNpFFZN6VSdHdSKwBiiSzM5r8oDLuWRfO5XpDU8kEWxYNDFrhDis4tUOgw2zehNpue0 yev6ihH2kvXLAVVxEQQa+pbd/ZO/Uvp3FbMP1LV5JWjgWDdnXInbPSdy7qJp/5SMJLxM PrHw== X-Gm-Message-State: AOJu0Yyx4Ysw3LVfwJL61xcO4swp6/CLA6Cqtqkdo1G0XIc6FKM2PQOE Fu1KBEqyhHKgeMpn3FD+LcWpmPuHb1usXRSXEqyNrMQU/NSufv5ZnbuuVTU7dRAR98bc/ZpgCsu xID5P X-Gm-Gg: ASbGnctVVxpetgE+fnL8hz9B1dv8+zuS1WIZnccnwUxos7KqF/NQ8lFjIyP+iwwebNb 8UzMNmXEgkKTaWgrdmrKZr/nawZH1T+NCPFelgcdk6SirIuIRwdG4p9a5TeK9WFS4YpAGM9zj7P i3TtL0NqvDkoMKtvTetjvQ5824G9XwTghQqYyzEyU82MqACxCA68DLpATJ25GRMUBqiF8JJD+qC oevfChIYcqKwgCzl74dho+o6r3oWXXv5jesr1DIdrgBsOkzTxCVAAP1++HbjGk7GYJTaJSADbLQ fzxyzkcv9jgYeEibxYT573tLw4DsLOTRbV9fnSaIQtpt5bW5g7ZE0FJuhD1p2ltfOMd+CliWPBp gsZH953ZEL15npqGVCgtl2xaaC+puCIhkR+NDpQFbSps6lzBO2sU= X-Google-Smtp-Source: AGHT+IHc00KUVB/kxPe8sfKR0OtVeS/vkOIejPZg0ij1m1BkyIrS7/s1CE90RdgELsCpKiIweM6jwA== X-Received: by 2002:a05:6000:144b:b0:3a5:2599:4178 with SMTP id ffacd0b85a97d-3b60e4c8f5fmr2355418f8f.19.1752679370977; Wed, 16 Jul 2025 08:22:50 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:22ad:92e1:8848:8bb0]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-3b5e8e0d872sm18362725f8f.60.2025.07.16.08.22.49 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 16 Jul 2025 08:22:50 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 1/4] asyncrpc: Avoid file not found traceback in logs Date: Wed, 16 Jul 2025 16:22:46 +0100 Message-ID: <20250716152249.96126-1-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.48.1 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 16 Jul 2025 15:22:56 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/17779 If the server is quickly stopped, we see tracebacks in the locks due to the file not existing. Hide these as they're not errors. Signed-off-by: Richard Purdie Reviewed-by: Joshua Watt --- lib/bb/asyncrpc/serv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/bb/asyncrpc/serv.py b/lib/bb/asyncrpc/serv.py index 667217c5c1b..35e93f7c969 100644 --- a/lib/bb/asyncrpc/serv.py +++ b/lib/bb/asyncrpc/serv.py @@ -211,7 +211,10 @@ class UnixStreamServer(StreamServer): self.server.close() def cleanup(self): - os.unlink(self.path) + try: + os.unlink(self.path) + except FileNotFoundError: + pass class WebsocketsServer(object): From patchwork Wed Jul 16 15:22:47 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 66988 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id EADB4C83F22 for ; Wed, 16 Jul 2025 15:22:56 +0000 (UTC) Received: from mail-wr1-f46.google.com (mail-wr1-f46.google.com [209.85.221.46]) by mx.groups.io with SMTP id smtpd.web11.25143.1752679374456471674 for ; Wed, 16 Jul 2025 08:22:54 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=ITusLlP9; spf=pass (domain: linuxfoundation.org, ip: 209.85.221.46, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wr1-f46.google.com with SMTP id ffacd0b85a97d-3b611665b96so238188f8f.2 for ; Wed, 16 Jul 2025 08:22:54 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1752679372; x=1753284172; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=EE5eqzPs1fr7gdra8X3hZDVtcYv4iKxxx5+gTGpMb7M=; b=ITusLlP96GuxzFwr98oBOcrhpWW6Zz8nCDr7lkUf86mbmcmZTcDO/x8eTIc+r+o14o dbs1TfQc5v9t7rLn/Qf9JwSUEg2A3X9QbDFcnDCrpLvmPR/eZPXCfDn/vt0bfeRBM276 +KHgElk3IBs8wGJSM05HHtbPsCk1tQhm5rBkA= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1752679372; x=1753284172; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=EE5eqzPs1fr7gdra8X3hZDVtcYv4iKxxx5+gTGpMb7M=; b=L7USYEoeVe5GGPek/70b+FicZaQGkH3+CmdJho9CkZM3yO8HX8TlKwnDPAEJCKTuVn czw3x9FHduIjXcqo0VNzTqfja/egZzm58ZMpMY6aeRqdhK6dwX8uNi2FZrB1a4jNcFJz pNyf9s8TZus/qPvBELTnJg6OVqUnNF5a6THW3/AzpEIFUhLM2dw6P/kKCEdrGCsim22B D43MU1bWPMX/rtMRa5GW3Dw6o0K5AB7XzAfGj6v3+M4YNbUOUc4igKV2w0lNEpsD2yTz EnoNURMEo1WCPu0aVhLecBv1YVcvVLiI5P309RTxV7tvnLF2lfBfCPjudaWwKznby5cl nqJA== X-Gm-Message-State: AOJu0Yyssz3c2yXBtfYYQkrH/xtkw1daqW/HtKAOrQAbjGwFFmSc53MC ywBYkyKREkUbCvr3g1XhUK8vBZew42N/s1mdTAbf5fDe01S4c46ds26YRpoWdKDCOHQRQ/mdcCd B1VLr X-Gm-Gg: ASbGncsnS2EW/NhZYX5/35/s3EiFEnPYOrv40HDhI6YNw13e8LOxh/i/7qVisrpZkcH UuUlRjrDcFk57/UHBRmLF7MbHAR3NYLSC8tHQTpvD6ExfZ6FT4OkviGVvY4qyT9fJL9eMYZyfvn 6nT1lZMjR18ybQeLAB9+IuWZx2ah3+rzgH4hRyyb7g+5P+g4SnGZi5QJTgmdv6/sqHKafYfMgGg OtBCyujv6RoARkA/7soWW6BrPsrlo+BATZkbIIEGXZiZ6gUrurTf72irdbIZg51K/5INESIDMUb qdiiejiDG87BqhJ1LFrpBx57KFLZxZX+n//ukvdB2QMmfZPPmZRwodvhvPzE40xUBbfbc+qu23O +g4Ir6Ihlp7lC32w3f84oYV3MKO7vjV4taA5cDqOyQHH2slsFbxs= X-Google-Smtp-Source: AGHT+IFIHBb4Fy9Ze6K2lwHE2swuJoP8aZRUX4H8TNx4bmX6gY4wYYsoGUu0NldgE2mCFn2Jb7BvFA== X-Received: by 2002:a05:6000:2484:b0:3a5:276b:1ec0 with SMTP id ffacd0b85a97d-3b60e51cebamr2421140f8f.45.1752679372333; Wed, 16 Jul 2025 08:22:52 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:22ad:92e1:8848:8bb0]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-3b5e8e0d872sm18362725f8f.60.2025.07.16.08.22.51 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 16 Jul 2025 08:22:51 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 2/4] cooker/process/utils: Create profiling common function to remove code duplication Date: Wed, 16 Jul 2025 16:22:47 +0100 Message-ID: <20250716152249.96126-2-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.48.1 In-Reply-To: <20250716152249.96126-1-richard.purdie@linuxfoundation.org> References: <20250716152249.96126-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 16 Jul 2025 15:22:56 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/17780 We have code duplication in the way we handle profiling of code sections. Create a common function in utils which covers this. Signed-off-by: Richard Purdie --- lib/bb/cooker.py | 16 +--------------- lib/bb/server/process.py | 33 ++------------------------------- lib/bb/utils.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 46 deletions(-) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index dc131939ed0..74b2d075d00 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -2027,21 +2027,7 @@ class Parser(multiprocessing.Process): self.exit = True def run(self): - - if not self.profile: - self.realrun() - return - - try: - import cProfile as profile - except: - import profile - prof = profile.Profile() - try: - profile.Profile.runcall(prof, self.realrun) - finally: - logfile = "profile-parse-%s.log" % multiprocessing.current_process().name - prof.dump_stats(logfile) + bb.utils.profle_function(self.profile, self.realrun, "profile-parse-%s.log" % multiprocessing.current_process().name, process=False) def realrun(self): # Signal handling here is hard. We must not terminate any process or thread holding the write diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 4b35be62cd7..78d0a606e23 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -140,23 +140,7 @@ class ProcessServer(): serverlog("Error writing to lock file: %s" % str(e)) pass - if self.cooker.configuration.profile: - try: - import cProfile as profile - except: - import profile - prof = profile.Profile() - - ret = profile.Profile.runcall(prof, self.main) - - prof.dump_stats("profile.log") - bb.utils.process_profilelog("profile.log") - serverlog("Raw profiling information saved to profile.log and processed statistics to profile.log.processed") - - else: - ret = self.main() - - return ret + return bb.utils.profle_function(self.cooker.configuration.profile, self.main, "profile.log") def _idle_check(self): return len(self._idlefuns) == 0 and self.cooker.command.currentAsyncCommand is None @@ -417,20 +401,7 @@ class ProcessServer(): serverlog("".join(msg)) def idle_thread(self): - if self.cooker.configuration.profile: - try: - import cProfile as profile - except: - import profile - prof = profile.Profile() - - ret = profile.Profile.runcall(prof, self.idle_thread_internal) - - prof.dump_stats("profile-mainloop.log") - bb.utils.process_profilelog("profile-mainloop.log") - serverlog("Raw profiling information saved to profile-mainloop.log and processed statistics to profile-mainloop.log.processed") - else: - self.idle_thread_internal() + bb.utils.profle_function(self.cooker.configuration.profile, self.idle_thread_internal, "profile-idleloop.log") def idle_thread_internal(self): def remove_idle_func(function): diff --git a/lib/bb/utils.py b/lib/bb/utils.py index 1cc74ed5466..0289862c5dd 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -1418,6 +1418,34 @@ def cpu_count(): def nonblockingfd(fd): fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) +def profle_function(profile, function, output_fn, process=True): + """Common function to profile a code block and optionally process the + output using or processing function. + + Arguments: + + - ``profile``: a boolean saying whether to enable profiling or not + - ``function``: the function call to profile/run + - ``outputfn``: where to write the profiling data + - ``process``: whether to process the profiling data and write a report + + Returns the wrapped function return value + """ + if profile: + try: + import cProfile as profile + except: + import profile + prof = profile.Profile() + ret = profile.Profile.runcall(prof, function) + prof.dump_stats(output_fn) + if process: + process_profilelog(output_fn) + serverlog("Raw profiling information saved to %s and processed statistics to %s.processed" % (output_fn, output_fn)) + return ret + else: + return function() + def process_profilelog(fn, pout = None): # Either call with a list of filenames and set pout or a filename and optionally pout. if not pout: From patchwork Wed Jul 16 15:22:48 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 66987 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id DE88BC83F27 for ; Wed, 16 Jul 2025 15:22:56 +0000 (UTC) Received: from mail-wm1-f46.google.com (mail-wm1-f46.google.com [209.85.128.46]) by mx.groups.io with SMTP id smtpd.web10.25322.1752679375887855231 for ; Wed, 16 Jul 2025 08:22:56 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=La/AM1mg; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.46, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f46.google.com with SMTP id 5b1f17b1804b1-451d6ade159so49587985e9.1 for ; Wed, 16 Jul 2025 08:22:55 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1752679374; x=1753284174; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=jvDjIny/jFjz6X9EywLy6Fk/Hf6+FQazl+h5OrxV0WM=; b=La/AM1mgnGcPH6qfuaxhzo3dvUJ2ueH7A4+PU0d0TXLX3OGVcFAx0IksVfylOTmKbA IArtA31QNmmzN2XEncfky7j8zW2B1lpJLyuBQ9QAkF+P2oUw+j3Wq2GtGX6NxmCY/Evn UFYb5YSb4Uj4ghYEbKH6RJ1r9UQbS4tDMS2Do= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1752679374; x=1753284174; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=jvDjIny/jFjz6X9EywLy6Fk/Hf6+FQazl+h5OrxV0WM=; b=aECPKUg+5fZlOdGpSAsA/CxIt0Lh4wR4pcrRha/IhEe1c//dKCCke6XlV6KcEo+IQR YvR2ZK19eKY70as+1GjgGVwx4NBtJNdyJItWGPBmkHZbepO0rQV9mCb7mAZeDC7iYs8h 8MaafU1/lM0B59WabF2uuTY8wr+4KAja0tQzF9/Dc5LD7L6iO+JTwWDr+pG+Lcx6O/IS 27lQ/1a1sRT7V24eUrvpm6DFI+48Q/8svwv04/nZSRj1RvOhvVGWP8fslbJo0Cq95wx0 BgL943xORi9N2KaFboDpzgZcNnhhUgPmjXWToh3i2Wuw4e0NMKfOoxJN/MaAK8XcUM4N feig== X-Gm-Message-State: AOJu0YyWSPQhnSFQyvzRwEi9pJNztok1WvkW6AxQtvDXxIsMfZXs8orX esRnFpF1J0WBnMURUKguY8LhBeKTl1Abf/+CPYE9gQzufN/wuCoVg72MzvXvhnNC4yHstKoI1Mg xdZjO X-Gm-Gg: ASbGncvWPns1NyKTgNij7dui5Um7DQ9lv3PnoNiOlhuJF7BKvnGwS1lt9x6KSVMJvgU PayPEZxP48wfD/3akA+Tct14izkt2LL1repgObyQ3NALkStjpnN4/1LRf0/v0GjsGqBCsHYIy0n ixSbyF5AVKpYhQO3C9r+A9k+22kev9CDviziLEfg/BgEsEfGlZJ6zpAHYic03d0bICrjyyTYWTF HPpI7073uifejyjmu4ZBxf9Po1tlaNPtEk924hoXsKx/P/Dx4lQPZAhqpAu/6pq787j5CMzY+QR yrFP0fVBQh4PN6FTYczi4kLcnHYnRVVib/oM/lM/RvH9ROqOSANnYrgrEfOL6ahEeTvipMoXHjz nAPzrmVGllHWOC/x3NpTfLMKKGNTIjAJQxVCdpMjqkVhHFAnBwPY= X-Google-Smtp-Source: AGHT+IFw6Yv33VGkQubDD99nygIAwQbVLFn9jnRYfQCQfuunfnTBkWPUtNmTvxtWha6XlRqrgnaP6w== X-Received: by 2002:a05:600c:1c0f:b0:450:d30e:ff96 with SMTP id 5b1f17b1804b1-45631f025aemr18070015e9.0.1752679373645; Wed, 16 Jul 2025 08:22:53 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:22ad:92e1:8848:8bb0]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-3b5e8e0d872sm18362725f8f.60.2025.07.16.08.22.52 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 16 Jul 2025 08:22:52 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 3/4] event: Fix an event duplication race Date: Wed, 16 Jul 2025 16:22:48 +0100 Message-ID: <20250716152249.96126-3-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.48.1 In-Reply-To: <20250716152249.96126-1-richard.purdie@linuxfoundation.org> References: <20250716152249.96126-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 16 Jul 2025 15:22:56 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/17781 It is possible for multple bitbake threads to empty ui_queue in parallel leading to duplicate console messages and much confusion when debuging. Use the lock to extract the queue data which means only one thread will processing, removing the duplicate out of order messages. Signed-off-by: Richard Purdie --- lib/bb/event.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/bb/event.py b/lib/bb/event.py index b29f0a55687..2256fed7648 100644 --- a/lib/bb/event.py +++ b/lib/bb/event.py @@ -236,9 +236,12 @@ def fire(event, d): # If messages have been queued up, clear the queue global _uiready, ui_queue if _uiready and ui_queue: - for queue_event in ui_queue: + with bb.utils.lock_timeout_nocheck(_thread_lock): + queue = ui_queue + ui_queue = [] + for queue_event in queue: fire_ui_handlers(queue_event, d) - ui_queue = [] + fire_ui_handlers(event, d) def fire_from_worker(event, d): From patchwork Wed Jul 16 15:22:49 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 66989 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id C1B72C83F1B for ; Wed, 16 Jul 2025 15:23:06 +0000 (UTC) Received: from mail-wm1-f49.google.com (mail-wm1-f49.google.com [209.85.128.49]) by mx.groups.io with SMTP id smtpd.web11.25146.1752679377200204447 for ; Wed, 16 Jul 2025 08:22:57 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=U9gSsznc; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.49, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f49.google.com with SMTP id 5b1f17b1804b1-451d6ade159so49588225e9.1 for ; Wed, 16 Jul 2025 08:22:56 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1752679375; x=1753284175; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=tVQLt1CUON5KJWr6SzIIaRU2HUAEfuMIimTjk6DBmMI=; b=U9gSszncQnR8A3XYkHDbeoCFfUrQ386YT9feWQqEhGaoxzIJdpcsGepn8OKo5Tp4KI gVcCnINijtak14WSr+xreS0Xwxj2Hd1K0Vog2lv8A9ewkUvh1RJDHgqfGNPBFOUza1qb /ZxOGQ1uNUigvrNImPgqXOGQP5n8l9PjRM6vk= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1752679375; x=1753284175; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=tVQLt1CUON5KJWr6SzIIaRU2HUAEfuMIimTjk6DBmMI=; b=E6mV0xOU6qxgib1iFnwFtjrT0L9fIS6y6qX86LjBU0NVSUKj1gdoyyVnIaBlsb489+ zxxzS+FTCcNPRvfCUutYlmb0TuDCZB8l3EaSK/F1AGU3DeCpw0esPjjusX3plPCaBf9R RFNZdWNUIoIUV7JGb6BK65a7B1wVKu6CA4CryZYy6MfxsYlQPMDZASzMGIQ2YlIC/59L WYmoRozOY/mEXB6tuTzih+5tBfm+frySWzcCjgJ/Noz4zgRk/Je7w0NyApPGFhjhxJTZ rDSMjdsryzKM9HwIMWjTMGZHXl5iGyN7GrxjOxO/tZ5BxG+YhIdXK8co6SNHjc3astXf 9Ipw== X-Gm-Message-State: AOJu0YxBlu1ZiMW6MWMhVuU97kQAuDP+jrZZmCIQnXo+uTaN4ZnSKfkg Isp/7N1uP15AtQWl52EGh5+9yjpNohnwKnJ8vjxm5iQ622k7y37mvBKveE3wVnWAx+DV73CywHB ZNrvg X-Gm-Gg: ASbGncubiLMw0CxHbRD8Wbr6xa6kMgbML6p/ls99Ap0OQCGnGtXBHKgorgheh10EXmD cmcl3XIcUJ09OAhY+pFLsQGYUkF03aRXUy/h2P1ziPIlP9f8PPNMP8ryLboNAOHk7tzoEcE/Amf GvHh6obzbHKMrZVsZHCXVy+iRTh5Y29OYWuCZUk4nUKw7JqRsSSJTg2c8+3wSDRfXVck41/iWk5 wePr0fta8xlBBWYUom43p1joVPzmjlVPgLycEWvCY7CJnPYieKZAuvxKFcyL0Gs4pHTTfk93Ejy cxud0xr3/f/Jlznp4V/YPKWjPPKJYGX31cJZBEn5WThdpViexIckLYKkwKBpJu6pCjQ7m3piMPQ Y9gTB8sNqqg0C10zBUBpMsDT8xjMetq6koqam/Yf2Ox3KvJTreBY= X-Google-Smtp-Source: AGHT+IEiGgNNKh7h0wg+9BDGZirJhZR56VI9DGnqYOpqQ7lynJonf4IMqBgaLQZ9Vv+jcQFsTlw4lQ== X-Received: by 2002:a05:6000:26d3:b0:3a4:febd:39f2 with SMTP id ffacd0b85a97d-3b60e523ce1mr2510006f8f.37.1752679374978; Wed, 16 Jul 2025 08:22:54 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:22ad:92e1:8848:8bb0]) by smtp.gmail.com with ESMTPSA id ffacd0b85a97d-3b5e8e0d872sm18362725f8f.60.2025.07.16.08.22.53 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 16 Jul 2025 08:22:53 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 4/4] main: Add an option to specify what to profile Date: Wed, 16 Jul 2025 16:22:49 +0100 Message-ID: <20250716152249.96126-4-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.48.1 In-Reply-To: <20250716152249.96126-1-richard.purdie@linuxfoundation.org> References: <20250716152249.96126-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 16 Jul 2025 15:23:06 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/17782 Starting with python 3.12, profiling now stays enabled over threads yet you can't extract the profile data in the threads themselves, which makes it difficult to use for our use case. Our main loop starts the idle loop which starts the parsing threads and this means we can't profile in the main loop and the parsing threads or the idle loop at the same time due to this. Add options to the commandline so you can specify which piece of bitbake you want to enable profiling for. This allows some profiling with python 3.12 onwards rather than crashing. Signed-off-by: Richard Purdie --- bin/bitbake-server | 2 +- lib/bb/cooker.py | 2 +- lib/bb/main.py | 6 ++++-- lib/bb/server/process.py | 6 +++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/bin/bitbake-server b/bin/bitbake-server index a559109e3f5..7a13de60de6 100755 --- a/bin/bitbake-server +++ b/bin/bitbake-server @@ -30,7 +30,7 @@ logfile = sys.argv[4] lockname = sys.argv[5] sockname = sys.argv[6] timeout = float(sys.argv[7]) -profile = bool(int(sys.argv[8])) +profile = list(sys.argv[8]) xmlrpcinterface = (sys.argv[9], int(sys.argv[10])) if xmlrpcinterface[0] == "None": xmlrpcinterface = (None, xmlrpcinterface[1]) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index 74b2d075d00..260cba1e212 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -2027,7 +2027,7 @@ class Parser(multiprocessing.Process): self.exit = True def run(self): - bb.utils.profle_function(self.profile, self.realrun, "profile-parse-%s.log" % multiprocessing.current_process().name, process=False) + bb.utils.profle_function("parsing" in self.profile, self.realrun, "profile-parse-%s.log" % multiprocessing.current_process().name, process=False) def realrun(self): # Signal handling here is hard. We must not terminate any process or thread holding the write diff --git a/lib/bb/main.py b/lib/bb/main.py index bca8ebfa090..597cb278464 100755 --- a/lib/bb/main.py +++ b/lib/bb/main.py @@ -208,8 +208,10 @@ def create_bitbake_parser(): "failed and anything depending on it cannot be built, as much as " "possible will be built before stopping.") - exec_group.add_argument("-P", "--profile", action="store_true", - help="Profile the command and save reports.") + exec_group.add_argument("-P", "--profile", action="append", + default=[], + help="Profile the command and save reports. Specify 'main', 'idle' or 'parsing' " + "to indicate which bitbake code to profile.") exec_group.add_argument("-S", "--dump-signatures", action="append", default=[], metavar="SIGNATURE_HANDLER", diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 78d0a606e23..da3f833f1d5 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -140,7 +140,7 @@ class ProcessServer(): serverlog("Error writing to lock file: %s" % str(e)) pass - return bb.utils.profle_function(self.cooker.configuration.profile, self.main, "profile.log") + return bb.utils.profle_function("main" in self.cooker.configuration.profile, self.main, "profile.log") def _idle_check(self): return len(self._idlefuns) == 0 and self.cooker.command.currentAsyncCommand is None @@ -401,7 +401,7 @@ class ProcessServer(): serverlog("".join(msg)) def idle_thread(self): - bb.utils.profle_function(self.cooker.configuration.profile, self.idle_thread_internal, "profile-idleloop.log") + bb.utils.profle_function("idle" in self.cooker.configuration.profile, self.idle_thread_internal, "profile-idleloop.log") def idle_thread_internal(self): def remove_idle_func(function): @@ -603,7 +603,7 @@ class BitBakeServer(object): os.set_inheritable(self.bitbake_lock.fileno(), True) os.set_inheritable(self.readypipein, True) serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server") - os.execl(sys.executable, sys.executable, serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout or 0), str(int(self.profile)), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1])) + os.execl(sys.executable, sys.executable, serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout or 0), str(list(self.profile)), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1])) def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface, profile):