From patchwork Thu Dec 22 23:47:17 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17143 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 9A305C46467 for ; Thu, 22 Dec 2022 23:47:34 +0000 (UTC) Received: from mail-wm1-f49.google.com (mail-wm1-f49.google.com [209.85.128.49]) by mx.groups.io with SMTP id smtpd.web11.60900.1671752850177943149 for ; Thu, 22 Dec 2022 15:47:30 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=BrgdUntP; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.49, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f49.google.com with SMTP id o5-20020a05600c510500b003d21f02fbaaso5047511wms.4 for ; Thu, 22 Dec 2022 15:47:29 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:message-id:date:subject:to :from:from:to:cc:subject:date:message-id:reply-to; bh=rjUR8wW045Dmbgzbvu2jqdBO9AgPEXWTs4vUQPlmhSM=; b=BrgdUntPSG4nmPkQLFhLvC7LW6I+FpfgpThlVc4aT0Lo7dnrlT2Uxefe+iG08xJqmf dPLH3WCmN2DrXKccjSqEs77V7NTBpSaKFXszzJ0ytZl3CdnONKWs/h1TF36lf/jzGXwa edrajH/0nOqpM2sYxBArc66ZwyjR6vYzGCrm8= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:message-id:date:subject:to :from:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=rjUR8wW045Dmbgzbvu2jqdBO9AgPEXWTs4vUQPlmhSM=; b=MOtCRafGkjvUOTRCC5c7jT0QwRfayjj61jlGGZsqgsch0r+pVNrcJjsNXF/ZqQvb9x 6nqZKl27JfFBI4AFvuVJJ0/AJXPxIW2RhNxoIfEC3Qk4RlDGQrtzfTjf9342s8XnRp1z UF2Ov1kLwingtkVjO/jh6hKNk3O4TVjPE89ILyUYWNkLVgGv/97mCutFP4QYN5LbA8LS c35u0oUDdxcvUbFoaPt653j7xMdRmJnAlYKA86j4Lm8HlzTQhwRd8krIy359NRUfAqw0 YHVCAsuvTYekXWeNMjf46Pidoc0IgtXjMTWop4pDzQOFAgSRqx9ImKTRzIjqBdVDU5My ib2g== X-Gm-Message-State: AFqh2kqyEd5G3vtLS+NgBg5sHGDmf8FDBC2JVQCHNNFJwMKy9rXWOUZu YPV1L/4ZzrL55Pfu2wtQjMPOfFvPYAGzW38d X-Google-Smtp-Source: AMrXdXvSzknU4A988DZ0PpI6oXtpE+KXQcSO3rgnNM6US+NzePebuZwy1uSdEyXlZE5nj8ee0vNNhA== X-Received: by 2002:a05:600c:600a:b0:3d1:ed41:57c0 with SMTP id az10-20020a05600c600a00b003d1ed4157c0mr8406933wmb.30.1671752847802; Thu, 22 Dec 2022 15:47:27 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:bc21:fec7:83cf:e0c3]) by smtp.gmail.com with ESMTPSA id bd25-20020a05600c1f1900b003cfd4cf0761sm7322189wmb.1.2022.12.22.15.47.27 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 22 Dec 2022 15:47:27 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 01/10] knotty: Ping the server/cooker periodically Date: Thu, 22 Dec 2022 23:47:17 +0000 Message-Id: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 22 Dec 2022 23:47:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14231 We've seeing failures where the UI hangs if the server disappears. Ping the cooker/server if we've not had any events in the last minute so we can check if it is still alive. Signed-off-by: Richard Purdie --- lib/bb/ui/knotty.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/bb/ui/knotty.py b/lib/bb/ui/knotty.py index 61cf0a37f4..ab1a367be0 100644 --- a/lib/bb/ui/knotty.py +++ b/lib/bb/ui/knotty.py @@ -625,7 +625,8 @@ def main(server, eventHandler, params, tf = TerminalFilter): printintervaldelta = 10 * 60 # 10 minutes printinterval = printintervaldelta - lastprint = time.time() + pinginterval = 1 * 60 # 1 minutes + lastevent = lastprint = time.time() termfilter = tf(main, helper, console_handlers, params.options.quiet) atexit.register(termfilter.finish) @@ -637,6 +638,14 @@ def main(server, eventHandler, params, tf = TerminalFilter): printinterval += printintervaldelta event = eventHandler.waitEvent(0) if event is None: + if (lastevent + pinginterval) <= time.time(): + ret, error = server.runCommand(["ping"]) + if error or not ret: + termfilter.clearFooter() + print("No reply after pinging server (%s, %s), exiting." % (str(error), str(ret))) + return_value = 3 + main.shutdown = 2 + lastevent = time.time() if main.shutdown > 1: break if not parseprogress: @@ -644,6 +653,7 @@ def main(server, eventHandler, params, tf = TerminalFilter): event = eventHandler.waitEvent(0.25) if event is None: continue + lastevent = time.time() helper.eventHandler(event) if isinstance(event, bb.runqueue.runQueueExitWait): if not main.shutdown: From patchwork Thu Dec 22 23:47:18 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17139 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 7D4FCC4167B for ; Thu, 22 Dec 2022 23:47:34 +0000 (UTC) Received: from mail-wm1-f52.google.com (mail-wm1-f52.google.com [209.85.128.52]) by mx.groups.io with SMTP id smtpd.web11.60899.1671752850177769269 for ; Thu, 22 Dec 2022 15:47:30 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=dnPp/g9W; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.52, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f52.google.com with SMTP id v124-20020a1cac82000000b003cf7a4ea2caso5044507wme.5 for ; Thu, 22 Dec 2022 15:47:29 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=pN4lANml8++f8XUrrE/hEDZrzvwj32RGT8aPCf00bUM=; b=dnPp/g9WU6G1nbqbofE9oFxKnPczONi8etRQcRgFazwaocA6qM/iyax5J5wdNOGFJ8 aBrnFZD3Y0BQRGAJlFy2Zhj9PRAwCjVfhq7hEp3g3Xp5+0YmY0BInSqhm+9OrYuMPOhd Q9n5LHOI5dEiA2xA06BsD1TAnIorV1nmMdas8= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=pN4lANml8++f8XUrrE/hEDZrzvwj32RGT8aPCf00bUM=; b=YYuu1xDsvjkxxZxoZZaWeLg8DC+TuU8Xlc8fLwkHmrE4gqZCd1EySDfU1fjZ1kaXen vvN7mo6nz3TKCI0OJdSmBBdhnQou4RqVgtBK7bub7JJ8ckBR+tLhrq7J4KOM61p+8pEt QyPGU+oXBIbNpr9YMO5Gu5Fs1hxcpryfHbf8WkLBcPV0xg58aypuuoMDQOY3Gf8Zm7uJ 4cca88J0J2fk3QqAQ61EQf9/c8aIX8S8LyLNdUZhlr+O+9xvPk+mH+9jleFx2PcwBi7b RvIUOu//T33oYw5egNEXy367AjIPUbs0IkWFfocrvgvLdWEDYZICSpG8ekDjsEGQFgyh fiZg== X-Gm-Message-State: AFqh2kqKsKJEv3Wlb3nuHBGjbmyy8tVQoagL1JY1tJgWfPhdf1KJj0IY CAi6cqefp2u7kzmwij2fOtdmQmlE5SiyKUAB X-Google-Smtp-Source: AMrXdXtMw1ht6NCY/ECvbHfCmoNZ64maBvNE6rdfJ1P96RjnGbH/Cb3+Sust5nVsb65DsnWgT4rQ0g== X-Received: by 2002:a05:600c:3596:b0:3cf:497c:c4f5 with SMTP id p22-20020a05600c359600b003cf497cc4f5mr5783882wmq.13.1671752848329; Thu, 22 Dec 2022 15:47:28 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:bc21:fec7:83cf:e0c3]) by smtp.gmail.com with ESMTPSA id bd25-20020a05600c1f1900b003cfd4cf0761sm7322189wmb.1.2022.12.22.15.47.27 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 22 Dec 2022 15:47:28 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 02/10] event: Add enable/disable heartbeat code Date: Thu, 22 Dec 2022 23:47:18 +0000 Message-Id: <20221222234726.579702-2-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> References: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 22 Dec 2022 23:47:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14230 Currently heartbeat events are always generated by the server whilst it is active. Change this so they only appear when builds are running, which is when most code would expect to be executed. This removes a number of races around changes in the datastore which can happen outside of builds. Signed-off-by: Richard Purdie --- lib/bb/cooker.py | 4 ++++ lib/bb/event.py | 9 +++++++++ lib/bb/server/process.py | 4 ++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index 48c3002ce3..815610ff82 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -1467,6 +1467,7 @@ class BBCooker: buildname = self.databuilder.mcdata[mc].getVar("BUILDNAME") if fireevents: bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.databuilder.mcdata[mc]) + bb.event.enable_heartbeat() # Execute the runqueue runlist = [[mc, item, task, fn]] @@ -1500,6 +1501,7 @@ class BBCooker: if not retval: if fireevents: bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.databuilder.mcdata[mc]) + bb.event.disable_heartbeat() self.command.finishAsyncCommand(msg) # We trashed self.recipecaches above self.parsecache_valid = False @@ -1545,6 +1547,7 @@ class BBCooker: for mc in self.multiconfigs: bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.databuilder.mcdata[mc]) finally: + bb.event.disable_heartbeat() self.command.finishAsyncCommand(msg) return False if retval is True: @@ -1578,6 +1581,7 @@ class BBCooker: for mc in self.multiconfigs: bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.databuilder.mcdata[mc]) + bb.event.enable_heartbeat() rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) if 'universe' in targets: diff --git a/lib/bb/event.py b/lib/bb/event.py index 97668601a1..21e9a1b025 100644 --- a/lib/bb/event.py +++ b/lib/bb/event.py @@ -69,6 +69,7 @@ _eventfilter = None _uiready = False _thread_lock = threading.Lock() _thread_lock_enabled = False +_heartbeat_enabled = False if hasattr(__builtins__, '__setitem__'): builtins = __builtins__ @@ -83,6 +84,14 @@ def disable_threadlock(): global _thread_lock_enabled _thread_lock_enabled = False +def enable_heartbeat(): + global _heartbeat_enabled + _heartbeat_enabled = True + +def disable_heartbeat(): + global _heartbeat_enabled + _heartbeat_enabled = False + def execute_handler(name, handler, event, d): event.data = d addedd = False diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 586d46af88..91eb6e0ad9 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -382,7 +382,7 @@ class ProcessServer(): # Create new heartbeat event? now = time.time() - if now >= self.next_heartbeat: + if bb.event._heartbeat_enabled and now >= self.next_heartbeat: # We might have missed heartbeats. Just trigger once in # that case and continue after the usual delay. self.next_heartbeat += self.heartbeat_seconds @@ -396,7 +396,7 @@ class ProcessServer(): if not isinstance(exc, bb.BBHandledException): logger.exception('Running heartbeat function') self.quit = True - if nextsleep and now + nextsleep > self.next_heartbeat: + if nextsleep and bb.event._heartbeat_enabled and now + nextsleep > self.next_heartbeat: # Shorten timeout so that we we wake up in time for # the heartbeat. nextsleep = self.next_heartbeat - now From patchwork Thu Dec 22 23:47:19 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17142 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 93AC3C4708D for ; Thu, 22 Dec 2022 23:47:34 +0000 (UTC) Received: from mail-wm1-f41.google.com (mail-wm1-f41.google.com [209.85.128.41]) by mx.groups.io with SMTP id smtpd.web11.60901.1671752850665460756 for ; Thu, 22 Dec 2022 15:47:31 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=hJynPUNK; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.41, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f41.google.com with SMTP id o15so2564662wmr.4 for ; Thu, 22 Dec 2022 15:47:30 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=dHHlILX9B2Dcb3I3IxVcze1ZIeSnGaT/zwJmYdZTp9I=; b=hJynPUNKBJjdhgZ6eYj+FfJT6/vzyoVgW+ZGoUhEkwxJZt5Cp2SejXbrxJMq568AO1 jUvyapEnbppagpn5zGoM2FlSx6Dx7pKFbJzQSV+uwqcx2N8qOd+ak8iGO171sieOrEGX Jur0xhS6TutqYFdc+cowtXV5q9WNmWe4n+QQc= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=dHHlILX9B2Dcb3I3IxVcze1ZIeSnGaT/zwJmYdZTp9I=; b=CNo94XALtrDPpeL9W8uTjXEtDDxJARLm6zI2b0yETzDX6UYxRgNXNRh8jyS/EdeNk1 beAS/eQ55IVIhre37s4nnZyw7uFertwZ+KuS/Dn1Oi/0guPBFpuO0lIc1KzIzp8n2LEm cF2dKPnO5o4hZOeeWapLDqiqAH9o+HEfO+iALi6CtnmU3kftzp4H6//517fDbAODrhHG TL0qavOk6Hxf92UugF4Oa6akAnmphS9lSYolMlTHz79B9dawbeS6hWRFQry2Lwb940tW JOCNjbsJvSMTH/Oe0MJPfwzYest1/cGfbl819FbhPDWys4YaiGJPWsQyEl34Fp9FxB3L 6hgA== X-Gm-Message-State: AFqh2kotMUe12MIh6Z5dBXeHvb0EddRZ7XVJ4YX5/aRjOPoiFAHnUpYl BJRlVs1sLIQc8g1lO9rQRtZCMXVmp7xzB5WX X-Google-Smtp-Source: AMrXdXtIlNXwQDE22k4pcl4ysnJvSyReXtKPHpYQBCmrize1BWZ0EwUJ1WG7zcKW1zMcgDBZHqTfMA== X-Received: by 2002:a7b:c4d3:0:b0:3d3:5cd6:760 with SMTP id g19-20020a7bc4d3000000b003d35cd60760mr6403557wmk.20.1671752848941; Thu, 22 Dec 2022 15:47:28 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:bc21:fec7:83cf:e0c3]) by smtp.gmail.com with ESMTPSA id bd25-20020a05600c1f1900b003cfd4cf0761sm7322189wmb.1.2022.12.22.15.47.28 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 22 Dec 2022 15:47:28 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 03/10] event: builtins fix for 'd' deletion Date: Thu, 22 Dec 2022 23:47:19 +0000 Message-Id: <20221222234726.579702-3-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> References: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 22 Dec 2022 23:47:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14232 I've been seeing event handlers where 'd' seems to disappear half way through event handler execution. This is problematic when multiple threads are active since this code assumes single threading. The easiest fix is to change the handler function calls to contain d as a parameter as we do elsewhere for other functions. This will break any non-text handlers but I was only able to spot one of those in runqueue. It will also break handlers than call functions that assume 'd' is in the global namespace but those failures should be obvious and we can fix those to pass d around. This solution avoids manipulating builtins which was always a horrible thing to do anyway and solves the issue without needing locking, thankfully. Signed-off-by: Richard Purdie --- lib/bb/event.py | 18 ++++-------------- lib/bb/runqueue.py | 2 +- lib/bb/tests/color.py | 2 +- lib/bb/tests/event.py | 21 +++++++++++---------- 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/lib/bb/event.py b/lib/bb/event.py index 21e9a1b025..db90724444 100644 --- a/lib/bb/event.py +++ b/lib/bb/event.py @@ -71,11 +71,6 @@ _thread_lock = threading.Lock() _thread_lock_enabled = False _heartbeat_enabled = False -if hasattr(__builtins__, '__setitem__'): - builtins = __builtins__ -else: - builtins = __builtins__.__dict__ - def enable_threadlock(): global _thread_lock_enabled _thread_lock_enabled = True @@ -94,12 +89,8 @@ def disable_heartbeat(): def execute_handler(name, handler, event, d): event.data = d - addedd = False - if 'd' not in builtins: - builtins['d'] = d - addedd = True try: - ret = handler(event) + ret = handler(event, d) except (bb.parse.SkipRecipe, bb.BBHandledException): raise except Exception: @@ -113,8 +104,7 @@ def execute_handler(name, handler, event, d): raise finally: del event.data - if addedd: - del builtins['d'] + def fire_class_handlers(event, d): if isinstance(event, logging.LogRecord): @@ -262,12 +252,12 @@ def register(name, handler, mask=None, filename=None, lineno=None, data=None): if handler is not None: # handle string containing python code if isinstance(handler, str): - tmp = "def %s(e):\n%s" % (name, handler) + tmp = "def %s(e, d):\n%s" % (name, handler) try: code = bb.methodpool.compile_cache(tmp) if not code: if filename is None: - filename = "%s(e)" % name + filename = "%s(e, d)" % name code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST) if lineno is not None: ast.increment_lineno(code, lineno-1) diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index b9dd830b31..ce711b6252 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -1511,7 +1511,7 @@ class RunQueue: if not self.dm_event_handler_registered: res = bb.event.register(self.dm_event_handler_name, - lambda x: self.dm.check(self) if self.state in [runQueueRunning, runQueueCleanUp] else False, + lambda x, y: self.dm.check(self) if self.state in [runQueueRunning, runQueueCleanUp] else False, ('bb.event.HeartbeatEvent',), data=self.cfgData) self.dm_event_handler_registered = True diff --git a/lib/bb/tests/color.py b/lib/bb/tests/color.py index 88dd278006..bb70cb393d 100644 --- a/lib/bb/tests/color.py +++ b/lib/bb/tests/color.py @@ -20,7 +20,7 @@ class ProgressWatcher: def __init__(self): self._reports = [] - def handle_event(self, event): + def handle_event(self, event, d): self._reports.append((event.progress, event.rate)) def reports(self): diff --git a/lib/bb/tests/event.py b/lib/bb/tests/event.py index 9ca7e9bc8e..4de4cced5e 100644 --- a/lib/bb/tests/event.py +++ b/lib/bb/tests/event.py @@ -157,7 +157,7 @@ class EventHandlingTest(unittest.TestCase): self._test_process.event_handler, event, None) - self._test_process.event_handler.assert_called_once_with(event) + self._test_process.event_handler.assert_called_once_with(event, None) def test_fire_class_handlers(self): """ Test fire_class_handlers method """ @@ -175,10 +175,10 @@ class EventHandlingTest(unittest.TestCase): bb.event.fire_class_handlers(event1, None) bb.event.fire_class_handlers(event2, None) bb.event.fire_class_handlers(event2, None) - expected_event_handler1 = [call(event1)] - expected_event_handler2 = [call(event1), - call(event2), - call(event2)] + expected_event_handler1 = [call(event1, None)] + expected_event_handler2 = [call(event1, None), + call(event2, None), + call(event2, None)] self.assertEqual(self._test_process.event_handler1.call_args_list, expected_event_handler1) self.assertEqual(self._test_process.event_handler2.call_args_list, @@ -205,7 +205,7 @@ class EventHandlingTest(unittest.TestCase): bb.event.fire_class_handlers(event2, None) bb.event.fire_class_handlers(event2, None) expected_event_handler1 = [] - expected_event_handler2 = [call(event1)] + expected_event_handler2 = [call(event1, None)] self.assertEqual(self._test_process.event_handler1.call_args_list, expected_event_handler1) self.assertEqual(self._test_process.event_handler2.call_args_list, @@ -223,7 +223,7 @@ class EventHandlingTest(unittest.TestCase): self.assertEqual(result, bb.event.Registered) bb.event.fire_class_handlers(event1, None) bb.event.fire_class_handlers(event2, None) - expected = [call(event1), call(event2)] + expected = [call(event1, None), call(event2, None)] self.assertEqual(self._test_process.event_handler1.call_args_list, expected) @@ -237,7 +237,7 @@ class EventHandlingTest(unittest.TestCase): self.assertEqual(result, bb.event.Registered) bb.event.fire_class_handlers(event1, None) bb.event.fire_class_handlers(event2, None) - expected = [call(event1), call(event2), call(event1)] + expected = [call(event1, None), call(event2, None), call(event1, None)] self.assertEqual(self._test_process.event_handler1.call_args_list, expected) @@ -251,7 +251,7 @@ class EventHandlingTest(unittest.TestCase): self.assertEqual(result, bb.event.Registered) bb.event.fire_class_handlers(event1, None) bb.event.fire_class_handlers(event2, None) - expected = [call(event1), call(event2), call(event1), call(event2)] + expected = [call(event1,None), call(event2, None), call(event1, None), call(event2, None)] self.assertEqual(self._test_process.event_handler1.call_args_list, expected) @@ -359,9 +359,10 @@ class EventHandlingTest(unittest.TestCase): event1 = bb.event.ConfigParsed() bb.event.fire(event1, None) - expected = [call(event1)] + expected = [call(event1, None)] self.assertEqual(self._test_process.event_handler1.call_args_list, expected) + expected = [call(event1)] self.assertEqual(self._test_ui1.event.send.call_args_list, expected) From patchwork Thu Dec 22 23:47:20 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17141 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 92479C3DA7D for ; Thu, 22 Dec 2022 23:47:34 +0000 (UTC) Received: from mail-wm1-f42.google.com (mail-wm1-f42.google.com [209.85.128.42]) by mx.groups.io with SMTP id smtpd.web11.60902.1671752851328459093 for ; Thu, 22 Dec 2022 15:47:31 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=AgT/NRyg; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.42, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f42.google.com with SMTP id b24-20020a05600c4a9800b003d21efdd61dso2453560wmp.3 for ; Thu, 22 Dec 2022 15:47:31 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=758nh+IjW+PDtVxDuFX7FJkJ6XmbtKXSkAdsuA5dm5M=; b=AgT/NRyggBnXNMnlSm6DqjSYg4KR2O7yt6INUDboWud5MFzth2uVaHlcsGCKsAg9wD DFgFWhOc5JrbEwRw4GxDadujPG/gPgq6WRGaT9f/nqVDnJqRDyhXQymDC8ErYDF3hbjA PD8YI0JhoPYS1S1QmDijUbrG7yjtCY+uzYI+Q= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=758nh+IjW+PDtVxDuFX7FJkJ6XmbtKXSkAdsuA5dm5M=; b=I9YX8wavaofP4qCK5SSPUn/DyxZ3DzmvCzAu9LHukVTfmcSj6LqwP4H/Oa5r9okMoR PEqG2FCPVICYOF1anVnKwFTc5K7biNLcvWLA5j1Ki/PLV4c43IImABrmkIhAnbiHbiRS KC3reMMV613KMtagu8iTx+Wjhx0JG3egzf0u4In2xkfCCHU1/ugW8osMTArjgQYj7/k6 gRMn+e1sBFqBnzKJqN+cV/DqL3VMj5i+8DMIMVYSmq3iR15SY9CTtgIxxVKVBifGBoeG iE0clINllvoPiVCbtIVvNt3f+BuHWQDW1bEykMAWPoFrtcn8r6LNPpIXzKygyb+N8Fex ql9Q== X-Gm-Message-State: AFqh2kpvG9Iw8ojwttSeLdoSzpZMkStGIpFxw5Avb45V/9K2qKk8v6lm otO9KKhaN71+d50bukzojgGwx/X5tGnK5GW3 X-Google-Smtp-Source: AMrXdXtCTcwSMjul1A0pRtx4ASVz1XvnbeiVryayu952/h82kRB5L5Vg7K98/sfc00ju/DicZqHXzg== X-Received: by 2002:a7b:c3c9:0:b0:3d2:e28:647f with SMTP id t9-20020a7bc3c9000000b003d20e28647fmr6336037wmj.15.1671752849607; Thu, 22 Dec 2022 15:47:29 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:bc21:fec7:83cf:e0c3]) by smtp.gmail.com with ESMTPSA id bd25-20020a05600c1f1900b003cfd4cf0761sm7322189wmb.1.2022.12.22.15.47.29 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 22 Dec 2022 15:47:29 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 04/10] server/process: Run idle commands in a separate idle thread Date: Thu, 22 Dec 2022 23:47:20 +0000 Message-Id: <20221222234726.579702-4-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> References: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 22 Dec 2022 23:47:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14233 When bitbake is off running heavier "idle" commands, it doesn't service it's command socket which means stopping/interrupting it is hard. It also means we can't "ping" from the UI to know if it is still alive. For those reasons, split idle command execution into it's own thread. The commands are generally already self containted so this is easier than expected. We do have to be careful to only handle inotify poll() from a single thread at a time. It also means we always have to use a thread lock when sending events since both the idle thread and the command thread may generate log messages (and hence events). The patch does depend on a couple of previous fixes to the builtins locking in event.py and the heartbeat enable/disable changes. Signed-off-by: Richard Purdie --- lib/bb/command.py | 4 +-- lib/bb/cooker.py | 19 +++++++++---- lib/bb/server/process.py | 60 +++++++++++++++++++++++++--------------- 3 files changed, 54 insertions(+), 29 deletions(-) diff --git a/lib/bb/command.py b/lib/bb/command.py index 0208e30ec4..20a8b86653 100644 --- a/lib/bb/command.py +++ b/lib/bb/command.py @@ -84,7 +84,7 @@ class Command: if not hasattr(command_method, 'readonly') or not getattr(command_method, 'readonly'): return None, "Not able to execute not readonly commands in readonly mode" try: - self.cooker.process_inotify_updates() + self.cooker.process_inotify_updates_apply() if getattr(command_method, 'needconfig', True): self.cooker.updateCacheSync() result = command_method(self, commandline) @@ -109,7 +109,7 @@ class Command: def runAsyncCommand(self): try: - self.cooker.process_inotify_updates() + self.cooker.process_inotify_updates_apply() if self.cooker.state in (bb.cooker.state.error, bb.cooker.state.shutdown, bb.cooker.state.forceshutdown): # updateCache will trigger a shutdown of the parser # and then raise BBHandledException triggering an exit diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index 815610ff82..1daa587ac5 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -220,6 +220,8 @@ class BBCooker: bb.debug(1, "BBCooker startup complete %s" % time.time()) sys.stdout.flush() + self.inotify_threadlock = threading.Lock() + def init_configdata(self): if not hasattr(self, "data"): self.initConfigurationData() @@ -248,11 +250,18 @@ class BBCooker: self.notifier = pyinotify.Notifier(self.watcher, self.notifications) def process_inotify_updates(self): - for n in [self.confignotifier, self.notifier]: - if n and n.check_events(timeout=0): - # read notified events and enqueue them - n.read_events() - n.process_events() + with self.inotify_threadlock: + for n in [self.confignotifier, self.notifier]: + if n and n.check_events(timeout=0): + # read notified events and enqueue them + n.read_events() + + def process_inotify_updates_apply(self): + with self.inotify_threadlock: + for n in [self.confignotifier, self.notifier]: + if n and n.check_events(timeout=0): + n.read_events() + n.process_events() def config_notifications(self, event): if event.maskname == "IN_Q_OVERFLOW": diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 91eb6e0ad9..6f43330fae 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -88,6 +88,7 @@ class ProcessServer(): self.maxuiwait = 30 self.xmlrpc = False + self.idle = None self._idlefuns = {} self.bitbake_lock = lock @@ -148,6 +149,7 @@ class ProcessServer(): self.cooker.pre_serve() bb.utils.set_process_name("Cooker") + bb.event.enable_threadlock() ready = [] newconnections = [] @@ -278,6 +280,9 @@ class ProcessServer(): ready = self.idle_commands(.1, fds) + if self.idle: + self.idle.join() + serverlog("Exiting (socket: %s)" % os.path.exists(self.sockname)) # Remove the socket file so we don't get any more connections to avoid races # The build directory could have been renamed so if the file isn't the one we created @@ -352,33 +357,44 @@ class ProcessServer(): msg.append(":\n%s" % procs) serverlog("".join(msg)) + def idle_thread(self): + while not self.quit: + nextsleep = 0.1 + fds = [] + for function, data in list(self._idlefuns.items()): + try: + retval = function(self, data, False) + if retval is False: + del self._idlefuns[function] + nextsleep = None + elif retval is True: + nextsleep = None + elif isinstance(retval, float) and nextsleep: + if (retval < nextsleep): + nextsleep = retval + elif nextsleep is None: + continue + else: + fds = fds + retval + except SystemExit: + raise + except Exception as exc: + if not isinstance(exc, bb.BBHandledException): + logger.exception('Running idle function') + del self._idlefuns[function] + self.quit = True + + if nextsleep is not None: + select.select(fds,[],[],nextsleep)[0] + def idle_commands(self, delay, fds=None): nextsleep = delay if not fds: fds = [] - for function, data in list(self._idlefuns.items()): - try: - retval = function(self, data, False) - if retval is False: - del self._idlefuns[function] - nextsleep = None - elif retval is True: - nextsleep = None - elif isinstance(retval, float) and nextsleep: - if (retval < nextsleep): - nextsleep = retval - elif nextsleep is None: - continue - else: - fds = fds + retval - except SystemExit: - raise - except Exception as exc: - if not isinstance(exc, bb.BBHandledException): - logger.exception('Running idle function') - del self._idlefuns[function] - self.quit = True + if not self.idle: + self.idle = threading.Thread(target=self.idle_thread) + self.idle.start() # Create new heartbeat event? now = time.time() From patchwork Thu Dec 22 23:47:21 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17138 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 8A9AFC3DA7A for ; Thu, 22 Dec 2022 23:47:34 +0000 (UTC) Received: from mail-wm1-f44.google.com (mail-wm1-f44.google.com [209.85.128.44]) by mx.groups.io with SMTP id smtpd.web11.60903.1671752851891527583 for ; Thu, 22 Dec 2022 15:47:32 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=BVQAvzxY; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.44, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f44.google.com with SMTP id l26so991997wme.5 for ; Thu, 22 Dec 2022 15:47:31 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=IfSl+kt8RGg2crfqf5/uzyzyb3yLXfwRohuX1rbJIVg=; b=BVQAvzxY7KNY6c5WzRBSBYfMIlDos27bnEKKQTVF1STEXGjvh8Gx94BwAKbjur1qPH cel311oYH1dRvK6Jzi7pQnTi1l1Hi9BEE51Db5KFcpTC1ut+xU7YbMS2FOtWIBIyJGfo 6ZoP9sDcrtR0FOqrF8lNoT14GwU2PhIgOi56U= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=IfSl+kt8RGg2crfqf5/uzyzyb3yLXfwRohuX1rbJIVg=; b=y7/kHsB8TQCdGOZnLR6ZpOO9vPMCiAMeCj1loPcyl0p0AL7A5fTkxWCHTqSoaUG++p Nfrrisg4LG67rNyC1qSXYa+eoBCR+9kL33WGjxzl33suUoTJpbvMJK4vFNfMAiXnibQW bAHhL/QB5T75Ak71CtIA980qIezfWRsHpvQftTwYsYfo97uz51hPowA2lmyMLO19ObL5 wENdRVayoJfS4hq9ihGsD1fPx09DD4ADIahmvSAqJ2b8zvOS1muvS49RwbKTfuWaYilJ Hcc1W8xMqcjIw19JlPvYaA3qeLM++5MnZVh+N0lsx+3cQk8eCMIWZa4d+74Q1gBy5SGa NcjQ== X-Gm-Message-State: AFqh2kpIyP/bzXfbGUi76mTjFnkg/BGoqkB3dvBn/bA5I8m61KulWrdg aGhmJHzbVJviDeeIZ9wCE0EXpLLnVqVzrzTf X-Google-Smtp-Source: AMrXdXurBr1ZtSiUarI2aY/dPANjzxXmwMbuwaLL3i+pU0LPZj7z/Zt/NWLwubg8xxoXzjY2fJKnGg== X-Received: by 2002:a05:600c:a4d:b0:3cf:6e78:e2ca with SMTP id c13-20020a05600c0a4d00b003cf6e78e2camr8203809wmq.5.1671752850267; Thu, 22 Dec 2022 15:47:30 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:bc21:fec7:83cf:e0c3]) by smtp.gmail.com with ESMTPSA id bd25-20020a05600c1f1900b003cfd4cf0761sm7322189wmb.1.2022.12.22.15.47.29 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 22 Dec 2022 15:47:29 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 05/10] process: Improve client disconnect/idle sync Date: Thu, 22 Dec 2022 23:47:21 +0000 Message-Id: <20221222234726.579702-5-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> References: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 22 Dec 2022 23:47:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14235 When clients are disconnecting, we need to ensure things are in sync with the idle thread. Use a threading event to show when things drop back to idle (there is only one always present idle handler for inotify). Later patches should be able to clean up the inotify handler to work differently and avoid hardcoded values. If we're not idle at client disconnect, shut down the server to solve the dilemma. Not ideal but shouldn't happen much in practise and would be restarted. Signed-off-by: Richard Purdie --- lib/bb/server/process.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 6f43330fae..895e39cd0f 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -90,6 +90,7 @@ class ProcessServer(): self.idle = None self._idlefuns = {} + self.is_idle = threading.Event() self.bitbake_lock = lock self.bitbake_lock_name = lockname @@ -169,6 +170,13 @@ class ProcessServer(): self.controllersock.close() self.controllersock = False if self.haveui: + # Wait for the idle loop to have cleared (30s max) + self.is_idle.clear() + self.is_idle.wait(timeout=30) + if self.cooker.command.currentAsyncCommand is not None: + serverlog("Idle loop didn't finish queued commands after 30s, exiting.") + self.quit = True + fds.remove(self.command_channel) bb.event.unregister_UIHhandler(self.event_handle, True) self.command_channel_reply.writer.close() @@ -180,7 +188,7 @@ class ProcessServer(): self.cooker.clientComplete() self.haveui = False ready = select.select(fds,[],[],0)[0] - if newconnections: + if newconnections and not self.quit: serverlog("Starting new client") conn = newconnections.pop(-1) fds.append(conn) @@ -384,6 +392,10 @@ class ProcessServer(): del self._idlefuns[function] self.quit = True + # FIXME - the 1 is the inotify processing in cooker which always runs + if len(self._idlefuns) <= 1: + self.is_idle.set() + if nextsleep is not None: select.select(fds,[],[],nextsleep)[0] From patchwork Thu Dec 22 23:47:22 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17140 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 88113C3DA7C for ; Thu, 22 Dec 2022 23:47:34 +0000 (UTC) Received: from mail-wm1-f41.google.com (mail-wm1-f41.google.com [209.85.128.41]) by mx.groups.io with SMTP id smtpd.web11.60901.1671752850665460756 for ; Thu, 22 Dec 2022 15:47:32 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=XOn5W+Pf; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.41, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f41.google.com with SMTP id o15so2564699wmr.4 for ; Thu, 22 Dec 2022 15:47:31 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=TAKaIm/rM9hNwILCcwB2RH+YraJeg3L9ZWheYT2NgnM=; b=XOn5W+PfEEvBjzPeeZpM/z950i+qxhPVl7+cmpeHNlh0F/eFJJgynsYexyFhM3iw0P 3Ye2FS7aSOlUZoplUViSOT4cHL7NI4zJj285I6LYdE9cXZh/NDrMze7Okrk+N+EVAMFG jNI0muQ4YXWxhF9ROqnDJNRwXS0LNzWgPbTYA= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=TAKaIm/rM9hNwILCcwB2RH+YraJeg3L9ZWheYT2NgnM=; b=UfQ6TbPtoI7PH2ZfeReGyLQkAtlXjURrGuUDlV6y4Bz92UbEnBRnAQSQZHRBwT3vAq ahqfyHey7XVW0qNx+zHb6sz4MMUSVY6/u7xnao4QYbgI80MvwG6j+PoCQE2WSyVp3gTO A8ic/+XEsQtpaeSg4sebyYEHwPNPKSwp6wV1Sia4KMd1/J62apHUxTqFG/Czsd7odsXp y0o6UC/MX8ZjBXeVikPZPTPfjF78Hy372DOmsVUVUgsCNnTDgSnrB2mHJ96HKO4IUf3a G0iuuFszzKc8jq7FGw3NMuizZzhm2BTRUFTznEYglq9GNBPU2VHlzTva4m2Q+8hQbABO eMOg== X-Gm-Message-State: AFqh2kqNdHGqLNksbJs+fcywS+AhxQTDyptfqOwKyGuuQs10VqriIGi2 Krgq6xanDkHPJ33K3m8j9VOwjFS+YQOOvIWa X-Google-Smtp-Source: AMrXdXtGcsAK9/S0XqHNIl0MfXBrdsTfb0q2tXu+vfflsP6QI70R8w5lAVb5ZV7S8HG5nqfZaa/5oQ== X-Received: by 2002:a05:600c:4f48:b0:3d6:8570:1239 with SMTP id m8-20020a05600c4f4800b003d685701239mr6602685wmq.16.1671752850929; Thu, 22 Dec 2022 15:47:30 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:bc21:fec7:83cf:e0c3]) by smtp.gmail.com with ESMTPSA id bd25-20020a05600c1f1900b003cfd4cf0761sm7322189wmb.1.2022.12.22.15.47.30 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 22 Dec 2022 15:47:30 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 06/10] process: Improve async command execution with idle interaction Date: Thu, 22 Dec 2022 23:47:22 +0000 Message-Id: <20221222234726.579702-6-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> References: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 22 Dec 2022 23:47:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14234 When running a new async command, the previous one might not have finished executing in the idle handler. Add some code to use the event logic to wait for the idle loop to finish for 30s before saying the server is busy. This avoids build failures when a UI quickly starts the next async command before the code has finished shutdown/cleanup from the last one. Signed-off-by: Richard Purdie --- lib/bb/command.py | 8 ++++++-- lib/bb/server/process.py | 2 +- lib/bb/server/xmlrpcserver.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/bb/command.py b/lib/bb/command.py index 20a8b86653..ebcee0b7eb 100644 --- a/lib/bb/command.py +++ b/lib/bb/command.py @@ -60,7 +60,7 @@ class Command: # FIXME Add lock for this self.currentAsyncCommand = None - def runCommand(self, commandline, ro_only = False): + def runCommand(self, commandline, process_server, ro_only = False): command = commandline.pop(0) # Ensure cooker is ready for commands @@ -100,7 +100,11 @@ class Command: else: return result, None if self.currentAsyncCommand is not None: - return None, "Busy (%s in progress)" % self.currentAsyncCommand[0] + # Wait for the idle loop to have cleared (30s max) + process_server.is_idle.clear() + process_server.is_idle.wait(timeout=30) + if self.currentAsyncCommand is not None: + return None, "Busy (%s in progress)" % self.currentAsyncCommand[0] if command not in CommandsAsync.__dict__: return None, "No such command" self.currentAsyncCommand = (command, commandline) diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 895e39cd0f..912f5f4091 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -260,7 +260,7 @@ class ProcessServer(): continue try: serverlog("Running command %s" % command) - self.command_channel_reply.send(self.cooker.command.runCommand(command)) + self.command_channel_reply.send(self.cooker.command.runCommand(command, self)) serverlog("Command Completed (socket: %s)" % os.path.exists(self.sockname)) except Exception as e: stack = traceback.format_exc() diff --git a/lib/bb/server/xmlrpcserver.py b/lib/bb/server/xmlrpcserver.py index 01f55538ae..2e65dc34a9 100644 --- a/lib/bb/server/xmlrpcserver.py +++ b/lib/bb/server/xmlrpcserver.py @@ -118,7 +118,7 @@ class BitBakeXMLRPCServerCommands(): """ Run a cooker command on the server """ - return self.server.cooker.command.runCommand(command, self.server.readonly) + return self.server.cooker.command.runCommand(command, self.server, self.server.readonly) def getEventHandle(self): return self.event_handle From patchwork Thu Dec 22 23:47:23 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17136 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 7C315C4332F for ; Thu, 22 Dec 2022 23:47:34 +0000 (UTC) Received: from mail-wm1-f49.google.com (mail-wm1-f49.google.com [209.85.128.49]) by mx.groups.io with SMTP id smtpd.web10.60847.1671752853128537071 for ; Thu, 22 Dec 2022 15:47:33 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=I2A5J2aM; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.49, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f49.google.com with SMTP id k22-20020a05600c1c9600b003d1ee3a6289so2461409wms.2 for ; Thu, 22 Dec 2022 15:47:32 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=Nvlhv/xI3RCqmomtv1v37EcS/G6BUEAe5DWKoRCliPo=; b=I2A5J2aMncT1Bke+X6J1nnYK2idBqXxQ3g6rdiWeFG9OURnisEz2sk/etc2bWFU1cW zKi8QUq2Oes/Y/RnStxKv5fZ7PzdS8r8mHiA9YPZnw3HzSKuCjIQQqddtBBXBN99MQdl nYR1l7EWw+t0V5UvcUQJbSyHeJdQ9SxhL5djU= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=Nvlhv/xI3RCqmomtv1v37EcS/G6BUEAe5DWKoRCliPo=; b=LP5RiKstbYcbH8OPKwlTzWh334Dcwt/4bquj/+Qzx7rkY0g9nAJV7I9ROD/mEBDFzB MFkao1/Mg2GRDeVbZ+/9xRiKjn+3UBCGijd+Afw4HD9vqyAVGVTChpUDkA5QszSX0O6m GaH9OiVodOw6hbq8aQKw1pGJsxm9BnsesUqJlaCN4Eqmbq9WTk0RH9Ow+K0osOwHBggB Stkz8ObYFZB49LFcznYRlo5t6cPfcCPhcO0XSA4yEETOTy18IVatq+GhOLZNB+gWOHA1 YbiLMnSfuT4PSulyJhyirKCujt2LAvsXg3o0oCIPpT2J3FM6L+T3GW2TpjTHCCu+cAhS sQFQ== X-Gm-Message-State: AFqh2kpAHQTV0gLEDfTUyP+WDBcqiDtm3BuRV0t/t+xIKlQLqCWqJtAk UQtK5E0oXwRBJFKpe+LnM5eTZMyuWj2Voq6Y X-Google-Smtp-Source: AMrXdXtt4on4bpDN/Z24dL4RdnRsw3SBtdoccxlI5ihRP2TWCE4ByVSoePvsjyH03a8/jEsFdEvXMg== X-Received: by 2002:a05:600c:4d24:b0:3c6:e63e:23d4 with SMTP id u36-20020a05600c4d2400b003c6e63e23d4mr5736608wmp.3.1671752851484; Thu, 22 Dec 2022 15:47:31 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:bc21:fec7:83cf:e0c3]) by smtp.gmail.com with ESMTPSA id bd25-20020a05600c1f1900b003cfd4cf0761sm7322189wmb.1.2022.12.22.15.47.31 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 22 Dec 2022 15:47:31 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 07/10] knotty: Avoid looping with tracebacks Date: Thu, 22 Dec 2022 23:47:23 +0000 Message-Id: <20221222234726.579702-7-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> References: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 22 Dec 2022 23:47:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14236 If there are events queued and there is an exception in the main loop of the UI code, it will print tracebacks on the console indefinitely. Avoid that by improving the loop exit conditions. Signed-off-by: Richard Purdie --- lib/bb/ui/knotty.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/bb/ui/knotty.py b/lib/bb/ui/knotty.py index ab1a367be0..c398f591e8 100644 --- a/lib/bb/ui/knotty.py +++ b/lib/bb/ui/knotty.py @@ -631,7 +631,7 @@ def main(server, eventHandler, params, tf = TerminalFilter): termfilter = tf(main, helper, console_handlers, params.options.quiet) atexit.register(termfilter.finish) - while True: + while main.shutdown < 2: try: if (lastprint + printinterval) <= time.time(): termfilter.keepAlive(printinterval) @@ -646,8 +646,6 @@ def main(server, eventHandler, params, tf = TerminalFilter): return_value = 3 main.shutdown = 2 lastevent = time.time() - if main.shutdown > 1: - break if not parseprogress: termfilter.updateFooter() event = eventHandler.waitEvent(0.25) From patchwork Thu Dec 22 23:47:24 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17137 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 7FE24C41535 for ; Thu, 22 Dec 2022 23:47:34 +0000 (UTC) Received: from mail-wm1-f47.google.com (mail-wm1-f47.google.com [209.85.128.47]) by mx.groups.io with SMTP id smtpd.web10.60848.1671752853905001340 for ; Thu, 22 Dec 2022 15:47:34 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=G5lGyjdn; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.47, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f47.google.com with SMTP id p13-20020a05600c468d00b003cf8859ed1bso2467299wmo.1 for ; Thu, 22 Dec 2022 15:47:33 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=/orBroC7VIG9kF20JsQoOhhU+3bXKNq5CAnYKFAIglQ=; b=G5lGyjdn77XhXek+r6LwTJYxzUyy5YNubSO6Ye5XHjelS5ixYR0XCkGxtMkiYyzmC+ xdmsyxYSoxSjcFAD9zHkeCT9fHQOEv24iRVaKIuGDi67pe5akJ/gN3TTmPkyPgVr+jj+ h4PvCNeE76hKhyS5ufH1Os6hzJzrjKwRdFlpI= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=/orBroC7VIG9kF20JsQoOhhU+3bXKNq5CAnYKFAIglQ=; b=ngoHDdZ/W3iFJqn2sBZrWrSZskXRVrv5j/YNIlJ7qZZmzH1v91qj1V5KYPB4A5wk0+ c4dT8b2Amj0xbeLIY2NTiMOA9eO9l3pSbEjz1Ufx6gYw9Z4pyP6tyHNaL+mAgKcimwQ9 f10BYBIP/hKbhxJP6TULYZ+S3QgeEyXPgQhOjIc5Ruv4WWAoGGIdDzLRF6MAiNfxtsLf aRbH6NwIpwY41T6rZwE2CC+kzmvn/J/Myt0T+Z3l6IiMKNE06UFbukPfaXdek4OSAc2U 6TrgmFfeJBKWKl74i5i0zQuk4rJm7fSPJucXD3mq26AKb2p9vcfWCKbhwXAX5yvky+65 JhYQ== X-Gm-Message-State: AFqh2kpD3hIsF7nPYzAZ+N88FaDGZrPu+oeLCPU0+wqXQ7kKTWNfQppj +gMIAbV//yAKptmMjJBZJPMI3rv8Urjpwn0O X-Google-Smtp-Source: AMrXdXu4ue8qw6vNJIHAm2czw9Hx3o4/Bb3yFXDjW5QDI4GNNftfGC3Ak0LhbgkUfuKasIHdlwEMag== X-Received: by 2002:a05:600c:4e08:b0:3cf:a41d:844b with SMTP id b8-20020a05600c4e0800b003cfa41d844bmr5589731wmq.5.1671752852075; Thu, 22 Dec 2022 15:47:32 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:bc21:fec7:83cf:e0c3]) by smtp.gmail.com with ESMTPSA id bd25-20020a05600c1f1900b003cfd4cf0761sm7322189wmb.1.2022.12.22.15.47.31 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 22 Dec 2022 15:47:31 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 08/10] event: Always use threadlock Date: Thu, 22 Dec 2022 23:47:24 +0000 Message-Id: <20221222234726.579702-8-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> References: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 22 Dec 2022 23:47:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14237 With the move to a server idle thread, we always need threading. The existing accessor functions could end up turning this off! I was going to hold the lock whilst changing it, check if the value was already set, cache the result and also fix the event code to always release the lock with a try/finally. Instead, disable the existing functions and use a with: block to handle the lock, keeping things much simpler. Signed-off-by: Richard Purdie --- lib/bb/event.py | 73 +++++++++++++++++++--------------------- lib/bb/server/process.py | 1 - lib/bb/tests/event.py | 17 +--------- 3 files changed, 35 insertions(+), 56 deletions(-) diff --git a/lib/bb/event.py b/lib/bb/event.py index db90724444..7826541a64 100644 --- a/lib/bb/event.py +++ b/lib/bb/event.py @@ -68,16 +68,15 @@ _catchall_handlers = {} _eventfilter = None _uiready = False _thread_lock = threading.Lock() -_thread_lock_enabled = False _heartbeat_enabled = False def enable_threadlock(): - global _thread_lock_enabled - _thread_lock_enabled = True + # Always needed now + return def disable_threadlock(): - global _thread_lock_enabled - _thread_lock_enabled = False + # Always needed now + return def enable_heartbeat(): global _heartbeat_enabled @@ -179,36 +178,30 @@ def print_ui_queue(): def fire_ui_handlers(event, d): global _thread_lock - global _thread_lock_enabled if not _uiready: # No UI handlers registered yet, queue up the messages ui_queue.append(event) return - if _thread_lock_enabled: - _thread_lock.acquire() - - errors = [] - for h in _ui_handlers: - #print "Sending event %s" % event - try: - if not _ui_logfilters[h].filter(event): - continue - # We use pickle here since it better handles object instances - # which xmlrpc's marshaller does not. Events *must* be serializable - # by pickle. - if hasattr(_ui_handlers[h].event, "sendpickle"): - _ui_handlers[h].event.sendpickle((pickle.dumps(event))) - else: - _ui_handlers[h].event.send(event) - except: - errors.append(h) - for h in errors: - del _ui_handlers[h] - - if _thread_lock_enabled: - _thread_lock.release() + with _thread_lock: + errors = [] + for h in _ui_handlers: + #print "Sending event %s" % event + try: + if not _ui_logfilters[h].filter(event): + continue + # We use pickle here since it better handles object instances + # which xmlrpc's marshaller does not. Events *must* be serializable + # by pickle. + if hasattr(_ui_handlers[h].event, "sendpickle"): + _ui_handlers[h].event.sendpickle((pickle.dumps(event))) + else: + _ui_handlers[h].event.send(event) + except: + errors.append(h) + for h in errors: + del _ui_handlers[h] def fire(event, d): """Fire off an Event""" @@ -322,21 +315,23 @@ def set_eventfilter(func): _eventfilter = func def register_UIHhandler(handler, mainui=False): - bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1 - _ui_handlers[_ui_handler_seq] = handler - level, debug_domains = bb.msg.constructLogOptions() - _ui_logfilters[_ui_handler_seq] = UIEventFilter(level, debug_domains) - if mainui: - global _uiready - _uiready = _ui_handler_seq - return _ui_handler_seq + with _thread_lock: + bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1 + _ui_handlers[_ui_handler_seq] = handler + level, debug_domains = bb.msg.constructLogOptions() + _ui_logfilters[_ui_handler_seq] = UIEventFilter(level, debug_domains) + if mainui: + global _uiready + _uiready = _ui_handler_seq + return _ui_handler_seq def unregister_UIHhandler(handlerNum, mainui=False): if mainui: global _uiready _uiready = False - if handlerNum in _ui_handlers: - del _ui_handlers[handlerNum] + with _thread_lock: + if handlerNum in _ui_handlers: + del _ui_handlers[handlerNum] return def get_uihandler(): diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 912f5f4091..63d37ae0d5 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -150,7 +150,6 @@ class ProcessServer(): self.cooker.pre_serve() bb.utils.set_process_name("Cooker") - bb.event.enable_threadlock() ready = [] newconnections = [] diff --git a/lib/bb/tests/event.py b/lib/bb/tests/event.py index 4de4cced5e..d959f2d95d 100644 --- a/lib/bb/tests/event.py +++ b/lib/bb/tests/event.py @@ -451,10 +451,9 @@ class EventHandlingTest(unittest.TestCase): and disable threadlocks tests """ bb.event.fire(bb.event.OperationStarted(), None) - def test_enable_threadlock(self): + def test_event_threadlock(self): """ Test enable_threadlock method """ self._set_threadlock_test_mockups() - bb.event.enable_threadlock() self._set_and_run_threadlock_test_workers() # Calls to UI handlers should be in order as all the registered # handlers for the event coming from the first worker should be @@ -462,20 +461,6 @@ class EventHandlingTest(unittest.TestCase): self.assertEqual(self._threadlock_test_calls, ["w1_ui1", "w1_ui2", "w2_ui1", "w2_ui2"]) - - def test_disable_threadlock(self): - """ Test disable_threadlock method """ - self._set_threadlock_test_mockups() - bb.event.disable_threadlock() - self._set_and_run_threadlock_test_workers() - # Calls to UI handlers should be intertwined together. Thanks to the - # delay in the registered handlers for the event coming from the first - # worker, the event coming from the second worker starts being - # processed before finishing handling the first worker event. - self.assertEqual(self._threadlock_test_calls, - ["w1_ui1", "w2_ui1", "w1_ui2", "w2_ui2"]) - - class EventClassesTest(unittest.TestCase): """ Event classes test class """ From patchwork Thu Dec 22 23:47:25 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17144 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 9005EC4167B for ; Thu, 22 Dec 2022 23:47:44 +0000 (UTC) Received: from mail-wm1-f52.google.com (mail-wm1-f52.google.com [209.85.128.52]) by mx.groups.io with SMTP id smtpd.web10.60849.1671752854374239049 for ; Thu, 22 Dec 2022 15:47:34 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=RYwF1pwm; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.52, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f52.google.com with SMTP id bi26-20020a05600c3d9a00b003d3404a89faso4559612wmb.1 for ; Thu, 22 Dec 2022 15:47:34 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=Yel8xUuSTh7qI0L/QoYQd90wgnTdh60jHGEUPsPnbNA=; b=RYwF1pwmvbl5lftaxKq2aJe8dyb3VUG0THeQDmmkofahCSMNQCHrD82ndnAReFVGNo s8vQ1L98OS7DopWDSMj5PUdwlMp2S7JASB9zpJpWNvXmXE/L6LF0zN6BKWo07PmpZVfE LOF0US7vL8/d34hpSrrLJG5NtrOoAY24b+o6k= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=Yel8xUuSTh7qI0L/QoYQd90wgnTdh60jHGEUPsPnbNA=; b=g7wE/YWMNfoMrz7W/oUJ1WJxspdkjXDpDpbDqvbLqJyF/WLWfXJ10Uq/mHAfVhN/kt OU4+VXRsJnLe6fDDBsOxSGCe6MdFzKhli3qS7c2gSU7B4dlDkk8St+/R0wymvhhP+MNv 5CTVdObqozbTheL9R5lETwgQumzYimZSCIusUPQ+dftcMjvg46I4eeeyYTcZLRQPhWbV Tcdy/Ke8+KQHh8suazyMwuWnOHaWCUjPMcccWsO5Ii3Ntp+31UxFioFTXgtPn7zzuraB 1J5av3pfAHEcdJzoGg2GxkbV3/19HYLjE8wt8i1EC0WozMkBFVsrAsUVIXuLg4mMPWES nNiw== X-Gm-Message-State: AFqh2kqGTaPUDbP62sVAdk1jxBS+VkQoiQmJROmlZBadT5twVT3F99cm 6LWZrWayduviGGYWJfsmNZya+C0dxgEtOKLh X-Google-Smtp-Source: AMrXdXvhLSp0tIY8JhqNg6TF5PHukbxg/MtB7gH71x7MZoN6z1fJbxM4kOUsv4wZtKxIfd5nv6JxVg== X-Received: by 2002:a05:600c:4193:b0:3c6:e60f:3f4f with SMTP id p19-20020a05600c419300b003c6e60f3f4fmr5570810wmh.6.1671752852669; Thu, 22 Dec 2022 15:47:32 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:bc21:fec7:83cf:e0c3]) by smtp.gmail.com with ESMTPSA id bd25-20020a05600c1f1900b003cfd4cf0761sm7322189wmb.1.2022.12.22.15.47.32 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 22 Dec 2022 15:47:32 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 09/10] server/process: Improve exception logging Date: Thu, 22 Dec 2022 23:47:25 +0000 Message-Id: <20221222234726.579702-9-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> References: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 22 Dec 2022 23:47:44 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14238 Currently if either idle functions loop suffers a traceback, it is silently dropped and there is no log message to say what happened. This change at least means the traceback is in the cooker log, making some debugging possible. Signed-off-by: Richard Purdie --- lib/bb/server/process.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 63d37ae0d5..49cca8d23d 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -389,6 +389,7 @@ class ProcessServer(): if not isinstance(exc, bb.BBHandledException): logger.exception('Running idle function') del self._idlefuns[function] + serverlog("Exception %s broke the idle_thread, exiting" % traceback.format_exc()) self.quit = True # FIXME - the 1 is the inotify processing in cooker which always runs @@ -422,6 +423,7 @@ class ProcessServer(): except Exception as exc: if not isinstance(exc, bb.BBHandledException): logger.exception('Running heartbeat function') + serverlog("Exception %s broke in idle_commands, exiting" % traceback.format_exc()) self.quit = True if nextsleep and bb.event._heartbeat_enabled and now + nextsleep > self.next_heartbeat: # Shorten timeout so that we we wake up in time for From patchwork Thu Dec 22 23:47:26 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17145 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 90036C4332F for ; Thu, 22 Dec 2022 23:47:44 +0000 (UTC) Received: from mail-wm1-f46.google.com (mail-wm1-f46.google.com [209.85.128.46]) by mx.groups.io with SMTP id smtpd.web10.60850.1671752855019544396 for ; Thu, 22 Dec 2022 15:47:35 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=avxPH7fa; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.46, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f46.google.com with SMTP id r126-20020a1c4484000000b003d6b8e8e07fso4855179wma.0 for ; Thu, 22 Dec 2022 15:47:34 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=+0ONtpbjCTH0su7cgVw1Ip0Nhv9RztuUYIq5L/Jh9fE=; b=avxPH7far0r9itEOmK0xZW4HMuC2dthETqb91RcVTuRYWXtJcTaTlPilobyGymCE34 WOB3Mp7g2L2uYFys4c1W+BphT2Uum7eILdkUOkQix9F5BlqyXz8dJEmV0GNlU1zX5Igy ONmxx1aYDzAuB3NczZdmNYszyNGwrQVi0kW3o= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=+0ONtpbjCTH0su7cgVw1Ip0Nhv9RztuUYIq5L/Jh9fE=; b=wPFsuAgwYOb3mvEDNTIAB3nlyDDBQppv5FSa8IAh9setQ6MUAdOUXMB8Oow0B+FPnq YotVxD78znvLWlcq0bnu6dqMfJxpPckK8ueoK0/YQhZPb8o4DflmwaED8WWk1hHbjJyM hQRlG5W5zhiMxxxkfzQaGWVuzl09KvUrL2Kn+HHxzTedHkVbet0SmtWt3e1NsHXFIEDd 6LHVNASctneCJnEqfXENbWfpGjgcXPdJWi2wkNDIN4PX7USXqxe8LD7+tI5u73eiuSV/ /Z1jm8SwKjCgYeYWTCvdHSgS8riOxsOSuJHY31WeHVvLCp9Eu6pmpm4Y1jQOY67ea18f JWPQ== X-Gm-Message-State: AFqh2koQ60SCZWcJq4hk7JTMhx/gPz+H0trGM/4w6QkNcHLjalJEnJEN eJKPn7UYV6qBXDxTp09LHd2gAOBvCdeD+ZdC X-Google-Smtp-Source: AMrXdXskkqPbzwGgKXkWiZGgPn08qq1Y2ce7oyO/cmOdyETTHzJASTse4Npp+h86I6/q3XtIqw3qEQ== X-Received: by 2002:a05:600c:3592:b0:3d1:ebdf:d58b with SMTP id p18-20020a05600c359200b003d1ebdfd58bmr5417155wmq.5.1671752853354; Thu, 22 Dec 2022 15:47:33 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:bc21:fec7:83cf:e0c3]) by smtp.gmail.com with ESMTPSA id bd25-20020a05600c1f1900b003cfd4cf0761sm7322189wmb.1.2022.12.22.15.47.32 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 22 Dec 2022 15:47:32 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 10/10] cooker/cookerdata: Rework the way the datastores are reset Date: Thu, 22 Dec 2022 23:47:26 +0000 Message-Id: <20221222234726.579702-10-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> References: <20221222234726.579702-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 22 Dec 2022 23:47:44 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14239 As far as I could tell, the current code could result in some strange situations where some data was set back to the original data store copy but the multiconfig data was not restored. There are also some changes made to the datastore which did not persist. The data store was also being reset at every client reset, which seems a little excessive if we can reset it to the original condition properly. Move the __depends -> __base_depends rename into databuilder along with the __bbclasstype change so these are saved in the original data. Tweak the databuilder code to be clearer and easier to follow on which copies are where, then save copies of all the mc datastores. Finally, drop the cache invalidation upon reset for the base config as we shouldn't need that now (oe-selftest -r tinfoil works with memory resident bitbake which was the original reproducer requiring that change). Signed-off-by: Richard Purdie --- lib/bb/cooker.py | 9 +++------ lib/bb/cookerdata.py | 31 +++++++++++++++++++------------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index 1daa587ac5..43e0842284 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -409,9 +409,7 @@ class BBCooker: self.disableDataTracking() for mc in self.databuilder.mcdata.values(): - mc.renameVar("__depends", "__base_depends") self.add_filewatch(mc.getVar("__base_depends", False), self.configwatcher) - mc.setVar("__bbclasstype", "recipe") self.baseconfig_valid = True self.parsecache_valid = False @@ -445,10 +443,8 @@ class BBCooker: upstream=upstream, ) self.hashserv.serve_as_process() - self.data.setVar("BB_HASHSERVE", self.hashservaddr) - self.databuilder.origdata.setVar("BB_HASHSERVE", self.hashservaddr) - self.databuilder.data.setVar("BB_HASHSERVE", self.hashservaddr) for mc in self.databuilder.mcdata: + self.databuilder.mcorigdata[mc].setVar("BB_HASHSERVE", self.hashservaddr) self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr) bb.parse.init_parser(self.data) @@ -1797,8 +1793,9 @@ class BBCooker: if hasattr(self, "data"): self.databuilder.reset() self.data = self.databuilder.data + # In theory tinfoil could have modified the base data before parsing, + # ideally need to track if anything did modify the datastore self.parsecache_valid = False - self.baseconfig_valid = False class CookerExit(bb.event.Event): diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py index 650ae05ec9..b4e0c4216b 100644 --- a/lib/bb/cookerdata.py +++ b/lib/bb/cookerdata.py @@ -263,6 +263,7 @@ class CookerDataBuilder(object): self.mcdata = {} def parseBaseConfiguration(self, worker=False): + mcdata = {} data_hash = hashlib.sha256() try: self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles) @@ -288,18 +289,18 @@ class CookerDataBuilder(object): bb.parse.init_parser(self.data) data_hash.update(self.data.get_hash().encode('utf-8')) - self.mcdata[''] = self.data + mcdata[''] = self.data multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split() for config in multiconfig: if config[0].isdigit(): bb.fatal("Multiconfig name '%s' is invalid as multiconfigs cannot start with a digit" % config) - mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config) - bb.event.fire(bb.event.ConfigParsed(), mcdata) - self.mcdata[config] = mcdata - data_hash.update(mcdata.get_hash().encode('utf-8')) + parsed_mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config) + bb.event.fire(bb.event.ConfigParsed(), parsed_mcdata) + mcdata[config] = parsed_mcdata + data_hash.update(parsed_mcdata.get_hash().encode('utf-8')) if multiconfig: - bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data) + bb.event.fire(bb.event.MultiConfigParsed(mcdata), self.data) self.data_hash = data_hash.hexdigest() except (SyntaxError, bb.BBHandledException): @@ -332,17 +333,23 @@ class CookerDataBuilder(object): if issues: raise bb.BBHandledException() + for mc in mcdata: + mcdata[mc].renameVar("__depends", "__base_depends") + mcdata[mc].setVar("__bbclasstype", "recipe") + # Create a copy so we can reset at a later date when UIs disconnect - self.origdata = self.data - self.data = bb.data.createCopy(self.origdata) - self.mcdata[''] = self.data + self.mcorigdata = mcdata + for mc in mcdata: + self.mcdata[mc] = bb.data.createCopy(mcdata[mc]) + self.data = self.mcdata[''] def reset(self): # We may not have run parseBaseConfiguration() yet - if not hasattr(self, 'origdata'): + if not hasattr(self, 'mcorigdata'): return - self.data = bb.data.createCopy(self.origdata) - self.mcdata[''] = self.data + for mc in self.mcorigdata: + self.mcdata[mc] = bb.data.createCopy(self.mcorigdata[mc]) + self.data = self.mcdata[''] def _findLayerConf(self, data): return findConfigFile("bblayers.conf", data)