From patchwork Mon Aug 12 15:53:32 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 47696 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 095ACC52D7C for ; Mon, 12 Aug 2024 15:53:46 +0000 (UTC) Received: from mail-lj1-f175.google.com (mail-lj1-f175.google.com [209.85.208.175]) by mx.groups.io with SMTP id smtpd.web11.49925.1723478019690460166 for ; Mon, 12 Aug 2024 08:53:40 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=XctzoNCd; spf=pass (domain: linuxfoundation.org, ip: 209.85.208.175, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-lj1-f175.google.com with SMTP id 38308e7fff4ca-2ef32fea28dso49653801fa.2 for ; Mon, 12 Aug 2024 08:53:39 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1723478017; x=1724082817; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:message-id:date:subject:to :from:from:to:cc:subject:date:message-id:reply-to; bh=D2vrfvZUlWDOvg+x6JdndqFmbSLr3uPkkPHr9QAnR+g=; b=XctzoNCd40nR4CfgIJMth9Mk9ktxaKiZkHzDwNeUmt3+lLjhWAJQEOWKAKTLWQJDzl k86AhJLemJMPyJs9VO2IpikOlmonhlikec+B1vALVAIWZgKbSSSo/UDNPngItIKeiugq /WHJ+EELDkMrNcJrMJzbc+2MAzNUpltTmxQvI= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1723478017; x=1724082817; h=content-transfer-encoding:mime-version:message-id:date:subject:to :from:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=D2vrfvZUlWDOvg+x6JdndqFmbSLr3uPkkPHr9QAnR+g=; b=jGHnaO7QvSbo0R/Nc7VWtqe6bDK78Cjft0z3qkrYe860N5JymgJScPd8gQea/GR2sR v00UII/BYCeXrYWs017/s6AM7We92k2Mz8833xyxLrZbZtmmQheWAVi2/1E1MLjdkh0j Jqx7E4bdZEBN8p+pDHlRiH6NZlaVgL3XzHDcmCCzJ1vsmZysvafrtIXURuZ2LkFGmg6x WCxX7Zvr2cgduSETZvduNgf8Vb5VC8SZf06CT88Zxu8TOLw7PhbFCL7zA6B6b/3Hktyw uh3a0NStB8+HIr/r4ppsOYPb82ZcYu+rozqlWNKnZD005YNlQ+KuKLZzTqjJgWkvwUDG 1tjw== X-Gm-Message-State: AOJu0YyeGKPGA65SzQ+ZNhp9c6iGyybmlnSouPmypmr98VNVvEDuqCby T9OCKrPUM15lMQtFpwuUOAT63PgL/m5BDaXErj5hK6uPMQKG4rnsCaFkMz9OUkpCICFjHRNCXQN Ar3U= X-Google-Smtp-Source: AGHT+IFq/Pf8IqmLe2cl0WqQDh4pI5BZVWaa2N7ITLIRc/uTPGdXDkDsVIMXa2v6hasw/ytjmptZQw== X-Received: by 2002:a2e:be27:0:b0:2ef:28d2:39cc with SMTP id 38308e7fff4ca-2f2b7146aaamr5493951fa.3.1723478017106; Mon, 12 Aug 2024 08:53:37 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:450:fb95:e7af:f3ab]) by smtp.gmail.com with ESMTPSA id 5b1f17b1804b1-429c77372e8sm109970015e9.33.2024.08.12.08.53.36 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 12 Aug 2024 08:53:36 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 1/4] cache: Drop unused function Date: Mon, 12 Aug 2024 16:53:32 +0100 Message-ID: <20240812155335.904273-1-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.43.0 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Mon, 12 Aug 2024 15:53:46 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16471 I can't spot any users of this function and it is poking at variables inside cooker that could and are about to change so drop it. Signed-off-by: Richard Purdie --- lib/bb/cache.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/lib/bb/cache.py b/lib/bb/cache.py index 4a96f5b313..c48feb7138 100644 --- a/lib/bb/cache.py +++ b/lib/bb/cache.py @@ -779,25 +779,6 @@ class MulticonfigCache(Mapping): for k in self.__caches: yield k -def init(cooker): - """ - The Objective: Cache the minimum amount of data possible yet get to the - stage of building packages (i.e. tryBuild) without reparsing any .bb files. - - To do this, we intercept getVar calls and only cache the variables we see - being accessed. We rely on the cache getVar calls being made for all - variables bitbake might need to use to reach this stage. For each cached - file we need to track: - - * Its mtime - * The mtimes of all its dependencies - * Whether it caused a parse.SkipRecipe exception - - Files causing parsing errors are evicted from the cache. - - """ - return Cache(cooker.configuration.data, cooker.configuration.data_hash) - class CacheData(object): """ From patchwork Mon Aug 12 15:53:33 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 47697 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 1FCCDC531DC for ; Mon, 12 Aug 2024 15:53:46 +0000 (UTC) Received: from mail-wm1-f50.google.com (mail-wm1-f50.google.com [209.85.128.50]) by mx.groups.io with SMTP id smtpd.web11.49926.1723478019945162949 for ; Mon, 12 Aug 2024 08:53:40 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=OXh8UhF0; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.50, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f50.google.com with SMTP id 5b1f17b1804b1-428119da952so32072025e9.0 for ; Mon, 12 Aug 2024 08:53:39 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1723478018; x=1724082818; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=ackeiZSarVZpnT4ejsk4OL8Bd37Bwu7ZTqEYSRKx81k=; b=OXh8UhF0miCYLKiWGvqts5Yro+UZDkXnFjuXp+wBhoWmiuw/w4MHe8VJc+un+IBjj9 8T5lMlVEUbThQjMNsGkwmnXZsIUZOhfmCNvDsWxJIXvEvwZ4KQ7S1cTMehavkSaoBlXJ CH+ruUn5NkR51rluYkfqXgTyTBjXWdQeO0iqk= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1723478018; x=1724082818; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=ackeiZSarVZpnT4ejsk4OL8Bd37Bwu7ZTqEYSRKx81k=; b=S7ZiqB3JrYLbdEbwR7vtb7xSCkdDuGhyiTBsOFDItiGKC0CGmgAvbmDXsesCjsZKWJ gDbeV7Bs0Ha7ZZoDHzuST0GuxWrn4PipzyqXg6RDIE+5l7YGAGo3Lj8scy6fYR3lQi4I WaNB5ZGbbI3hi0biVni6iTnmEGtCmDWD/tfDGutfVk/SNNj5Omp2iSCq+8SqnJqR0Q0i xjJaS9ALBF7LdZy+s55DQEbsPJIYdM/5aBa7L1F/0dc6iVEC1ZlqvC0TQOgzNu95D/qW Zx7X9p7fss/bSMnFvwWnIJVglRJjl0dkCGs8n382dzA3YQrC3CwB1iYFa8yO+L8gouEr uvdA== X-Gm-Message-State: AOJu0Yy2a8Go6n5sVOt1eBZReoeqrH/BoVxM7koBJpZIUSbWLH0dfwNy H4NqLfQEvyCTUr7AiuzOG0tlr2Zuh+l1y8OQ8GoMCFHjebupixrr8z6WyAUige7JJXJciVNvQpH /1KU= X-Google-Smtp-Source: AGHT+IFZfPGUQzr38EQPn2b7mPXjCreJmP//93CMk0xlvNCUJJAzDyga1uiXO8wqEERmPWqMBsSmjQ== X-Received: by 2002:a05:600c:3b88:b0:427:faa6:5648 with SMTP id 5b1f17b1804b1-429d48188c9mr6617475e9.3.1723478017764; Mon, 12 Aug 2024 08:53:37 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:450:fb95:e7af:f3ab]) by smtp.gmail.com with ESMTPSA id 5b1f17b1804b1-429c77372e8sm109970015e9.33.2024.08.12.08.53.37 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 12 Aug 2024 08:53:37 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 2/4] cookerdata: Separate out data_hash and hook to tinfoil Date: Mon, 12 Aug 2024 16:53:33 +0100 Message-ID: <20240812155335.904273-2-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20240812155335.904273-1-richard.purdie@linuxfoundation.org> References: <20240812155335.904273-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Mon, 12 Aug 2024 15:53:46 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16472 Within tinfoil, the user can write to the configuration data but it won't cause the data_hash checksum to be re-written, meaning cached parsing data can be reused when it would now be incorrect. Abstract out the data_hash code and add it to the invalidateCaches command, called by tinfoil.modified_files() meaning that tinfoil can instruct bitbake to update the caches and re-parse if necessary. Also move the data_hash entirely into databuilder and drop the copy in cooker as obsolete and not needed. Signed-off-by: Richard Purdie --- lib/bb/cooker.py | 10 +++++++--- lib/bb/cookerdata.py | 13 +++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index 6754f986bf..31fee2bead 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -281,7 +281,6 @@ class BBCooker: self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False) self.databuilder.parseBaseConfiguration() self.data = self.databuilder.data - self.data_hash = self.databuilder.data_hash self.extraconfigdata = {} eventlog = self.data.getVar("BB_DEFAULT_EVENTLOG") @@ -370,6 +369,11 @@ class BBCooker: if not clean: bb.parse.BBHandler.cached_statements = {} + # If writes were made to any of the data stores, we need to recalcuate the data + # store cache + if hasattr(self, "databuilder"): + self.databuilder.calc_datastore_hashes() + def parseConfiguration(self): self.updateCacheSync() @@ -1338,7 +1342,7 @@ class BBCooker: self.buildSetVars() self.reset_mtime_caches() - bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array) + bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.databuilder.data_hash, self.caches_array) layername = self.collections[mc].calc_bbfile_priority(fn)[2] infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn), layername) @@ -2112,7 +2116,7 @@ class CookerParser(object): self.mcfilelist = mcfilelist self.cooker = cooker self.cfgdata = cooker.data - self.cfghash = cooker.data_hash + self.cfghash = cooker.databuilder.data_hash self.cfgbuilder = cooker.databuilder # Accounting statistics diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py index 0649e40995..3ad5cf3dd0 100644 --- a/lib/bb/cookerdata.py +++ b/lib/bb/cookerdata.py @@ -254,9 +254,16 @@ class CookerDataBuilder(object): self.data = self.basedata self.mcdata = {} + def calc_datastore_hashes(self): + data_hash = hashlib.sha256() + data_hash.update(self.data.get_hash().encode('utf-8')) + multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split() + for config in multiconfig: + data_hash.update(self.mcdata[config].get_hash().encode('utf-8')) + self.data_hash = data_hash.hexdigest() + def parseBaseConfiguration(self, worker=False): mcdata = {} - data_hash = hashlib.sha256() try: self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles) @@ -279,7 +286,6 @@ class CookerDataBuilder(object): bb.event.fire(bb.event.ConfigParsed(), self.data) bb.parse.init_parser(self.data) - data_hash.update(self.data.get_hash().encode('utf-8')) mcdata[''] = self.data multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split() @@ -289,11 +295,9 @@ class CookerDataBuilder(object): parsed_mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config) bb.event.fire(bb.event.ConfigParsed(), parsed_mcdata) mcdata[config] = parsed_mcdata - data_hash.update(parsed_mcdata.get_hash().encode('utf-8')) if multiconfig: bb.event.fire(bb.event.MultiConfigParsed(mcdata), self.data) - self.data_hash = data_hash.hexdigest() except bb.data_smart.ExpansionError as e: logger.error(str(e)) raise bb.BBHandledException() @@ -328,6 +332,7 @@ class CookerDataBuilder(object): for mc in mcdata: self.mcdata[mc] = bb.data.createCopy(mcdata[mc]) self.data = self.mcdata[''] + self.calc_datastore_hashes() def reset(self): # We may not have run parseBaseConfiguration() yet From patchwork Mon Aug 12 15:53:34 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 47695 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0CB2FC3DA7F for ; Mon, 12 Aug 2024 15:53:46 +0000 (UTC) Received: from mail-wm1-f52.google.com (mail-wm1-f52.google.com [209.85.128.52]) by mx.groups.io with SMTP id smtpd.web11.49928.1723478020521829924 for ; Mon, 12 Aug 2024 08:53:40 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=NDPGQ8jo; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.52, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f52.google.com with SMTP id 5b1f17b1804b1-428085a3ad1so35548065e9.1 for ; Mon, 12 Aug 2024 08:53:40 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1723478019; x=1724082819; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=8l/r1seA2c9qLvCw6N7fgypnq3JoEIvNgbnvyNdrF/s=; b=NDPGQ8jovwJ2+FOrBOuZTRoC/O5bZZEbuEIQT+Ody1rRFT+XVHUMRmbaLScChV+fVz 2k3sc95d/GHRa21Ij3uJq7jiSgP3Q++GJD8lnaUfWSLGWoL/F/3IITAFOqWLRuDCk6tb PmzWDwsssjOLV1AEJQcEX0NDeJmsZgBOH9PsI= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1723478019; x=1724082819; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=8l/r1seA2c9qLvCw6N7fgypnq3JoEIvNgbnvyNdrF/s=; b=fMOCxCF19o7eO12nFj/2NWPTxTVSvfNyCdkD98hXzJxtYg6uy0sYJIOBdR4PkprH3q zHwA75QZ7EGWatcaF7kW3B78k7bf1KBLCnGxjoaHN/2gw3BVhM5Ty5JRsN05vWXHBLKS kjeveEHtfKccVKbu9bf1U8Lt0J8zuIzddXJOnGiBU1sx51QUK3Gml7JXiFLJzWur//RT 07L4t5efLVWKTaETCmjArnhHS7rlPdCrR2o+/2gWoTXkUSdszI2Rl7zWdJIaAEKpazHK bPZfaDp65JzogtjiJ9Bwys9JoRjyrE8ZDKGO3oE4xR4e4C7w8XY0YYVfBjdlsZxE/IBo 2a8A== X-Gm-Message-State: AOJu0YxuMl4qxWPjoQAFzpiqVGxuIJxf8N1hBtkAsn1/0sHrymFOz5H2 +61VvnqTGx7a+MX1pMA3/nzUOpz7DzgN1DKwt7I1Dh8v7BMXmUJTEadHL1yQJx1QaDKTpAvZm5J +XX4= X-Google-Smtp-Source: AGHT+IH+urJhMCuh/+lrQ9hfJm0aScNQ/kMK/hNW/lEuc26a8HvWY5m9NFKNASyYoKrLAfCln9OE/A== X-Received: by 2002:a05:600c:4584:b0:426:5416:67e0 with SMTP id 5b1f17b1804b1-429d48993e6mr6812815e9.31.1723478018373; Mon, 12 Aug 2024 08:53:38 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:450:fb95:e7af:f3ab]) by smtp.gmail.com with ESMTPSA id 5b1f17b1804b1-429c77372e8sm109970015e9.33.2024.08.12.08.53.37 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 12 Aug 2024 08:53:38 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 3/4] BBHandler/ast: Improve addtask handling Date: Mon, 12 Aug 2024 16:53:34 +0100 Message-ID: <20240812155335.904273-3-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20240812155335.904273-1-richard.purdie@linuxfoundation.org> References: <20240812155335.904273-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Mon, 12 Aug 2024 15:53:46 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16473 The recent addtask improvement to handle comments complicated the regex significantly and there are already a number of corner cases in that code which aren't handled well. Instead of trying to complicate the regex further, switch to code logic instead. This means the following cases are now handled: * addtask with multiple task names * addtask with multiple before constraints * addtask with multiple after constraints The testcase is updated to match the improvements. Signed-off-by: Richard Purdie --- lib/bb/parse/ast.py | 26 +++++++------------- lib/bb/parse/parse_py/BBHandler.py | 38 +++++++++++++++++------------- lib/bb/tests/parse.py | 15 ++++-------- 3 files changed, 35 insertions(+), 44 deletions(-) diff --git a/lib/bb/parse/ast.py b/lib/bb/parse/ast.py index 7581d003fd..001ba8d289 100644 --- a/lib/bb/parse/ast.py +++ b/lib/bb/parse/ast.py @@ -240,14 +240,16 @@ class ExportFuncsNode(AstNode): data.setVar(func, sentinel + " " + calledfunc + "\n", parsing=True) class AddTaskNode(AstNode): - def __init__(self, filename, lineno, func, before, after): + def __init__(self, filename, lineno, tasks, before, after): AstNode.__init__(self, filename, lineno) - self.func = func + self.tasks = tasks self.before = before self.after = after def eval(self, data): - bb.build.addtask(self.func, self.before, self.after, data) + tasks = self.tasks.split() + for task in tasks: + bb.build.addtask(task, self.before, self.after, data) class DelTaskNode(AstNode): def __init__(self, filename, lineno, tasks): @@ -348,21 +350,11 @@ def handlePythonMethod(statements, filename, lineno, funcname, modulename, body) def handleExportFuncs(statements, filename, lineno, m, classname): statements.append(ExportFuncsNode(filename, lineno, m.group(1), classname)) -def handleAddTask(statements, filename, lineno, m): - func = m.group("func") - before = m.group("before") - after = m.group("after") - if func is None: - return - - statements.append(AddTaskNode(filename, lineno, func, before, after)) - -def handleDelTask(statements, filename, lineno, m): - func = m.group(1) - if func is None: - return +def handleAddTask(statements, filename, lineno, tasks, before, after): + statements.append(AddTaskNode(filename, lineno, tasks, before, after)) - statements.append(DelTaskNode(filename, lineno, func)) +def handleDelTask(statements, filename, lineno, tasks): + statements.append(DelTaskNode(filename, lineno, tasks)) def handleBBHandlers(statements, filename, lineno, m): statements.append(BBHandlerNode(filename, lineno, m.group(1))) diff --git a/lib/bb/parse/parse_py/BBHandler.py b/lib/bb/parse/parse_py/BBHandler.py index c1653faeee..87c03e9238 100644 --- a/lib/bb/parse/parse_py/BBHandler.py +++ b/lib/bb/parse/parse_py/BBHandler.py @@ -23,7 +23,7 @@ __func_start_regexp__ = re.compile(r"(((?Ppython(?=(\s|\()))|(?Pfaker __inherit_regexp__ = re.compile(r"inherit\s+(.+)" ) __inherit_def_regexp__ = re.compile(r"inherit_defer\s+(.+)" ) __export_func_regexp__ = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" ) -__addtask_regexp__ = re.compile(r"addtask\s+(?P\w+)\s*((before\s*(?P(([^#\n]*(?=after))|([^#\n]*))))|(after\s*(?P(([^#\n]*(?=before))|([^#\n]*)))))*(?P#.*|.*?)") +__addtask_regexp__ = re.compile(r"addtask\s+([^#\n]+)(?P#.*|.*?)") __deltask_regexp__ = re.compile(r"deltask\s+([^#\n]+)(?P#.*|.*?)") __addhandler_regexp__ = re.compile(r"addhandler\s+(.+)" ) __def_regexp__ = re.compile(r"def\s+(\w+).*:" ) @@ -239,29 +239,35 @@ def feeder(lineno, s, fn, root, statements, eof=False): m = __addtask_regexp__.match(s) if m: - if len(m.group().split()) == 2: - # Check and warn for "addtask task1 task2" - m2 = re.match(r"addtask\s+(?P\w+)(?P.*)", s) - if m2 and m2.group('ignores'): - logger.warning('addtask ignored: "%s"' % m2.group('ignores')) - - # Check and warn for "addtask task1 before task2 before task3", the - # similar to "after" - taskexpression = s.split() - for word in ('before', 'after'): - if taskexpression.count(word) > 1: - logger.warning("addtask contained multiple '%s' keywords, only one is supported" % word) + after = "" + before = "" + tasks = m.group(1).split(" before ")[0].split(" after ")[0] + + for exp in m.group(1).split(" before "): + exp2 = exp.split(" after ") + if len(exp2) > 1: + after = after + " ".join(exp2[1:]) + + for exp in m.group(1).split(" after "): + exp2 = exp.split(" before ") + if len(exp2) > 1: + before = before + " ".join(exp2[1:]) - # Check and warn for having task with exprssion as part of task name + # Check and warn for having task with a keyword as part of task name + taskexpression = s.split() for te in taskexpression: if any( ( "%s_" % keyword ) in te for keyword in bb.data_smart.__setvar_keyword__ ): raise ParseError("Task name '%s' contains a keyword which is not recommended/supported.\nPlease rename the task not to include the keyword.\n%s" % (te, ("\n".join(map(str, bb.data_smart.__setvar_keyword__)))), fn) - ast.handleAddTask(statements, fn, lineno, m) + + if tasks is not None: + ast.handleAddTask(statements, fn, lineno, tasks, before, after) return m = __deltask_regexp__.match(s) if m: - ast.handleDelTask(statements, fn, lineno, m) + task = m.group(1) + if task is not None: + ast.handleDelTask(statements, fn, lineno, task) return m = __addhandler_regexp__.match(s) diff --git a/lib/bb/tests/parse.py b/lib/bb/tests/parse.py index d076fcc208..97df2c4590 100644 --- a/lib/bb/tests/parse.py +++ b/lib/bb/tests/parse.py @@ -177,7 +177,7 @@ python () { addtask_deltask = """ addtask do_patch after do_foo after do_unpack before do_configure before do_compile -addtask do_fetch do_patch +addtask do_fetch2 do_patch2 addtask do_myplaintask addtask do_myplaintask2 @@ -194,18 +194,11 @@ deltask do_fetch ${MYVAR} ${EMPTYVAR} deltask ${EMPTYVAR} """ def test_parse_addtask_deltask(self): - import sys - with self.assertLogs() as logs: - f = self.parsehelper(self.addtask_deltask) - d = bb.parse.handle(f.name, self.d)[''] + f = self.parsehelper(self.addtask_deltask) + d = bb.parse.handle(f.name, self.d)[''] - output = "".join(logs.output) - self.assertTrue("addtask contained multiple 'before' keywords" in output) - self.assertTrue("addtask contained multiple 'after' keywords" in output) - self.assertTrue('addtask ignored: " do_patch"' in output) - self.assertEqual(['do_myplaintask', 'do_mytask', 'do_mytask2'], d.getVar("__BBTASKS")) - #self.assertTrue('dependent task do_foo for do_patch does not exist' in output) + self.assertEqual(['do_fetch2', 'do_patch2', 'do_myplaintask', 'do_mytask', 'do_mytask2'], d.getVar("__BBTASKS")) broken_multiline_comment = """ # First line of comment \\ From patchwork Mon Aug 12 15:53:35 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 47694 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0B5EFC52D7D for ; Mon, 12 Aug 2024 15:53:46 +0000 (UTC) Received: from mail-wm1-f53.google.com (mail-wm1-f53.google.com [209.85.128.53]) by mx.groups.io with SMTP id smtpd.web10.49913.1723478021141851456 for ; Mon, 12 Aug 2024 08:53:41 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=VjbcUDKn; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.53, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f53.google.com with SMTP id 5b1f17b1804b1-4280ee5f1e3so34688815e9.0 for ; Mon, 12 Aug 2024 08:53:40 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; t=1723478019; x=1724082819; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=y+x2qSsVcwjRJDx93qCaA8G7lFTNpH0tDZHbg6IHE18=; b=VjbcUDKnjXe4rmd4Um6zZfkRHnhmv6SZiSJSGoq/M3rvFzf55Crub2z8p321EE/DZK dFD+F3x7+xOQe+tQEhD3M51CVwOh65CKDrVBt7daqYmAKk3bNFxKytpLn7fw5oRPzC3e AfZ33RqpKCqTfVyLDfWKpMIEKhdDL0kXYnBq0= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1723478019; x=1724082819; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=y+x2qSsVcwjRJDx93qCaA8G7lFTNpH0tDZHbg6IHE18=; b=pqIc/sIcs80J5r7U9pAev2tVCKcWM82ZdH7By1PO7bJ9EvNuzZg0F8noFKHfMAnAV+ VC/zepDvZEou9VFnpVEx98PiqwL2DH4Aovc1QpKWEswwlGu4cPfzVLdRaOrx8SulKWo6 EdtNRNKosySc12TWms6q3P9ZoSv5K+EFmLx6y4/4GR7jeUgoF6pi0qDW3NJZi1Ncy4Oi G8PXRYZtWSKeDkGYlfFFxwhfGedsXvn2RJQqxJOgNl8x+R07r9WIxqAXOTIusKEJHKpJ sMON9a2s3s/1MUq7bAd0qhyI1I6q0BLp/d2I1sJdyMaj280vboA28SVXBxq4YTTwtg/R CeOw== X-Gm-Message-State: AOJu0YymsH7f86792xVXKINnJ/tLRjzBIuV/AvfrZBk9fCyomKndNHey 7PCk/7YYW9bAb0jMNZPF3gwwTpAEtmjdJe1uJYOqJRQ8HV26/7O3y/CvWmWbLOSzDnx4FrI4JUN Ty/I= X-Google-Smtp-Source: AGHT+IFu7P68bOKIGUAp+Ge+YGW/JBhMKzOPbhe0hOJaTD+9DNdSAqeaYrzPBFRx2T788TExSxTAgg== X-Received: by 2002:a05:600c:548b:b0:427:ff3b:7a20 with SMTP id 5b1f17b1804b1-429d486fea7mr6809075e9.27.1723478019215; Mon, 12 Aug 2024 08:53:39 -0700 (PDT) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:450:fb95:e7af:f3ab]) by smtp.gmail.com with ESMTPSA id 5b1f17b1804b1-429c77372e8sm109970015e9.33.2024.08.12.08.53.38 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 12 Aug 2024 08:53:38 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 4/4] build: Ensure addtask before/after tasknames have prefix applied Date: Mon, 12 Aug 2024 16:53:35 +0100 Message-ID: <20240812155335.904273-4-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20240812155335.904273-1-richard.purdie@linuxfoundation.org> References: <20240812155335.904273-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Mon, 12 Aug 2024 15:53:46 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16474 "addtask do_XXX before YYY after ZZZ " where YYY or ZZZ is missing the "do_" prefix don't work as expected. Ajust the code so that it doesn't just silently do the wrong thing but works as expected. Expand a test case to cover this. Signed-off-by: Richard Purdie --- lib/bb/build.py | 13 +++++++++---- lib/bb/tests/parse.py | 6 +++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/bb/build.py b/lib/bb/build.py index ab8bce3d57..d226aadacb 100644 --- a/lib/bb/build.py +++ b/lib/bb/build.py @@ -932,9 +932,13 @@ def add_tasks(tasklist, d): # don't assume holding a reference d.setVar('_task_deps', task_deps) +def ensure_task_prefix(name): + if name[:3] != "do_": + name = "do_" + name + return name + def addtask(task, before, after, d): - if task[:3] != "do_": - task = "do_" + task + task = ensure_task_prefix(task) d.setVarFlag(task, "task", 1) bbtasks = d.getVar('__BBTASKS', False) or [] @@ -946,19 +950,20 @@ def addtask(task, before, after, d): if after is not None: # set up deps for function for entry in after.split(): + entry = ensure_task_prefix(entry) if entry not in existing: existing.append(entry) d.setVarFlag(task, "deps", existing) if before is not None: # set up things that depend on this func for entry in before.split(): + entry = ensure_task_prefix(entry) existing = d.getVarFlag(entry, "deps", False) or [] if task not in existing: d.setVarFlag(entry, "deps", [task] + existing) def deltask(task, d): - if task[:3] != "do_": - task = "do_" + task + task = ensure_task_prefix(task) bbtasks = d.getVar('__BBTASKS', False) or [] if task in bbtasks: diff --git a/lib/bb/tests/parse.py b/lib/bb/tests/parse.py index 97df2c4590..410679d5a1 100644 --- a/lib/bb/tests/parse.py +++ b/lib/bb/tests/parse.py @@ -188,6 +188,9 @@ addtask do_mytask3 deltask do_mytask3# comment deltask do_mytask4 # comment2 +# Ensure a missing task prefix on after works +addtask do_mytask5 after mytask + MYVAR = "do_patch" EMPTYVAR = "" deltask do_fetch ${MYVAR} ${EMPTYVAR} @@ -198,7 +201,8 @@ deltask ${EMPTYVAR} f = self.parsehelper(self.addtask_deltask) d = bb.parse.handle(f.name, self.d)[''] - self.assertEqual(['do_fetch2', 'do_patch2', 'do_myplaintask', 'do_mytask', 'do_mytask2'], d.getVar("__BBTASKS")) + self.assertEqual(['do_fetch2', 'do_patch2', 'do_myplaintask', 'do_mytask', 'do_mytask2', 'do_mytask5'], d.getVar("__BBTASKS")) + self.assertEqual(['do_mytask'], d.getVarFlag("do_mytask5", "deps")) broken_multiline_comment = """ # First line of comment \\