From patchwork Sat Nov 13 16:25:47 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 36 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 1212EC433FE for ; Sat, 13 Nov 2021 16:25:54 +0000 (UTC) Received: from mail-wr1-f45.google.com (mail-wr1-f45.google.com [209.85.221.45]) by mx.groups.io with SMTP id smtpd.web10.476.1636820752440552057 for ; Sat, 13 Nov 2021 08:25:52 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=hqbgY5lk; spf=pass (domain: linuxfoundation.org, ip: 209.85.221.45, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wr1-f45.google.com with SMTP id i5so21320352wrb.2 for ; Sat, 13 Nov 2021 08:25:52 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=from:to:subject:date:message-id:mime-version :content-transfer-encoding; bh=10tqX30b1r6m3u48GYWCmHxF5eby0XyY/Q9rLadSs7E=; b=hqbgY5lkZgd11gNvKRCKgNiegIVml/ZIxev2B/JstMyPy/96k4QGpb4I0bgnGyHdPk x8mZu4mPNSr/wVsfgRRT0o7HZ+UjJeTblFJTLGgp84rzKZRfbeaPZ33gyiuuPiczjTzy c7Ix+osTzqH/Wsu6InwKrLmmj+ZWahWACCMcM= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:subject:date:message-id:mime-version :content-transfer-encoding; bh=10tqX30b1r6m3u48GYWCmHxF5eby0XyY/Q9rLadSs7E=; b=mRlSXZ4wCJmtuqO9iEQDwVWyeVVQrWZH0qc0Luy5aE2kSOiEBEGmttDRy3vFd+E5/m GDXlj3npeSwAWQiBh9iZ4u1ncV14yYwf4Yv1ayc5nlJ9MiaPvuHkmI5kgL8wjvpkWexU sbUGozPi1ygwpul0yjCJSHLgK4kdUKu8Yy9kDVlj+vCbGUL1U5zPvoxgbvFmmbYXEXTo Z4kWkzoi5oyUYT3t5EpJzmhdlUTxqwRy72aN7wk/o1Bh+60BH5dLSj3IxkJ3zPPiAjyk xkDn7J2rE6j0oFKBa27fHeUhBSz4ySU89u65CgXHXILVmFrVyVzErsgz6p9kmvGcAYLQ Io9g== X-Gm-Message-State: AOAM532OWh4kRfn2wG8KWt2iYA7NTASRFAJ+7/2U5q2uHt8Fr4X7HSlY hHIkGKohVVEYecOuBhv92OD4MNTE+vHAqQ== X-Google-Smtp-Source: ABdhPJyk3DY4uk4oOWAKmyP5aK/V/hURQX5BAHJfVJ6Menw9vaW53gbf1ESni6zCItM2u9jHikJjFw== X-Received: by 2002:adf:ecce:: with SMTP id s14mr29108401wro.98.1636820750714; Sat, 13 Nov 2021 08:25:50 -0800 (PST) Received: from hex.int.rpsys.net ([2001:8b0:aba:5f3c:4f1d:7b2:7d47:57f5]) by smtp.gmail.com with ESMTPSA id b14sm11331714wrd.24.2021.11.13.08.25.50 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sat, 13 Nov 2021 08:25:50 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 1/3] tests/runqueue: Set TOPDIR in test Date: Sat, 13 Nov 2021 16:25:47 +0000 Message-Id: <20211113162549.3931833-1-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.32.0 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Sat, 13 Nov 2021 16:25:54 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/13038 It was clear with testing that we're asuming bitbake sets TOPDIR correctly when running these tests. Remove that implict assumption and make it explicit. Signed-off-by: Richard Purdie --- lib/bb/tests/runqueue.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/bb/tests/runqueue.py b/lib/bb/tests/runqueue.py index 5b6ada886a..2bf00dc62a 100644 --- a/lib/bb/tests/runqueue.py +++ b/lib/bb/tests/runqueue.py @@ -29,9 +29,10 @@ class RunQueueTests(unittest.TestCase): def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None, cleanup=False): env = os.environ.copy() env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests")) - env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS" + env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS TOPDIR" env["SSTATEVALID"] = sstatevalid env["SLOWTASKS"] = slowtasks + env["TOPDIR"] = builddir if extraenv: for k in extraenv: env[k] = extraenv[k] From patchwork Sat Nov 13 16:25:48 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 38 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 1002DC433F5 for ; Sat, 13 Nov 2021 16:25:54 +0000 (UTC) Received: from mail-wr1-f43.google.com (mail-wr1-f43.google.com [209.85.221.43]) by mx.groups.io with SMTP id smtpd.web11.539.1636820752639588733 for ; Sat, 13 Nov 2021 08:25:53 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=WEbSyVEk; spf=pass (domain: linuxfoundation.org, ip: 209.85.221.43, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wr1-f43.google.com with SMTP id t30so21229350wra.10 for ; Sat, 13 Nov 2021 08:25:52 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=from:to:subject:date:message-id:in-reply-to:references:mime-version :content-transfer-encoding; bh=9dNgO6nlPWjeZcFuExlEOEXXvjJkhYSWVl+vALymf+c=; b=WEbSyVEk9XP5wrmKFjwjRyWKmghXtDBcX76Bqra6E5lI42SPnDU5Wz+fZhijnoMY+J S+aDuWU1XTTTQJbiQs9zHzZWH8DpCe5fEp47GnEGMPqCGvky0dHdersI6ioFf4uafD1x jj/tds3PhaqHaZbPPIRZrV3FTr7uID7WMMMuM= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=9dNgO6nlPWjeZcFuExlEOEXXvjJkhYSWVl+vALymf+c=; b=NDYPOaQwn2toxtXxPeqsunlfgz1uy4mbkTY3Ze64/Hb7GtCveE/yMIoWZOFlG36iFm LIy1boyvphMyTWMI29AGjmnA8bI7l9kjJhZLA+lBIF39vBMZjeCBTBUYzEkvb9/Zn2Aj FGUtSrljgj4cJWxghKaKoiPFHL05ZO9MMVqsysG+ObRVLf3X4IjSdLVMDSCrW0WshCWC dn5LWkDXdZUbt4icR7Mg0z1ExTBiXeaWqPEpD1fKOXwoTR/kGnATK8lTEApgLq1XsrZT k+0Nue6NoauBj36vaFfn0XWd6Oe6zaoVrie3ySqlz+8yUCK8M3rdcUO4eRaNqVAElXY1 5NmQ== X-Gm-Message-State: AOAM532PeFjWalnQg/OIVy7lXh5dVVcmnL1+2otNqrdwoGjKYWlnqkXr fuyewjNshzi6lz/+lXzWZICb4l/KiMvRVg== X-Google-Smtp-Source: ABdhPJwV92IS4fT+bb725ePmDHV6tMdSZwEKCNXNdXv3okbZlYQP15pWDChE9Yjy0mCHvfV9UmjpdA== X-Received: by 2002:a05:6000:1862:: with SMTP id d2mr29468956wri.203.1636820751012; Sat, 13 Nov 2021 08:25:51 -0800 (PST) Received: from hex.int.rpsys.net ([2001:8b0:aba:5f3c:4f1d:7b2:7d47:57f5]) by smtp.gmail.com with ESMTPSA id b14sm11331714wrd.24.2021.11.13.08.25.50 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sat, 13 Nov 2021 08:25:50 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 2/3] cookerdata: Set TOPDIR explicitly and fix broke findTopdir() Date: Sat, 13 Nov 2021 16:25:48 +0000 Message-Id: <20211113162549.3931833-2-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.32.0 In-Reply-To: <20211113162549.3931833-1-richard.purdie@linuxfoundation.org> References: <20211113162549.3931833-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Sat, 13 Nov 2021 16:25:54 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/13039 TOPDIR is set internally deep within the parser to os.getcwd(). Rather than do that, set it explicitly if not set. Note that modern code will almost always have a bblayers.conf file which would have already set TOPDIR before this new code. Also fix findTopdir since the conf/bitbake.conf codepath is just plain incorrect, it would find build metadata, not the current build directory that bitbake would use. Again, the use of bblayers.conf means hitting the fallback code was unlikely. This change makes everything clear and explicit. Signed-off-by: Richard Purdie --- lib/bb/cookerdata.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py index 592bc2968e..397b43dfa7 100644 --- a/lib/bb/cookerdata.py +++ b/lib/bb/cookerdata.py @@ -210,7 +210,7 @@ def findConfigFile(configfile, data): # # We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working -# up to /. If that fails, we search for a conf/bitbake.conf in BBPATH. +# up to /. If that fails, bitbake would fall back to cwd. # def findTopdir(): @@ -223,11 +223,8 @@ def findTopdir(): layerconf = findConfigFile("bblayers.conf", d) if layerconf: return os.path.dirname(os.path.dirname(layerconf)) - if bbpath: - bitbakeconf = bb.utils.which(bbpath, "conf/bitbake.conf") - if bitbakeconf: - return os.path.dirname(os.path.dirname(bitbakeconf)) - return None + + return os.path.abspath(os.getcwd()) class CookerDataBuilder(object): @@ -417,6 +414,9 @@ class CookerDataBuilder(object): " invoked bitbake from the wrong directory?") raise SystemExit(msg) + if not data.getVar("TOPDIR"): + data.setVar("TOPDIR", os.path.abspath(os.getcwd())) + data = parse_config_file(os.path.join("conf", "bitbake.conf"), data) # Parse files for loading *after* bitbake.conf and any includes From patchwork Sat Nov 13 16:25:49 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 37 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 13535C433EF for ; Sat, 13 Nov 2021 16:25:54 +0000 (UTC) Received: from mail-wm1-f41.google.com (mail-wm1-f41.google.com [209.85.128.41]) by mx.groups.io with SMTP id smtpd.web09.474.1636820753188875560 for ; Sat, 13 Nov 2021 08:25:53 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=HDuq9bj5; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.41, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f41.google.com with SMTP id y84-20020a1c7d57000000b00330cb84834fso12292719wmc.2 for ; Sat, 13 Nov 2021 08:25:52 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=from:to:subject:date:message-id:in-reply-to:references:mime-version :content-transfer-encoding; bh=Nod75QqsWO77cNsFuYxLiS1JteZrNz7uxFrawiUzJ+4=; b=HDuq9bj5dK3BktMFf1BSM7M6s5ErcysYJlhC1F4sKTPrZmje+LJ10gGuPEdfxogpFH wOria0LLLhToPLFjlS+9h8NWywHU1dTCe1oOrcybV1FwR5q1v93osIyazouSZRaMNSGQ Iqb7zpw7XmspspGRsCXdjwBvUoB80+6+I/pqQ= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=Nod75QqsWO77cNsFuYxLiS1JteZrNz7uxFrawiUzJ+4=; b=5MEb3OlKiM5kNX1GCGuTDZxZmvDx2OrQnVCvPxiuaChPgAYt97mByiK354JqLRTiT9 nZD0BW6mBhGS/DStJXWOTAHHF4ggWCXwKyCns8rXmULxVcVGBQ03Hcgqq5jp/0OC2UsG Ga935SryuKmM6rTBUUYOz4BdlzYHewenwfcCGumQ3YYM6+QekY8mJL/fKpFdGSNNE19A 3SiXU/yd5wkvLMzHPtMuccg+n9Fz/M89/MpTfdPzLU2MYYza+aCV8WGkQxE+KoiiohOP Jy59NE4q5yx/vaiOLGcdnkjrUZ1c9x+fec0z/JryCQaK4BtM34TTBLAv2w0bRR+bqLoL Anug== X-Gm-Message-State: AOAM532WAFn4jg/aPlNtZQkbdMDyKUo7akAnGqu9l8GlOIxHdosRWJ7D 9/7Cg58SCHxLk+TFrqSK+b/o3y6EWf6N2w== X-Google-Smtp-Source: ABdhPJyuksugrn8UJP7OcoyEmjDABZUsDeL86n2V8BvC5EemGtst0ehkLaaliyemIPM1JicBQIAPUg== X-Received: by 2002:a05:600c:3586:: with SMTP id p6mr45017754wmq.34.1636820751548; Sat, 13 Nov 2021 08:25:51 -0800 (PST) Received: from hex.int.rpsys.net ([2001:8b0:aba:5f3c:4f1d:7b2:7d47:57f5]) by smtp.gmail.com with ESMTPSA id b14sm11331714wrd.24.2021.11.13.08.25.51 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sat, 13 Nov 2021 08:25:51 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 3/3] cache/ConfHandler: Drop TOPDIR/chdir manipulations Date: Sat, 13 Nov 2021 16:25:49 +0000 Message-Id: <20211113162549.3931833-3-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.32.0 In-Reply-To: <20211113162549.3931833-1-richard.purdie@linuxfoundation.org> References: <20211113162549.3931833-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Sat, 13 Nov 2021 16:25:54 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/13040 This code has been unchanged since 2006 apart from attempts to optimise performance by minimising chdir() calls. There is no reason the modern bitbake parser should be changing directory all the time. We did have some path assumptions in the mists of time but those were resovled and the code is deterministic and doesn't depend on cwd now for parsing. We can therefore drop the changes in directory. Also, TOPDIR is now being set by cookerdata in all cases so we don't need the fallbacks in this code (which was used to effectively initialise a value). We don't need to change TOPDIR when parsing a recipe, that makes no sense. If we stop all the other messing around, we don't need to expand TMPDIR either. These changes have the potential to break some obscure use cases such as an anonymous function assuming the current working directory, or some case which depends on TOPDIR changing but I believe any such uses should be fixed at this point. Signed-off-by: Richard Purdie --- lib/bb/cache.py | 29 ++++------------------------ lib/bb/parse/parse_py/ConfHandler.py | 5 +---- 2 files changed, 5 insertions(+), 29 deletions(-) diff --git a/lib/bb/cache.py b/lib/bb/cache.py index 4e08c100ab..fcb15796cc 100644 --- a/lib/bb/cache.py +++ b/lib/bb/cache.py @@ -284,36 +284,15 @@ def parse_recipe(bb_data, bbfile, appends, mc=''): Parse a recipe """ - chdir_back = False - bb_data.setVar("__BBMULTICONFIG", mc) - # expand tmpdir to include this topdir - bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR') or "") bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) - oldpath = os.path.abspath(os.getcwd()) bb.parse.cached_mtime_noerror(bbfile_loc) - # The ConfHandler first looks if there is a TOPDIR and if not - # then it would call getcwd(). - # Previously, we chdir()ed to bbfile_loc, called the handler - # and finally chdir()ed back, a couple of thousand times. We now - # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet. - if not bb_data.getVar('TOPDIR', False): - chdir_back = True - bb_data.setVar('TOPDIR', bbfile_loc) - try: - if appends: - bb_data.setVar('__BBAPPEND', " ".join(appends)) - bb_data = bb.parse.handle(bbfile, bb_data) - if chdir_back: - os.chdir(oldpath) - return bb_data - except: - if chdir_back: - os.chdir(oldpath) - raise - + if appends: + bb_data.setVar('__BBAPPEND', " ".join(appends)) + bb_data = bb.parse.handle(bbfile, bb_data) + return bb_data class NoCache(object): diff --git a/lib/bb/parse/parse_py/ConfHandler.py b/lib/bb/parse/parse_py/ConfHandler.py index 0834fe3f9b..b895d5b5ef 100644 --- a/lib/bb/parse/parse_py/ConfHandler.py +++ b/lib/bb/parse/parse_py/ConfHandler.py @@ -48,10 +48,7 @@ __unset_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)$" ) __unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.]+)\]$" ) def init(data): - topdir = data.getVar('TOPDIR', False) - if not topdir: - data.setVar('TOPDIR', os.getcwd()) - + return def supports(fn, d): return fn[-5:] == ".conf"