From patchwork Tue Apr 29 08:11:17 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Philip Lorenz X-Patchwork-Id: 62086 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id B06A7C3ABAF for ; Tue, 29 Apr 2025 08:18:05 +0000 (UTC) Received: from esa2.hc324-48.eu.iphmx.com (esa2.hc324-48.eu.iphmx.com [207.54.68.120]) by mx.groups.io with SMTP id smtpd.web11.13074.1745914681685012812 for ; Tue, 29 Apr 2025 01:18:03 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bmw.de header.s=mailing1 header.b=hjDWhdKf; spf=pass (domain: bmw.de, ip: 207.54.68.120, mailfrom: prvs=207d80f5d=philip.lorenz@bmw.de) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bmw.de; i=@bmw.de; q=dns/txt; s=mailing1; t=1745914682; x=1777450682; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=vg2+lyh2R2Sueo4iIR3js+irAiQxe4PHeUw6uhRoWMM=; b=hjDWhdKfifPrQ32/cDfpGDr8Qktu8mS5K0xqWwwl5jlWtOUteDaUH5BY o5CEF5v3KC4K6SjQXXVYPE30r9H2PFFHiS27LlCzogH7XruHAPdVV8Dov 6AORDCov0Y/fxdgWKNYzXgujSJXcgBxM2kGVfM0zufMaeFn4W71T8EBwR E=; X-CSE-ConnectionGUID: J3Herv3aQBOlJq+WY/Y6Ng== X-CSE-MsgGUID: +T66giY6T2ChqyJ8FtG7Tw== Received: from esagw2.bmwgroup.com (HELO esagw2.muc) ([160.46.252.38]) by esa2.hc324-48.eu.iphmx.com with ESMTP/TLS; 29 Apr 2025 10:17:59 +0200 Received: from esabb4.muc ([160.50.100.33]) by esagw2.muc with ESMTP/TLS; 29 Apr 2025 10:17:57 +0200 Received: from smucmp19d.bmwgroup.net (HELO smucmp19d.europe.bmw.corp) ([10.30.13.170]) by esabb4.muc with ESMTP/TLS; 29 Apr 2025 10:17:58 +0200 Received: from localhost.localdomain (10.30.85.212) by smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) with Microsoft SMTP Server (version=TLS; Tue, 29 Apr 2025 10:17:57 +0200 X-CSE-ConnectionGUID: 7MNMQBeASfydtVQLxl+vAQ== X-CSE-MsgGUID: ipv/60O2Qmi11J9PrataGw== X-CSE-ConnectionGUID: kpaKKZBpRNWwcLnhzod0Mg== X-CSE-MsgGUID: xyyowx3rRvivAfnr93Hd0A== From: Philip Lorenz To: CC: Philip Lorenz , , Subject: [PATCH v4 1/8] fetch2: Clean up no longer used name parameter Date: Tue, 29 Apr 2025 10:11:17 +0200 Message-ID: <20250429081124.3695505-2-philip.lorenz@bmw.de> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250429081124.3695505-1-philip.lorenz@bmw.de> References: <20250429081124.3695505-1-philip.lorenz@bmw.de> MIME-Version: 1.0 X-ClientProxiedBy: smucmp09b.europe.bmw.corp (2a03:1e80:a15:58f::203f) To smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Apr 2025 08:18:05 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/17585 There's no need to pass `name` when it is no longer used. Signed-off-by: Philip Lorenz --- lib/bb/fetch2/git.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index b47a53e3b..39c183927 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -327,7 +327,7 @@ class Git(FetchMethod): if self.clonedir_need_update(ud, d): return True - if not self._lfs_objects_downloaded(ud, d, ud.name, ud.clonedir): + if not self._lfs_objects_downloaded(ud, d, ud.clonedir): return True return False @@ -802,7 +802,7 @@ class Git(FetchMethod): raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) return output.split()[0] != "0" - def _lfs_objects_downloaded(self, ud, d, name, wd): + def _lfs_objects_downloaded(self, ud, d, wd): """ Verifies whether the LFS objects for requested revisions have already been downloaded """ @@ -841,7 +841,7 @@ class Git(FetchMethod): if ud.nobranch: # If no branch is specified, use the current git commit - refname = self._build_revision(ud, d, ud.name) + refname = ud.revision elif wd == ud.clonedir: # The bare clonedir doesn't use the remote names; it has the branch immediately. refname = ud.branch @@ -995,7 +995,7 @@ class Git(FetchMethod): Return a sortable revision number by counting commits in the history Based on gitpkgv.bblass in meta-openembedded """ - rev = self._build_revision(ud, d, name) + rev = ud.revision localpath = ud.localpath rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev) if not os.path.exists(localpath): From patchwork Tue Apr 29 08:11:18 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Philip Lorenz X-Patchwork-Id: 62088 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id B07A6C3ABB2 for ; Tue, 29 Apr 2025 08:18:05 +0000 (UTC) Received: from esa2.hc324-48.eu.iphmx.com (esa2.hc324-48.eu.iphmx.com [207.54.68.120]) by mx.groups.io with SMTP id smtpd.web10.13165.1745914683459549615 for ; Tue, 29 Apr 2025 01:18:04 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bmw.de header.s=mailing1 header.b=RtXI/TD+; spf=pass (domain: bmw.de, ip: 207.54.68.120, mailfrom: prvs=207d80f5d=philip.lorenz@bmw.de) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bmw.de; i=@bmw.de; q=dns/txt; s=mailing1; t=1745914683; x=1777450683; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=oFUs339dldr+AL+nMX5mR8UYsSCYNtjbSwumyxZ306Q=; b=RtXI/TD+dJdAJ8xmbFVFf5R6gMGNKT6w8fTYv3PhouqFlLFaw4PXiD7n UDNZXxZcW0N+ATe9AGUv7+JWidZSUpR4MVYJGmSOwDS1vNSU/pfGqJZ7b XccmmGCHq9ZEjZ/uGrdX+b7paxc1Ett8LDLmO+Udwod78v8tfvL/wvbz8 k=; X-CSE-ConnectionGUID: J3Herv3aQBOlJq+WY/Y6Ng== X-CSE-MsgGUID: pLCsnZVsQlqiWtk+6nXlqg== Received: from esagw2.bmwgroup.com (HELO esagw2.muc) ([160.46.252.38]) by esa2.hc324-48.eu.iphmx.com with ESMTP/TLS; 29 Apr 2025 10:17:59 +0200 Received: from esabb4.muc ([160.50.100.33]) by esagw2.muc with ESMTP/TLS; 29 Apr 2025 10:17:57 +0200 Received: from smucmp19d.bmwgroup.net (HELO smucmp19d.europe.bmw.corp) ([10.30.13.170]) by esabb4.muc with ESMTP/TLS; 29 Apr 2025 10:17:58 +0200 Received: from localhost.localdomain (10.30.85.212) by smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) with Microsoft SMTP Server (version=TLS; Tue, 29 Apr 2025 10:17:57 +0200 X-CSE-ConnectionGUID: V/K8fL8HTiSlGbQWFUg7EA== X-CSE-MsgGUID: jV7QFlezREKCuaeDQ9SMNg== X-CSE-ConnectionGUID: kxbv8TJ7RVGijJ1+6GC5fQ== X-CSE-MsgGUID: tcc4XL3ZRZ6IsZgXvipu8w== From: Philip Lorenz To: CC: Philip Lorenz , , Subject: [PATCH v4 2/8] tests/fetch: Move commonly used imports to top Date: Tue, 29 Apr 2025 10:11:18 +0200 Message-ID: <20250429081124.3695505-3-philip.lorenz@bmw.de> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250429081124.3695505-1-philip.lorenz@bmw.de> References: <20250429081124.3695505-1-philip.lorenz@bmw.de> MIME-Version: 1.0 X-ClientProxiedBy: smucmp09b.europe.bmw.corp (2a03:1e80:a15:58f::203f) To smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Apr 2025 08:18:05 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/17586 Avoid multiple import statements for anything that is used more than once. Additionally, drop no longer used imports. Signed-off-by: Philip Lorenz --- lib/bb/tests/fetch.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 196d93c41..f0c628524 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -7,7 +7,9 @@ # import contextlib +import shutil import unittest +import urllib.parse import hashlib import tempfile import collections @@ -1275,7 +1277,6 @@ class FetcherNetworkTest(FetcherTest): class SVNTest(FetcherTest): def skipIfNoSvn(): - import shutil if not shutil.which("svn"): return unittest.skip("svn not installed, tests being skipped") @@ -1398,8 +1399,6 @@ class TrustedNetworksTest(FetcherTest): self.assertFalse(bb.fetch.trusted_network(self.d, url)) class URLHandle(unittest.TestCase): - import urllib.parse - # Quote password as per RFC3986 password = urllib.parse.quote(r"!#$%^&*()-_={}[]\|:?,.<>~`", r"!$&'/()*+,;=") datatable = { @@ -1426,7 +1425,6 @@ class URLHandle(unittest.TestCase): self.assertEqual(result, v) def test_encodeurl(self): - import urllib.parse for k, v in self.datatable.items(): result = bb.fetch.encodeurl(v) if result.startswith("file:"): @@ -2271,7 +2269,6 @@ class GitShallowTest(FetcherTest): class GitLfsTest(FetcherTest): def skipIfNoGitLFS(): - import shutil if not shutil.which('git-lfs'): return unittest.skip('git-lfs not installed') return lambda f: f @@ -2391,8 +2388,6 @@ class GitLfsTest(FetcherTest): @skipIfNoGitLFS() def test_lfs_enabled(self): - import shutil - uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir self.d.setVar('SRC_URI', uri) @@ -2403,8 +2398,6 @@ class GitLfsTest(FetcherTest): @skipIfNoGitLFS() def test_lfs_disabled(self): - import shutil - uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir self.d.setVar('SRC_URI', uri) @@ -2414,8 +2407,6 @@ class GitLfsTest(FetcherTest): fetcher.unpack(self.d.getVar('WORKDIR')) def test_lfs_enabled_not_installed(self): - import shutil - uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir self.d.setVar('SRC_URI', uri) @@ -2436,8 +2427,6 @@ class GitLfsTest(FetcherTest): ud.method._find_git_lfs = old_find_git_lfs def test_lfs_disabled_not_installed(self): - import shutil - uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir self.d.setVar('SRC_URI', uri) @@ -2611,7 +2600,6 @@ class CrateTest(FetcherTest): class NPMTest(FetcherTest): def skipIfNoNpm(): - import shutil if not shutil.which('npm'): return unittest.skip('npm not installed') return lambda f: f @@ -3294,7 +3282,6 @@ class FetchPremirroronlyNetworkTest(FetcherTest): self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") def make_git_repo(self): - import shutil self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz" os.makedirs(self.clonedir) self.git("clone --bare {}".format(self.recipe_url), self.clonedir) @@ -3324,7 +3311,6 @@ class FetchPremirroronlyMercurialTest(FetcherTest): the test covers also basic hg:// clone (see fetch_and_create_tarball """ def skipIfNoHg(): - import shutil if not shutil.which('hg'): return unittest.skip('Mercurial not installed') return lambda f: f @@ -3380,7 +3366,6 @@ class FetchPremirroronlyBrokenTarball(FetcherTest): targz.write("This is not tar.gz file!") def test_mirror_broken_download(self): - import sys self.d.setVar("SRCREV", "0"*40) fetcher = bb.fetch.Fetch([self.recipe_url], self.d) with self.assertRaises(bb.fetch2.FetchError), self.assertLogs() as logs: From patchwork Tue Apr 29 08:11:19 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Philip Lorenz X-Patchwork-Id: 62087 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 8BD91C369DC for ; Tue, 29 Apr 2025 08:18:05 +0000 (UTC) Received: from esa8.hc324-48.eu.iphmx.com (esa8.hc324-48.eu.iphmx.com [207.54.65.242]) by mx.groups.io with SMTP id smtpd.web11.13072.1745914679924280858 for ; Tue, 29 Apr 2025 01:18:01 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bmw.de header.s=mailing1 header.b=At+7Y3bo; spf=pass (domain: bmw.de, ip: 207.54.65.242, mailfrom: prvs=207d80f5d=philip.lorenz@bmw.de) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bmw.de; i=@bmw.de; q=dns/txt; s=mailing1; t=1745914679; x=1777450679; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=ta4KW303dgY6m04ScGAo5B4pqFBGvcMdDMyGs4MWM/o=; b=At+7Y3boRLlT8NMQQJPNgIHC3xc58ngxsJ9WZY9KVtKWSKtx+UENg23E k6hXG6eTSnEEvETVSe/cy35LJ3mXJWDVkgfhp5+kpdmOwMaqe5s166it2 UH0O7ieJvxJJUoZSM4i9VUZRzWR6UaK0iMOlyAak9GGNGOC/F9WUk1GtP 8=; X-CSE-ConnectionGUID: vzAxsUweRQuOBajErrDHOQ== X-CSE-MsgGUID: BaRxysb7Rpi1TESt7lxqrw== Received: from 160.46.252.34.spf.bmwgroup.com (HELO esagw1.muc) ([160.46.252.34]) by esa8.hc324-48.eu.iphmx.com with ESMTP/TLS; 29 Apr 2025 10:17:57 +0200 Received: from esabb3.muc ([160.50.100.30]) by esagw1.muc with ESMTP/TLS; 29 Apr 2025 10:17:57 +0200 Received: from smucmp19d.bmwgroup.net (HELO smucmp19d.europe.bmw.corp) ([10.30.13.170]) by esabb3.muc with ESMTP/TLS; 29 Apr 2025 10:17:57 +0200 Received: from localhost.localdomain (10.30.85.212) by smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) with Microsoft SMTP Server (version=TLS; Tue, 29 Apr 2025 10:17:57 +0200 X-CSE-ConnectionGUID: xsz7vaq3TASiPOrVMsEDtA== X-CSE-MsgGUID: CdA1zMygSO+wD+rhct1jKg== X-CSE-ConnectionGUID: 7RlI4xICRhq2xDMSz6E8+Q== X-CSE-MsgGUID: T1TQqzEhTfe5HlMwuNEcfA== From: Philip Lorenz To: CC: Philip Lorenz , , Subject: [PATCH v4 3/8] fetch2: Check for git-lfs existence before using it Date: Tue, 29 Apr 2025 10:11:19 +0200 Message-ID: <20250429081124.3695505-4-philip.lorenz@bmw.de> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250429081124.3695505-1-philip.lorenz@bmw.de> References: <20250429081124.3695505-1-philip.lorenz@bmw.de> MIME-Version: 1.0 X-ClientProxiedBy: smucmp09b.europe.bmw.corp (2a03:1e80:a15:58f::203f) To smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Apr 2025 08:18:05 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/17582 So far, existence of `git-lfs` was only checked during unpacking. As the binary is also used in earlier steps also check for its existence there. Additionally, factor out the LFS existence check into a dedicated function and call it wherever git-lfs is used for the first time. Signed-off-by: Philip Lorenz --- V2 -> V3: * Only check for git-lfs existence after verifying that repository is using it * Add test case to verify that git-lfs is not required when the repository does not have a single LFS filter configured V3 -> V4: * Move "lfs enabled but git-lfs not installed during unpack" into a dedicated test case (which can be skipped) to avoid test failures when the host doesn't have git-lfs which is needed for test case preparation --- lib/bb/fetch2/git.py | 26 ++++++++++----- lib/bb/tests/fetch.py | 73 +++++++++++++++++++++++++++++-------------- 2 files changed, 68 insertions(+), 31 deletions(-) diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 39c183927..9e5833735 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -324,6 +324,9 @@ class Git(FetchMethod): return False def lfs_need_update(self, ud, d): + if not self._need_lfs(ud): + return False + if self.clonedir_need_update(ud, d): return True @@ -507,7 +510,9 @@ class Git(FetchMethod): def lfs_fetch(self, ud, d, clonedir, revision, fetchall=False, progresshandler=None): """Helper method for fetching Git LFS data""" try: - if self._need_lfs(ud) and self._contains_lfs(ud, d, clonedir) and self._find_git_lfs(d) and len(revision): + if self._need_lfs(ud) and self._contains_lfs(ud, d, clonedir) and len(revision): + self._ensure_git_lfs(d, ud) + # Using worktree with the revision because .lfsconfig may exists worktree_add_cmd = "%s worktree add wt %s" % (ud.basecmd, revision) runfetchcmd(worktree_add_cmd, d, log=progresshandler, workdir=clonedir) @@ -740,11 +745,11 @@ class Git(FetchMethod): runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) if self._contains_lfs(ud, d, destdir): - if need_lfs and not self._find_git_lfs(d): - raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl)) - elif not need_lfs: + if not need_lfs: bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) else: + self._ensure_git_lfs(d, ud) + runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir) if not ud.nocheckout: @@ -807,9 +812,11 @@ class Git(FetchMethod): Verifies whether the LFS objects for requested revisions have already been downloaded """ # Bail out early if this repository doesn't use LFS - if not self._need_lfs(ud) or not self._contains_lfs(ud, d, wd): + if not self._contains_lfs(ud, d, wd): return True + self._ensure_git_lfs(d, ud) + # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file # existence. # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git @@ -859,11 +866,14 @@ class Git(FetchMethod): pass return False - def _find_git_lfs(self, d): + def _ensure_git_lfs(self, d, ud): """ - Return True if git-lfs can be found, False otherwise. + Ensures that git-lfs is available, raising a FetchError if it isn't. """ - return shutil.which("git-lfs", path=d.getVar('PATH')) is not None + if shutil.which("git-lfs", path=d.getVar('PATH')) is None: + raise bb.fetch2.FetchError( + "Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 " + "to ignore it)" % self._get_repo_url(ud)) def _get_repo_url(self, ud): """ diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index f0c628524..c77725190 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -9,6 +9,7 @@ import contextlib import shutil import unittest +import unittest.mock import urllib.parse import hashlib import tempfile @@ -2292,12 +2293,18 @@ class GitLfsTest(FetcherTest): self.git_init(cwd=self.srcdir) self.commit_file('.gitattributes', '*.mp3 filter=lfs -text') - def commit_file(self, filename, content): - with open(os.path.join(self.srcdir, filename), "w") as f: + def commit(self, *, cwd=None): + cwd = cwd or self.srcdir + self.git(["commit", "-m", "Change"], cwd=cwd) + return self.git(["rev-parse", "HEAD"], cwd=cwd).strip() + + def commit_file(self, filename, content, *, cwd=None): + cwd = cwd or self.srcdir + + with open(os.path.join(cwd, filename), "w") as f: f.write(content) - self.git(["add", filename], cwd=self.srcdir) - self.git(["commit", "-m", "Change"], cwd=self.srcdir) - return self.git(["rev-parse", "HEAD"], cwd=self.srcdir).strip() + self.git(["add", filename], cwd=cwd) + return self.commit(cwd=cwd) def fetch(self, uri=None, download=True): uris = self.d.getVar('SRC_URI').split() @@ -2406,6 +2413,21 @@ class GitLfsTest(FetcherTest): fetcher, ud = self.fetch() fetcher.unpack(self.d.getVar('WORKDIR')) + @skipIfNoGitLFS() + def test_lfs_enabled_not_installed_during_unpack(self): + uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir + self.d.setVar('SRC_URI', uri) + + # Careful: suppress initial attempt at downloading + fetcher, ud = self.fetch(uri=None, download=False) + + fetcher.download() + # If git-lfs cannot be found, the unpack should throw an error + with self.assertRaises(bb.fetch2.FetchError): + with unittest.mock.patch("shutil.which", return_value=None): + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) + def test_lfs_enabled_not_installed(self): uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir self.d.setVar('SRC_URI', uri) @@ -2413,18 +2435,10 @@ class GitLfsTest(FetcherTest): # Careful: suppress initial attempt at downloading fetcher, ud = self.fetch(uri=None, download=False) - # Artificially assert that git-lfs is not installed, so - # we can verify a failure to unpack in it's absence. - old_find_git_lfs = ud.method._find_git_lfs - try: - # If git-lfs cannot be found, the unpack should throw an error + # If git-lfs cannot be found, the download should throw an error + with unittest.mock.patch("shutil.which", return_value=None): with self.assertRaises(bb.fetch2.FetchError): fetcher.download() - ud.method._find_git_lfs = lambda d: False - shutil.rmtree(self.gitdir, ignore_errors=True) - fetcher.unpack(self.d.getVar('WORKDIR')) - finally: - ud.method._find_git_lfs = old_find_git_lfs def test_lfs_disabled_not_installed(self): uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir @@ -2433,17 +2447,30 @@ class GitLfsTest(FetcherTest): # Careful: suppress initial attempt at downloading fetcher, ud = self.fetch(uri=None, download=False) - # Artificially assert that git-lfs is not installed, so - # we can verify a failure to unpack in it's absence. - old_find_git_lfs = ud.method._find_git_lfs - try: - # Even if git-lfs cannot be found, the unpack should be successful + # Even if git-lfs cannot be found, the download / unpack should be successful + with unittest.mock.patch("shutil.which", return_value=None): + fetcher.download() + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) + + def test_lfs_enabled_not_installed_but_not_needed(self): + srcdir = os.path.join(self.tempdir, "emptygit") + bb.utils.mkdirhier(srcdir) + self.git_init(srcdir) + self.commit_file("test", "test content", cwd=srcdir) + + uri = 'git://%s;protocol=file;lfs=1;branch=master' % srcdir + self.d.setVar('SRC_URI', uri) + + # Careful: suppress initial attempt at downloading + fetcher, ud = self.fetch(uri=None, download=False) + + # It shouldnt't matter that git-lfs cannot be found as the repository configuration does not + # specify any LFS filters. + with unittest.mock.patch("shutil.which", return_value=None): fetcher.download() - ud.method._find_git_lfs = lambda d: False shutil.rmtree(self.gitdir, ignore_errors=True) fetcher.unpack(self.d.getVar('WORKDIR')) - finally: - ud.method._find_git_lfs = old_find_git_lfs class GitURLWithSpacesTest(FetcherTest): test_git_urls = { From patchwork Tue Apr 29 08:11:20 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Philip Lorenz X-Patchwork-Id: 62085 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id AED4CC3ABAC for ; Tue, 29 Apr 2025 08:18:05 +0000 (UTC) Received: from esa2.hc324-48.eu.iphmx.com (esa2.hc324-48.eu.iphmx.com [207.54.68.120]) by mx.groups.io with SMTP id smtpd.web11.13074.1745914681685012812 for ; Tue, 29 Apr 2025 01:18:05 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bmw.de header.s=mailing1 header.b=KBNqxge3; spf=pass (domain: bmw.de, ip: 207.54.68.120, mailfrom: prvs=207d80f5d=philip.lorenz@bmw.de) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bmw.de; i=@bmw.de; q=dns/txt; s=mailing1; t=1745914684; x=1777450684; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=yUkA4U4bV0OVe8hatckw3yuaYOZ/U9GLf8dn48Pay9Q=; b=KBNqxge3/KzQs1OEt6+AqxDqwfCBCHH7Pcf1szPqqm1thM/nqh7y1guh J68ULlfxfBHHN//RhCEg/smlMKZ6L0+eVZaZcePTMIm4oYI3JFsnUbS2y jJg9T9jJgC2/YnL9Y69AGa6uMty1scc62P/l8GiYedoCv9bAF952v5LRS 8=; X-CSE-ConnectionGUID: pjNjmoYTSpW0jtEL712mxQ== X-CSE-MsgGUID: XZE6ezYvQVKMIIMztnZRew== Received: from 160.46.252.35.spf.bmwgroup.com.bmwgroup.com (HELO esagw3.muc) ([160.46.252.35]) by esa2.hc324-48.eu.iphmx.com with ESMTP/TLS; 29 Apr 2025 10:18:00 +0200 Received: from esabb2.muc ([160.50.100.34]) by esagw3.muc with ESMTP/TLS; 29 Apr 2025 10:17:58 +0200 Received: from smucmp19d.bmwgroup.net (HELO smucmp19d.europe.bmw.corp) ([10.30.13.170]) by esabb2.muc with ESMTP/TLS; 29 Apr 2025 10:17:58 +0200 Received: from localhost.localdomain (10.30.85.212) by smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) with Microsoft SMTP Server (version=TLS; Tue, 29 Apr 2025 10:17:57 +0200 X-CSE-ConnectionGUID: YpWCJIbrTYOMudO5ITUPvQ== X-CSE-MsgGUID: d7xThcrtSfGZc10G+cl6zQ== X-CSE-ConnectionGUID: apNWwBaeT1mJE3YeIhfoFg== X-CSE-MsgGUID: 8aIGHnV1SP+WWiT8kLbHKg== From: Philip Lorenz To: CC: Philip Lorenz , , Subject: [PATCH v4 4/8] fetch2: Simplify git LFS detection Date: Tue, 29 Apr 2025 10:11:20 +0200 Message-ID: <20250429081124.3695505-5-philip.lorenz@bmw.de> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250429081124.3695505-1-philip.lorenz@bmw.de> References: <20250429081124.3695505-1-philip.lorenz@bmw.de> MIME-Version: 1.0 X-ClientProxiedBy: smucmp09b.europe.bmw.corp (2a03:1e80:a15:58f::203f) To smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Apr 2025 08:18:05 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/17589 Its unclear why this function does not operate on the desired source revision to begin with (which really should be the decider on whether a particular source revision uses LFS or not). Simplify the decision logic by always checking the `.gitattributes` file of the target revision. Signed-off-by: Philip Lorenz --- lib/bb/fetch2/git.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 9e5833735..9b2ad455c 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -845,18 +845,8 @@ class Git(FetchMethod): """ Check if the repository has 'lfs' (large file) content """ - - if ud.nobranch: - # If no branch is specified, use the current git commit - refname = ud.revision - elif wd == ud.clonedir: - # The bare clonedir doesn't use the remote names; it has the branch immediately. - refname = ud.branch - else: - refname = "origin/%s" % ud.branch - cmd = "%s grep lfs %s:.gitattributes | wc -l" % ( - ud.basecmd, refname) + ud.basecmd, ud.revision) try: output = runfetchcmd(cmd, d, quiet=True, workdir=wd) From patchwork Tue Apr 29 08:11:21 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Philip Lorenz X-Patchwork-Id: 62082 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 852CBC3ABAA for ; Tue, 29 Apr 2025 08:18:05 +0000 (UTC) Received: from esa2.hc324-48.eu.iphmx.com (esa2.hc324-48.eu.iphmx.com [207.54.68.120]) by mx.groups.io with SMTP id smtpd.web11.13074.1745914681685012812 for ; Tue, 29 Apr 2025 01:18:04 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bmw.de header.s=mailing1 header.b=Yw5JANqv; spf=pass (domain: bmw.de, ip: 207.54.68.120, mailfrom: prvs=207d80f5d=philip.lorenz@bmw.de) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bmw.de; i=@bmw.de; q=dns/txt; s=mailing1; t=1745914684; x=1777450684; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=9Hv77MH5/tIplL7alvTM3RgdwLfBF7nTTeonzXa++DE=; b=Yw5JANqvePv/MeeUoQh+FmbqKlfZEXBhvJOWJAlzTkoBKF4E6uetqeLJ +kgvGFgXzXusTzYhaIR+mrJcHqxCGTXUohifHyw2qqODu9TI5y4y0D5Ue AYMRu6gCEAmbort1VWglZydAKcGnttfCc19s1D3eetzOonWRgnGGtdWB7 w=; X-CSE-ConnectionGUID: J3Herv3aQBOlJq+WY/Y6Ng== X-CSE-MsgGUID: 6nNRz1IvSgu3veW5fHdz3g== Received: from esagw2.bmwgroup.com (HELO esagw2.muc) ([160.46.252.38]) by esa2.hc324-48.eu.iphmx.com with ESMTP/TLS; 29 Apr 2025 10:17:59 +0200 Received: from esabb4.muc ([160.50.100.33]) by esagw2.muc with ESMTP/TLS; 29 Apr 2025 10:17:58 +0200 Received: from smucmp19d.bmwgroup.net (HELO smucmp19d.europe.bmw.corp) ([10.30.13.170]) by esabb4.muc with ESMTP/TLS; 29 Apr 2025 10:17:58 +0200 Received: from localhost.localdomain (10.30.85.212) by smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) with Microsoft SMTP Server (version=TLS; Tue, 29 Apr 2025 10:17:57 +0200 X-CSE-ConnectionGUID: RjVLhOM/Q3WTzpPFOy/MWQ== X-CSE-MsgGUID: O4SecT03T6WKZedN9q2oNg== X-CSE-ConnectionGUID: TsJs5m1kTS2M4JofsJW2pw== X-CSE-MsgGUID: CEzNvmfCRrS38TDBnquh1A== From: Philip Lorenz To: CC: Philip Lorenz , , Subject: [PATCH v4 5/8] fetch2: Use git-lfs fetch to download objects Date: Tue, 29 Apr 2025 10:11:21 +0200 Message-ID: <20250429081124.3695505-6-philip.lorenz@bmw.de> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250429081124.3695505-1-philip.lorenz@bmw.de> References: <20250429081124.3695505-1-philip.lorenz@bmw.de> MIME-Version: 1.0 X-ClientProxiedBy: smucmp09b.europe.bmw.corp (2a03:1e80:a15:58f::203f) To smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Apr 2025 08:18:05 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/17587 Its not clear which exact git-lfs versions failed to work with bare repositories, however git-lfs 2.13.2 which is shipped by Debian 10 (i.e. the oldest supported distribution by scarthgap) shows no issue when fetching into a bare repository. Switch to git-lfs fetch which in turn eliminates issues seen when using the gitsm fetcher with submodules utilizing lfs. In these scenarios, fetching of LFS objects did not actually happen as the gitsm fetcher parametrizes the to be fetched repositories with `bareclone=1` which in turn means that the target revision was never checked out (and therefore no LFS objects were fetched). Signed-off-by: Philip Lorenz --- lib/bb/fetch2/git.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 9b2ad455c..11cda2007 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -482,30 +482,7 @@ class Git(FetchMethod): raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) if self.lfs_need_update(ud, d): - # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching - # of all LFS blobs needed at the srcrev. - # - # It would be nice to just do this inline here by running 'git-lfs fetch' - # on the bare clonedir, but that operation requires a working copy on some - # releases of Git LFS. - with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir: - # Do the checkout. This implicitly involves a Git LFS fetch. - Git.unpack(self, ud, tmpdir, d) - - # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into - # the bare clonedir. - # - # As this procedure is invoked repeatedly on incremental fetches as - # a recipe's SRCREV is bumped throughout its lifetime, this will - # result in a gradual accumulation of LFS blobs in /lfs - # corresponding to all the blobs reachable from the different revs - # fetched across time. - # - # Only do this if the unpack resulted in a .git/lfs directory being - # created; this only happens if at least one blob needed to be - # downloaded. - if os.path.exists(os.path.join(ud.destdir, ".git", "lfs")): - runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/.git" % ud.destdir) + self.lfs_fetch(ud, d, ud.clonedir, ud.revision) def lfs_fetch(self, ud, d, clonedir, revision, fetchall=False, progresshandler=None): """Helper method for fetching Git LFS data""" From patchwork Tue Apr 29 08:11:22 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Philip Lorenz X-Patchwork-Id: 62084 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 9F289C3ABAD for ; Tue, 29 Apr 2025 08:18:05 +0000 (UTC) Received: from esa2.hc324-48.eu.iphmx.com (esa2.hc324-48.eu.iphmx.com [207.54.68.120]) by mx.groups.io with SMTP id smtpd.web10.13165.1745914683459549615 for ; Tue, 29 Apr 2025 01:18:05 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bmw.de header.s=mailing1 header.b=CIrCAxKB; spf=pass (domain: bmw.de, ip: 207.54.68.120, mailfrom: prvs=207d80f5d=philip.lorenz@bmw.de) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bmw.de; i=@bmw.de; q=dns/txt; s=mailing1; t=1745914684; x=1777450684; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=L0jRWYKOpU/THuyRs8hJumOmCP40EsWRqNd5BOnPqtE=; b=CIrCAxKBwdM4Dv6r4L9hgih5fDSs2YiKL32Dapp9yRRxZABnoa1sa+nc hZcPogd6PkQxoAPVgef5BUtFMfeAJF+AzpfhqaylAOf7lz2PRlw7+GURP XAmXH/0ZCipX5hU8LiOaX/H0Tz2IGR2bEHPZFSE0jVQI2766nkXnAIPVS k=; X-CSE-ConnectionGUID: J3Herv3aQBOlJq+WY/Y6Ng== X-CSE-MsgGUID: yIxEu45xT7OL58PW78x9Bw== Received: from esagw2.bmwgroup.com (HELO esagw2.muc) ([160.46.252.38]) by esa2.hc324-48.eu.iphmx.com with ESMTP/TLS; 29 Apr 2025 10:17:59 +0200 Received: from esabb4.muc ([160.50.100.33]) by esagw2.muc with ESMTP/TLS; 29 Apr 2025 10:17:58 +0200 Received: from smucmp19d.bmwgroup.net (HELO smucmp19d.europe.bmw.corp) ([10.30.13.170]) by esabb4.muc with ESMTP/TLS; 29 Apr 2025 10:17:58 +0200 Received: from localhost.localdomain (10.30.85.212) by smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) with Microsoft SMTP Server (version=TLS; Tue, 29 Apr 2025 10:17:58 +0200 X-CSE-ConnectionGUID: PGmc/tVZSQm3HiPANj4C6w== X-CSE-MsgGUID: uWBd12Q2QRSag32aG0SKGQ== X-CSE-ConnectionGUID: JYpH1IKwRzyVHOloPYv8oQ== X-CSE-MsgGUID: N7l2ShYERGmPtBE1pD1KXQ== From: Philip Lorenz To: CC: Philip Lorenz , , Subject: [PATCH v4 6/8] fetch2: Fix incorrect lfs parametrization for submodules Date: Tue, 29 Apr 2025 10:11:22 +0200 Message-ID: <20250429081124.3695505-7-philip.lorenz@bmw.de> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250429081124.3695505-1-philip.lorenz@bmw.de> References: <20250429081124.3695505-1-philip.lorenz@bmw.de> MIME-Version: 1.0 X-ClientProxiedBy: smucmp09b.europe.bmw.corp (2a03:1e80:a15:58f::203f) To smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Apr 2025 08:18:05 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/17588 The existing code would pass `True` or `False` to the git fetcher. As the fetcher expects `lfs` to be set to `1` this always lead to LFS fetching being disabled. Signed-off-by: Philip Lorenz --- V1 -> V2: Fix invalid value being passed when LFS was disabled --- lib/bb/fetch2/gitsm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py index f514aedaf..c8bc54e6a 100644 --- a/lib/bb/fetch2/gitsm.py +++ b/lib/bb/fetch2/gitsm.py @@ -122,7 +122,7 @@ class GitSM(Git): url += ";name=%s" % module url += ";subpath=%s" % module url += ";nobranch=1" - url += ";lfs=%s" % self._need_lfs(ud) + url += ";lfs=%s" % ("1" if self._need_lfs(ud) else "0") # Note that adding "user=" here to give credentials to the # submodule is not supported. Since using SRC_URI to give git:// # URL a password is not supported, one have to use one of the From patchwork Tue Apr 29 08:11:23 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Philip Lorenz X-Patchwork-Id: 62083 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 847CEC3ABA9 for ; Tue, 29 Apr 2025 08:18:05 +0000 (UTC) Received: from esa8.hc324-48.eu.iphmx.com (esa8.hc324-48.eu.iphmx.com [207.54.65.242]) by mx.groups.io with SMTP id smtpd.web11.13072.1745914679924280858 for ; Tue, 29 Apr 2025 01:18:02 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bmw.de header.s=mailing1 header.b=aLfGFPs2; spf=pass (domain: bmw.de, ip: 207.54.65.242, mailfrom: prvs=207d80f5d=philip.lorenz@bmw.de) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bmw.de; i=@bmw.de; q=dns/txt; s=mailing1; t=1745914681; x=1777450681; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=TUD2uGxRiXTHBDBRgSiw+CpX3u4k4VXkgNfcoU769mM=; b=aLfGFPs2AkcPrsl/NeB7E+W8V2MeEvlNLF1SIoh/0FoaZGWiOFvMBiKt v/qN0mbSe+WPQ3t5euOUAA1DmbPPljJdG1sfFIOjtsgUI/1DZ74O1gqRc DRggWpIyLXWs1Rum7yxXyMOpSyfpRdi3Wiuy/vioiHZbRO26evVpvVsyZ Y=; X-CSE-ConnectionGUID: vzAxsUweRQuOBajErrDHOQ== X-CSE-MsgGUID: 0V10TYpwSzOhG/Yk5g91SQ== Received: from 160.46.252.34.spf.bmwgroup.com (HELO esagw1.muc) ([160.46.252.34]) by esa8.hc324-48.eu.iphmx.com with ESMTP/TLS; 29 Apr 2025 10:17:58 +0200 Received: from esabb3.muc ([160.50.100.30]) by esagw1.muc with ESMTP/TLS; 29 Apr 2025 10:17:58 +0200 Received: from smucmp19d.bmwgroup.net (HELO smucmp19d.europe.bmw.corp) ([10.30.13.170]) by esabb3.muc with ESMTP/TLS; 29 Apr 2025 10:17:58 +0200 Received: from localhost.localdomain (10.30.85.212) by smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) with Microsoft SMTP Server (version=TLS; Tue, 29 Apr 2025 10:17:58 +0200 X-CSE-ConnectionGUID: olDvVinmT/mL8wrDqj6IGw== X-CSE-MsgGUID: L8z8r7kZS+SSslFYIoweJw== X-CSE-ConnectionGUID: sBSa3eQeSyefcAqo+1ncig== X-CSE-MsgGUID: ksl7j2RfQtWd1F1V6FHgaQ== From: Philip Lorenz To: CC: Philip Lorenz , , Subject: [PATCH v4 7/8] fetch2: Fix LFS object checkout in submodules Date: Tue, 29 Apr 2025 10:11:23 +0200 Message-ID: <20250429081124.3695505-8-philip.lorenz@bmw.de> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250429081124.3695505-1-philip.lorenz@bmw.de> References: <20250429081124.3695505-1-philip.lorenz@bmw.de> MIME-Version: 1.0 X-ClientProxiedBy: smucmp09b.europe.bmw.corp (2a03:1e80:a15:58f::203f) To smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Apr 2025 08:18:05 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/17583 Skipping smudging prevents the LFS objects from replacing their placeholder files when `git submodule update` actually checks out the target revision in the submodule. Smudging cannot happen earlier as the clone stored in `.git/modules` is bare. This should be fine as long as all LFS objects are available in the download cache (which they are after the other fixes are applied). Signed-off-by: Philip Lorenz --- V1 -> V2: Continue to set GIT_LFS_SKIP_SMUDGE=1 when LFS is explicitly disabled --- lib/bb/fetch2/gitsm.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/bb/fetch2/gitsm.py b/lib/bb/fetch2/gitsm.py index c8bc54e6a..5869e1b99 100644 --- a/lib/bb/fetch2/gitsm.py +++ b/lib/bb/fetch2/gitsm.py @@ -245,12 +245,11 @@ class GitSM(Git): ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) if not ud.bareclone and ret: - # All submodules should already be downloaded and configured in the tree. This simply - # sets up the configuration and checks out the files. The main project config should - # remain unmodified, and no download from the internet should occur. As such, lfs smudge - # should also be skipped as these files were already smudged in the fetch stage if lfs - # was enabled. - runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) + cmdprefix = "" + # Avoid LFS smudging (replacing the LFS pointers with the actual content) when LFS shouldn't be used but git-lfs is installed. + if not self._need_lfs(ud): + cmdprefix = "GIT_LFS_SKIP_SMUDGE=1 " + runfetchcmd("%s%s submodule update --recursive --no-fetch" % (cmdprefix, ud.basecmd), d, quiet=True, workdir=ud.destdir) def clean(self, ud, d): def clean_submodule(ud, url, module, modpath, workdir, d): url += ";bareclone=1;nobranch=1" From patchwork Tue Apr 29 08:11:24 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Philip Lorenz X-Patchwork-Id: 62089 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id B405CC369DC for ; Tue, 29 Apr 2025 08:18:15 +0000 (UTC) Received: from esa2.hc324-48.eu.iphmx.com (esa2.hc324-48.eu.iphmx.com [207.54.68.120]) by mx.groups.io with SMTP id smtpd.web10.13167.1745914685315898404 for ; Tue, 29 Apr 2025 01:18:05 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bmw.de header.s=mailing1 header.b=NY574HRh; spf=pass (domain: bmw.de, ip: 207.54.68.120, mailfrom: prvs=207d80f5d=philip.lorenz@bmw.de) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bmw.de; i=@bmw.de; q=dns/txt; s=mailing1; t=1745914685; x=1777450685; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=kfOm7CmDHgPs5lhCh+sqPunuZsakXMSltY7UDs3SF4c=; b=NY574HRhZijqGDFrUR05lB4UEC2h8OnbDKS+tdWBlrn/FRjZAI/xJUbD NNYAjIgCyquQxnyLSA+bTMmuWfIDbGF7cUKGf5r1joL6BJsg8sxil5h09 agzokcBrxournqSCKcPJHuuqPaaLRizz/NM/PD1zbG5mCG2iTTU9Dm9gy I=; X-CSE-ConnectionGUID: pjNjmoYTSpW0jtEL712mxQ== X-CSE-MsgGUID: p9tcQL4MS5uUEohqAlVkZQ== Received: from 160.46.252.35.spf.bmwgroup.com.bmwgroup.com (HELO esagw3.muc) ([160.46.252.35]) by esa2.hc324-48.eu.iphmx.com with ESMTP/TLS; 29 Apr 2025 10:18:00 +0200 Received: from esabb2.muc ([160.50.100.34]) by esagw3.muc with ESMTP/TLS; 29 Apr 2025 10:18:00 +0200 Received: from smucmp19d.bmwgroup.net (HELO smucmp19d.europe.bmw.corp) ([10.30.13.170]) by esabb2.muc with ESMTP/TLS; 29 Apr 2025 10:17:59 +0200 Received: from localhost.localdomain (10.30.85.212) by smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) with Microsoft SMTP Server (version=TLS; Tue, 29 Apr 2025 10:17:58 +0200 X-CSE-ConnectionGUID: XateTI5LSVKCOu6tDDMyZQ== X-CSE-MsgGUID: 13lq00HJTf+3IqbMnGIOtg== X-CSE-ConnectionGUID: UwnmxN5iQca44QD4LGHGPA== X-CSE-MsgGUID: dKXEPOjkRDqy9DQsE7Re7A== From: Philip Lorenz To: CC: Philip Lorenz , , Subject: [PATCH v4 8/8] tests/fetch: Test gitsm with LFS Date: Tue, 29 Apr 2025 10:11:24 +0200 Message-ID: <20250429081124.3695505-9-philip.lorenz@bmw.de> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250429081124.3695505-1-philip.lorenz@bmw.de> References: <20250429081124.3695505-1-philip.lorenz@bmw.de> MIME-Version: 1.0 X-ClientProxiedBy: smucmp09b.europe.bmw.corp (2a03:1e80:a15:58f::203f) To smucmp19d.europe.bmw.corp (2a03:1e80:a15:58f::205d) List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Apr 2025 08:18:15 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/17590 Add a test case to verify that the gitsm fetcher properly handles repositories storing objects with LFS. The test case verifies that LFS objects are fetched on the initial clone but also ensures that consecutive updates extend the original clone with any newly referenced LFS objects. Signed-off-by: Philip Lorenz --- V1 -> V2: * Add testcase for lfs=0 * Fix missing LFS installation in submodules --- lib/bb/tests/fetch.py | 122 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 111 insertions(+), 11 deletions(-) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index c77725190..009172360 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -20,6 +20,7 @@ import tarfile from bb.fetch2 import URI from bb.fetch2 import FetchMethod import bb +import bb.utils from bb.tests.support.httpserver import HTTPService def skipIfNoNetwork(): @@ -27,6 +28,18 @@ def skipIfNoNetwork(): return unittest.skip("network test") return lambda f: f + +@contextlib.contextmanager +def hide_directory(directory): + """Hide the given directory and restore it after the context is left""" + temp_name = directory + ".bak" + os.rename(directory, temp_name) + try: + yield + finally: + os.rename(temp_name, directory) + + class TestTimeout(Exception): # Indicate to pytest that this is not a test suite __test__ = False @@ -2324,26 +2337,113 @@ class GitLfsTest(FetcherTest): unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg") return unpacked_lfs_file + @skipIfNoGitLFS() + def test_gitsm_lfs(self): + """Test that the gitsm fetcher caches objects stored via LFS""" + self.git(["lfs", "install", "--local"], cwd=self.srcdir) + + def fetch_and_verify(revision, filename, content): + self.d.setVar('SRCREV', revision) + fetcher, ud = self.fetch() + + with hide_directory(submoduledir), hide_directory(self.srcdir): + workdir = self.d.getVar('WORKDIR') + fetcher.unpack(workdir) + + with open(os.path.join(workdir, "git", filename)) as f: + self.assertEqual(f.read(), content) + + # Create the git repository that will later be used as a submodule + submoduledir = self.tempdir + "/submodule" + bb.utils.mkdirhier(submoduledir) + self.git_init(submoduledir) + self.git(["lfs", "install", "--local"], cwd=submoduledir) + self.commit_file('.gitattributes', '*.mp3 filter=lfs -text', cwd=submoduledir) + + submodule_commit_1 = self.commit_file("a.mp3", "submodule version 1", cwd=submoduledir) + _ = self.commit_file("a.mp3", "submodule version 2", cwd=submoduledir) + + # Add the submodule to the repository at its current HEAD revision + self.git(["-c", "protocol.file.allow=always", "submodule", "add", submoduledir, "submodule"], + cwd=self.srcdir) + base_commit_1 = self.commit() + + # Let the submodule point at a different revision + self.git(["checkout", submodule_commit_1], self.srcdir + "/submodule") + self.git(["add", "submodule"], cwd=self.srcdir) + base_commit_2 = self.commit() + + # Add a LFS file to the repository + base_commit_3 = self.commit_file("a.mp3", "version 1") + # Update the added LFS file + base_commit_4 = self.commit_file("a.mp3", "version 2") + + self.d.setVar('SRC_URI', "gitsm://%s;protocol=file;lfs=1;branch=master" % self.srcdir) + + # Verify that LFS objects referenced from submodules are fetched and checked out + fetch_and_verify(base_commit_1, "submodule/a.mp3", "submodule version 2") + # Verify that the repository inside the download cache of a submodile is extended with any + # additional LFS objects needed when checking out a different revision. + fetch_and_verify(base_commit_2, "submodule/a.mp3", "submodule version 1") + # Verify that LFS objects referenced from the base repository are fetched and checked out + fetch_and_verify(base_commit_3, "a.mp3", "version 1") + # Verify that the cached repository is extended with any additional LFS objects required + # when checking out a different revision. + fetch_and_verify(base_commit_4, "a.mp3", "version 2") + + @skipIfNoGitLFS() + def test_gitsm_lfs_disabled(self): + """Test that the gitsm fetcher does not use LFS when explicitly disabled""" + self.git(["lfs", "install", "--local"], cwd=self.srcdir) + + def fetch_and_verify(revision, filename, content): + self.d.setVar('SRCREV', revision) + fetcher, ud = self.fetch() + + with hide_directory(submoduledir), hide_directory(self.srcdir): + workdir = self.d.getVar('WORKDIR') + fetcher.unpack(workdir) + + with open(os.path.join(workdir, "git", filename)) as f: + # Assume that LFS did not perform smudging when the expected content is + # missing. + self.assertNotEqual(f.read(), content) + + # Create the git repository that will later be used as a submodule + submoduledir = self.tempdir + "/submodule" + bb.utils.mkdirhier(submoduledir) + self.git_init(submoduledir) + self.git(["lfs", "install", "--local"], cwd=submoduledir) + self.commit_file('.gitattributes', '*.mp3 filter=lfs -text', cwd=submoduledir) + + submodule_commit_1 = self.commit_file("a.mp3", "submodule version 1", cwd=submoduledir) + + # Add the submodule to the repository at its current HEAD revision + self.git(["-c", "protocol.file.allow=always", "submodule", "add", submoduledir, "submodule"], + cwd=self.srcdir) + base_commit_1 = self.commit() + + # Add a LFS file to the repository + base_commit_2 = self.commit_file("a.mp3", "version 1") + + self.d.setVar('SRC_URI', "gitsm://%s;protocol=file;lfs=1;branch=master;lfs=0" % self.srcdir) + + # Verify that LFS objects referenced from submodules are not fetched nor checked out + fetch_and_verify(base_commit_1, "submodule/a.mp3", "submodule version 1") + # Verify that the LFS objects referenced from the base repository are not fetched nor + # checked out + fetch_and_verify(base_commit_2, "a.mp3", "version 1") + @skipIfNoGitLFS() def test_fetch_lfs_on_srcrev_change(self): """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested""" self.git(["lfs", "install", "--local"], cwd=self.srcdir) - @contextlib.contextmanager - def hide_upstream_repository(): - """Hide the upstream repository to make sure that git lfs cannot pull from it""" - temp_name = self.srcdir + ".bak" - os.rename(self.srcdir, temp_name) - try: - yield - finally: - os.rename(temp_name, self.srcdir) - def fetch_and_verify(revision, filename, content): self.d.setVar('SRCREV', revision) fetcher, ud = self.fetch() - with hide_upstream_repository(): + with hide_directory(self.srcdir): workdir = self.d.getVar('WORKDIR') fetcher.unpack(workdir)