diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 2a3c06fe..d8aa3bf1 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -310,7 +310,10 @@ class Git(FetchMethod): return ud.clonedir def need_update(self, ud, d): - return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud) + return self.clonedir_need_update(ud, d) \ + or self.shallow_tarball_need_update(ud) \ + or self.tarball_need_update(ud) \ + or self.lfs_need_update(ud, d) def clonedir_need_update(self, ud, d): if not os.path.exists(ud.clonedir): @@ -322,6 +325,15 @@ class Git(FetchMethod): return True return False + def lfs_need_update(self, ud, d): + if self.clonedir_need_update(ud, d): + return True + + for name in ud.names: + if not self._lfs_objects_downloaded(ud, d, name, ud.clonedir): + return True + return False + def clonedir_need_shallow_revs(self, ud, d): for rev in ud.shallow_revs: try: @@ -414,7 +426,7 @@ class Git(FetchMethod): if missing_rev: raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) - if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): + if self.lfs_need_update(ud, d): # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching # of all LFS blobs needed at the srcrev. # @@ -653,6 +665,35 @@ class Git(FetchMethod): raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) return output.split()[0] != "0" + def _lfs_objects_downloaded(self, ud, d, name, wd): + """ + Verifies whether the LFS objects for requested revisions have already been downloaded + """ + # Bail out early if this repository doesn't use LFS + if not self._need_lfs(ud) or not self._contains_lfs(ud, d, wd): + return True + + # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file + # existence. + # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git + cmd = "%s lfs ls-files -l %s" \ + % (ud.basecmd, ud.revisions[name]) + output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip() + # Do not do any further matching if no objects are managed by LFS + if not output: + return True + + # Match all lines beginning with the hexadecimal OID + oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)") + for line in output.split("\n"): + oid = re.search(oid_regex, line) + if not oid: + bb.warn("git lfs ls-files output '%s' did not match expected format." % line) + if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))): + return False + + return True + def _need_lfs(self, ud): return ud.parm.get("lfs", "1") == "1" diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 6ef0836f..02234ca3 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -6,6 +6,7 @@ # SPDX-License-Identifier: GPL-2.0-only # +import contextlib import unittest import hashlib import tempfile @@ -2222,10 +2223,14 @@ class GitLfsTest(FetcherTest): bb.utils.mkdirhier(self.srcdir) self.git_init(cwd=self.srcdir) - with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs: - attrs.write('*.mp3 filter=lfs -text') - self.git(['add', '.gitattributes'], cwd=self.srcdir) - self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir) + self.commit_file('.gitattributes', '*.mp3 filter=lfs -text') + + def commit_file(self, filename, content): + with open(os.path.join(self.srcdir, filename), "w") as f: + f.write(content) + self.git(["add", filename], cwd=self.srcdir) + self.git(["commit", "-m", "Change"], cwd=self.srcdir) + return self.git(["rev-parse", "HEAD"], cwd=self.srcdir).strip() def fetch(self, uri=None, download=True): uris = self.d.getVar('SRC_URI').split() @@ -2245,6 +2250,44 @@ class GitLfsTest(FetcherTest): unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg") return unpacked_lfs_file + @skipIfNoGitLFS() + def test_fetch_lfs_on_srcrev_change(self): + """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested""" + self.git(["lfs", "install", "--local"], cwd=self.srcdir) + + @contextlib.contextmanager + def hide_upstream_repository(): + """Hide the upstream repository to make sure that git lfs cannot pull from it""" + temp_name = self.srcdir + ".bak" + os.rename(self.srcdir, temp_name) + try: + yield + finally: + os.rename(temp_name, self.srcdir) + + def fetch_and_verify(revision, filename, content): + self.d.setVar('SRCREV', revision) + fetcher, ud = self.fetch() + + with hide_upstream_repository(): + workdir = self.d.getVar('WORKDIR') + fetcher.unpack(workdir) + + with open(os.path.join(workdir, "git", filename)) as f: + self.assertEqual(f.read(), content) + + commit_1 = self.commit_file("a.mp3", "version 1") + commit_2 = self.commit_file("a.mp3", "version 2") + + self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir) + + # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are + # available even when the upstream repository disappears. + fetch_and_verify(commit_2, "a.mp3", "version 2") + # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download + # folder. + fetch_and_verify(commit_1, "a.mp3", "version 1") + @skipIfNoGitLFS() @skipIfNoNetwork() def test_real_git_lfs_repo_succeeds_without_lfs_param(self):