diff --git a/lib/bb/fetch/__init__.py b/lib/bb/fetch/__init__.py index 55ab710f..4d5e7b0d 100644 --- a/lib/bb/fetch/__init__.py +++ b/lib/bb/fetch/__init__.py @@ -587,6 +587,11 @@ def mirror_from_string(data): bb.warn('Invalid mirror data %s, should have paired members.' % data) return list(zip(*[iter(mirrors)]*2)) +def downloaded_path(ud): + """Return the path obtained for a URL, including from a mirror.""" + return getattr(ud, "mirrorpath", ud.localpath) + + def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True): """ verify the MD5 and SHA256 checksum for downloaded src @@ -1147,11 +1152,18 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): if hasattr(origud.method, "build_mirror_data"): origud.method.build_mirror_data(origud, ld) return origud.localpath - # Otherwise the result is a local file:// and we symlink to it - # This may also be a link to a shallow archive - # When using shallow mode, add a symlink to the original fullshallow - # path to ensure a valid symlink even in the `PREMIRRORS` case - origud.method.update_mirror_links(ud, origud) + # Use origud to find the URL which started a mirror chain and record + # the successful result without changing an existing download. + mirrororigud = ud.origud + while hasattr(mirrororigud, "origud"): + mirrororigud = mirrororigud.origud + mirrororigud.mirrorpath = ud.localpath + if not os.path.lexists(mirrororigud.localpath): + bb.utils.mkdirhier(os.path.dirname(mirrororigud.localpath)) + try: + os.symlink(ud.localpath, mirrororigud.localpath) + except FileExistsError: + pass update_stamp(origud, ld) return ud.localpath @@ -1676,28 +1688,6 @@ class FetchMethod(object): """ bb.utils.remove(urldata.localpath) - def ensure_symlink(self, target, link_name): - if not os.path.exists(link_name): - dirname = os.path.dirname(link_name) - bb.utils.mkdirhier(dirname) - if os.path.islink(link_name): - # Broken symbolic link - os.unlink(link_name) - - # In case this is executing without any file locks held (as is - # the case for file:// URLs), two tasks may end up here at the - # same time, in which case we do not want the second task to - # fail when the link has already been created by the first task. - try: - os.symlink(target, link_name) - except FileExistsError: - pass - - def update_mirror_links(self, ud, origud): - # For local file:// results, create a symlink to them - # This may also be a link to a shallow archive - self.ensure_symlink(ud.localpath, origud.localpath) - def try_premirror(self, urldata, d): """ Should premirrors be used? @@ -1869,7 +1859,7 @@ class Fetch(object): self.ud[url] = FetchData(url, self.d) self.ud[url].setup_localpath(self.d) - return self.ud[url].localpath + return downloaded_path(self.ud[url]) def localpaths(self): """ @@ -1880,7 +1870,7 @@ class Fetch(object): for u in self.urls: ud = self.ud[u] ud.setup_localpath(self.d) - local.append(ud.localpath) + local.append(downloaded_path(ud)) return local diff --git a/lib/bb/fetch/git.py b/lib/bb/fetch/git.py index dc1b31b5..836def49 100644 --- a/lib/bb/fetch/git.py +++ b/lib/bb/fetch/git.py @@ -77,6 +77,7 @@ import bb import bb.progress from contextlib import contextmanager from bb.fetch import FetchMethod +from bb.fetch import downloaded_path from bb.fetch import runfetchcmd from bb.fetch import logger from bb.fetch import trusted_network @@ -317,14 +318,15 @@ class Git(FetchMethod): or self.tarball_need_update(ud) \ or self.lfs_need_update(ud, d) - def clonedir_need_update(self, ud, d): - if not os.path.exists(ud.clonedir): + def clonedir_need_update(self, ud, d, clonedir=None): + clonedir = clonedir or ud.clonedir + if not os.path.exists(clonedir): return True - if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d): + if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d, clonedir): return True - if not self._contains_ref(ud, d, ud.name, ud.clonedir): + if not self._contains_ref(ud, d, ud.name, clonedir): return True - if 'tag' in ud.parm and not self._contains_ref(ud, d, ud.name, ud.clonedir, tag=True): + if 'tag' in ud.parm and not self._contains_ref(ud, d, ud.name, clonedir, tag=True): return True return False @@ -339,10 +341,11 @@ class Git(FetchMethod): return True return False - def clonedir_need_shallow_revs(self, ud, d): + def clonedir_need_shallow_revs(self, ud, d, clonedir=None): + clonedir = clonedir or ud.clonedir for rev in ud.shallow_revs: try: - runfetchcmd(ud.basecmd + ['rev-parse', '-q', '--verify', rev], d, quiet=True, workdir=ud.clonedir) + runfetchcmd(ud.basecmd + ['rev-parse', '-q', '--verify', rev], d, quiet=True, workdir=clonedir) except bb.fetch.FetchError: return rev return None @@ -353,13 +356,6 @@ class Git(FetchMethod): def tarball_need_update(self, ud): return ud.write_tarballs and not os.path.exists(ud.fullmirror) - def update_mirror_links(self, ud, origud): - super().update_mirror_links(ud, origud) - # When using shallow mode, add a symlink to the original fullshallow - # path to ensure a valid symlink even in the `PREMIRRORS` case - if origud.shallow and not os.path.exists(origud.fullshallow): - self.ensure_symlink(ud.localpath, origud.fullshallow) - def try_premirror(self, ud, d): # If we don't do this, updating an existing checkout with only premirrors # is not possible @@ -695,28 +691,36 @@ class Git(FetchMethod): source_found = False update_mode = False source_error = [] - - clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) + clonedir = ud.clonedir + fullshallow = getattr(ud, "fullshallow", None) + if hasattr(ud, "mirrorpath"): + mirrorpath = downloaded_path(ud) + if os.path.isdir(mirrorpath): + clonedir = mirrorpath + elif ud.shallow: + fullshallow = mirrorpath + + clonedir_is_up_to_date = not self.clonedir_need_update(ud, d, clonedir) if clonedir_is_up_to_date: if update and os.path.exists(destdir): update_mode = True else: - runfetchcmd(ud.basecmd + ['clone'] + ud.cloneflags + [ud.clonedir, destdir], d, extraenv=extraenv) + runfetchcmd(ud.basecmd + ['clone'] + ud.cloneflags + [clonedir, destdir], d, extraenv=extraenv) source_found = True else: - source_error.append("clone directory not available or not up to date: " + ud.clonedir) + source_error.append("clone directory not available or not up to date: " + clonedir) if not source_found: if ud.shallow: - if os.path.exists(ud.fullshallow): + if os.path.exists(fullshallow): if update and os.path.exists(destdir): update_mode = True else: bb.utils.mkdirhier(destdir) - runfetchcmd(['tar', '-xzf', ud.fullshallow], d, workdir=destdir) + runfetchcmd(['tar', '-xzf', fullshallow], d, workdir=destdir) source_found = True else: - source_error.append("shallow clone not available: " + ud.fullshallow) + source_error.append("shallow clone not available: " + fullshallow) else: source_error.append("shallow clone not enabled") @@ -735,8 +739,8 @@ class Git(FetchMethod): try: runfetchcmd(ud.basecmd + ['remote', 'get-url', 'dldir'], d, workdir=destdir) except bb.fetch.FetchError: - if ud.clonedir: - runfetchcmd(ud.basecmd + ['remote', 'add', 'dldir', 'file://' + ud.clonedir], d, workdir=destdir) + if clonedir: + runfetchcmd(ud.basecmd + ['remote', 'add', 'dldir', 'file://' + clonedir], d, workdir=destdir) try: runfetchcmd(ud.basecmd + ['fetch', 'dldir'], d, workdir=destdir, extraenv=extraenv) except bb.fetch.FetchError as e: @@ -767,11 +771,11 @@ class Git(FetchMethod): repourl = self._get_repo_url(ud) runfetchcmd(ud.basecmd + ['remote', 'set-url', 'origin', repourl], d, workdir=destdir) - if ud.clonedir: + if clonedir: try: runfetchcmd(ud.basecmd + ['remote', 'get-url', 'dldir'], d, workdir=destdir) except bb.fetch.FetchError: - runfetchcmd(ud.basecmd + ['remote', 'add', 'dldir', "file://" + ud.clonedir], d, workdir=destdir) + runfetchcmd(ud.basecmd + ['remote', 'add', 'dldir', "file://" + clonedir], d, workdir=destdir) if self._contains_lfs(ud, d, destdir): if not need_lfs: @@ -802,12 +806,6 @@ class Git(FetchMethod): """ clean the git directory """ to_remove = [ud.localpath, ud.fullmirror, ud.fullmirror + ".done"] - # The localpath is a symlink to clonedir when it is cloned from a - # mirror, so remove both of them. - if os.path.islink(ud.localpath): - clonedir = os.path.realpath(ud.localpath) - to_remove.append(clonedir) - # Remove shallow mirror tarball if ud.shallow: to_remove.append(ud.fullshallow) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 71b0a63f..518d219b 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -616,6 +616,55 @@ class GitDownloadDirectoryNamingTest(FetcherTest): self.assertIn(self.recipe_dir, dir) +class GitPremirrorStaleCloneTest(FetcherTest): + + def test_premirror_ignores_stale_clone(self): + source = os.path.join(self.tempdir, "source") + os.mkdir(source) + self.git_init(cwd=source) + + testfile = os.path.join(source, "testfile") + with open(testfile, "w") as f: + f.write("first\n") + self.git(["add", "testfile"], cwd=source) + self.git(["commit", "-m", "first commit"], cwd=source) + stale_revision = self.git(["rev-parse", "HEAD"], cwd=source).strip() + + recipe_url = "git://upstream.invalid/project.git;branch=master;protocol=https" + self.d.setVar("SRCREV", stale_revision) + fetcher = bb.fetch.Fetch([recipe_url], self.d) + ud = fetcher.ud[recipe_url] + ud.setup_localpath(self.d) + os.makedirs(os.path.dirname(ud.clonedir), exist_ok=True) + self.git(["clone", "--bare", "--mirror", source, ud.clonedir], cwd=self.tempdir) + stale_clone = ud.clonedir + + with open(testfile, "a") as f: + f.write("second\n") + self.git(["commit", "-am", "second commit"], cwd=source) + revision = self.git(["rev-parse", "HEAD"], cwd=source).strip() + + self.d.setVar("SRCREV", revision) + self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") + self.d.setVar("BB_NO_NETWORK", "1") + self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipe_url, source)) + + fetcher = bb.fetch.Fetch([recipe_url], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + self.assertFalse(os.path.islink(stale_clone)) + self.assertEqual(stale_revision, + self.git(["rev-parse", "HEAD"], cwd=stale_clone).strip()) + unpack_revision = self.git(["rev-parse", "HEAD"], + cwd=os.path.join(self.unpackdir, "git")).strip() + self.assertEqual(revision, unpack_revision) + + mirrorpath = fetcher.ud[recipe_url].mirrorpath + fetcher.clean() + self.assertTrue(os.path.exists(mirrorpath)) + + class TarballNamingTest(FetcherTest): def setUp(self): super(TarballNamingTest, self).setUp()