@@ -587,6 +587,11 @@ def mirror_from_string(data):
bb.warn('Invalid mirror data %s, should have paired members.' % data)
return list(zip(*[iter(mirrors)]*2))
+def downloaded_path(ud):
+ """Return the path obtained for a URL, including from a mirror."""
+ return getattr(ud, "mirrorpath", ud.localpath)
+
+
def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True):
"""
verify the MD5 and SHA256 checksum for downloaded src
@@ -1147,11 +1152,18 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
if hasattr(origud.method, "build_mirror_data"):
origud.method.build_mirror_data(origud, ld)
return origud.localpath
- # Otherwise the result is a local file:// and we symlink to it
- # This may also be a link to a shallow archive
- # When using shallow mode, add a symlink to the original fullshallow
- # path to ensure a valid symlink even in the `PREMIRRORS` case
- origud.method.update_mirror_links(ud, origud)
+ # Use origud to find the URL which started a mirror chain and record
+ # the successful result without changing an existing download.
+ mirrororigud = ud.origud
+ while hasattr(mirrororigud, "origud"):
+ mirrororigud = mirrororigud.origud
+ mirrororigud.mirrorpath = ud.localpath
+ if not os.path.lexists(mirrororigud.localpath):
+ bb.utils.mkdirhier(os.path.dirname(mirrororigud.localpath))
+ try:
+ os.symlink(ud.localpath, mirrororigud.localpath)
+ except FileExistsError:
+ pass
update_stamp(origud, ld)
return ud.localpath
@@ -1676,28 +1688,6 @@ class FetchMethod(object):
"""
bb.utils.remove(urldata.localpath)
- def ensure_symlink(self, target, link_name):
- if not os.path.exists(link_name):
- dirname = os.path.dirname(link_name)
- bb.utils.mkdirhier(dirname)
- if os.path.islink(link_name):
- # Broken symbolic link
- os.unlink(link_name)
-
- # In case this is executing without any file locks held (as is
- # the case for file:// URLs), two tasks may end up here at the
- # same time, in which case we do not want the second task to
- # fail when the link has already been created by the first task.
- try:
- os.symlink(target, link_name)
- except FileExistsError:
- pass
-
- def update_mirror_links(self, ud, origud):
- # For local file:// results, create a symlink to them
- # This may also be a link to a shallow archive
- self.ensure_symlink(ud.localpath, origud.localpath)
-
def try_premirror(self, urldata, d):
"""
Should premirrors be used?
@@ -1869,7 +1859,7 @@ class Fetch(object):
self.ud[url] = FetchData(url, self.d)
self.ud[url].setup_localpath(self.d)
- return self.ud[url].localpath
+ return downloaded_path(self.ud[url])
def localpaths(self):
"""
@@ -1880,7 +1870,7 @@ class Fetch(object):
for u in self.urls:
ud = self.ud[u]
ud.setup_localpath(self.d)
- local.append(ud.localpath)
+ local.append(downloaded_path(ud))
return local
@@ -77,6 +77,7 @@ import bb
import bb.progress
from contextlib import contextmanager
from bb.fetch import FetchMethod
+from bb.fetch import downloaded_path
from bb.fetch import runfetchcmd
from bb.fetch import logger
from bb.fetch import trusted_network
@@ -317,14 +318,15 @@ class Git(FetchMethod):
or self.tarball_need_update(ud) \
or self.lfs_need_update(ud, d)
- def clonedir_need_update(self, ud, d):
- if not os.path.exists(ud.clonedir):
+ def clonedir_need_update(self, ud, d, clonedir=None):
+ clonedir = clonedir or ud.clonedir
+ if not os.path.exists(clonedir):
return True
- if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
+ if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d, clonedir):
return True
- if not self._contains_ref(ud, d, ud.name, ud.clonedir):
+ if not self._contains_ref(ud, d, ud.name, clonedir):
return True
- if 'tag' in ud.parm and not self._contains_ref(ud, d, ud.name, ud.clonedir, tag=True):
+ if 'tag' in ud.parm and not self._contains_ref(ud, d, ud.name, clonedir, tag=True):
return True
return False
@@ -339,10 +341,11 @@ class Git(FetchMethod):
return True
return False
- def clonedir_need_shallow_revs(self, ud, d):
+ def clonedir_need_shallow_revs(self, ud, d, clonedir=None):
+ clonedir = clonedir or ud.clonedir
for rev in ud.shallow_revs:
try:
- runfetchcmd(ud.basecmd + ['rev-parse', '-q', '--verify', rev], d, quiet=True, workdir=ud.clonedir)
+ runfetchcmd(ud.basecmd + ['rev-parse', '-q', '--verify', rev], d, quiet=True, workdir=clonedir)
except bb.fetch.FetchError:
return rev
return None
@@ -353,13 +356,6 @@ class Git(FetchMethod):
def tarball_need_update(self, ud):
return ud.write_tarballs and not os.path.exists(ud.fullmirror)
- def update_mirror_links(self, ud, origud):
- super().update_mirror_links(ud, origud)
- # When using shallow mode, add a symlink to the original fullshallow
- # path to ensure a valid symlink even in the `PREMIRRORS` case
- if origud.shallow and not os.path.exists(origud.fullshallow):
- self.ensure_symlink(ud.localpath, origud.fullshallow)
-
def try_premirror(self, ud, d):
# If we don't do this, updating an existing checkout with only premirrors
# is not possible
@@ -695,28 +691,36 @@ class Git(FetchMethod):
source_found = False
update_mode = False
source_error = []
-
- clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
+ clonedir = ud.clonedir
+ fullshallow = getattr(ud, "fullshallow", None)
+ if hasattr(ud, "mirrorpath"):
+ mirrorpath = downloaded_path(ud)
+ if os.path.isdir(mirrorpath):
+ clonedir = mirrorpath
+ elif ud.shallow:
+ fullshallow = mirrorpath
+
+ clonedir_is_up_to_date = not self.clonedir_need_update(ud, d, clonedir)
if clonedir_is_up_to_date:
if update and os.path.exists(destdir):
update_mode = True
else:
- runfetchcmd(ud.basecmd + ['clone'] + ud.cloneflags + [ud.clonedir, destdir], d, extraenv=extraenv)
+ runfetchcmd(ud.basecmd + ['clone'] + ud.cloneflags + [clonedir, destdir], d, extraenv=extraenv)
source_found = True
else:
- source_error.append("clone directory not available or not up to date: " + ud.clonedir)
+ source_error.append("clone directory not available or not up to date: " + clonedir)
if not source_found:
if ud.shallow:
- if os.path.exists(ud.fullshallow):
+ if os.path.exists(fullshallow):
if update and os.path.exists(destdir):
update_mode = True
else:
bb.utils.mkdirhier(destdir)
- runfetchcmd(['tar', '-xzf', ud.fullshallow], d, workdir=destdir)
+ runfetchcmd(['tar', '-xzf', fullshallow], d, workdir=destdir)
source_found = True
else:
- source_error.append("shallow clone not available: " + ud.fullshallow)
+ source_error.append("shallow clone not available: " + fullshallow)
else:
source_error.append("shallow clone not enabled")
@@ -735,8 +739,8 @@ class Git(FetchMethod):
try:
runfetchcmd(ud.basecmd + ['remote', 'get-url', 'dldir'], d, workdir=destdir)
except bb.fetch.FetchError:
- if ud.clonedir:
- runfetchcmd(ud.basecmd + ['remote', 'add', 'dldir', 'file://' + ud.clonedir], d, workdir=destdir)
+ if clonedir:
+ runfetchcmd(ud.basecmd + ['remote', 'add', 'dldir', 'file://' + clonedir], d, workdir=destdir)
try:
runfetchcmd(ud.basecmd + ['fetch', 'dldir'], d, workdir=destdir, extraenv=extraenv)
except bb.fetch.FetchError as e:
@@ -767,11 +771,11 @@ class Git(FetchMethod):
repourl = self._get_repo_url(ud)
runfetchcmd(ud.basecmd + ['remote', 'set-url', 'origin', repourl], d, workdir=destdir)
- if ud.clonedir:
+ if clonedir:
try:
runfetchcmd(ud.basecmd + ['remote', 'get-url', 'dldir'], d, workdir=destdir)
except bb.fetch.FetchError:
- runfetchcmd(ud.basecmd + ['remote', 'add', 'dldir', "file://" + ud.clonedir], d, workdir=destdir)
+ runfetchcmd(ud.basecmd + ['remote', 'add', 'dldir', "file://" + clonedir], d, workdir=destdir)
if self._contains_lfs(ud, d, destdir):
if not need_lfs:
@@ -802,12 +806,6 @@ class Git(FetchMethod):
""" clean the git directory """
to_remove = [ud.localpath, ud.fullmirror, ud.fullmirror + ".done"]
- # The localpath is a symlink to clonedir when it is cloned from a
- # mirror, so remove both of them.
- if os.path.islink(ud.localpath):
- clonedir = os.path.realpath(ud.localpath)
- to_remove.append(clonedir)
-
# Remove shallow mirror tarball
if ud.shallow:
to_remove.append(ud.fullshallow)
@@ -616,6 +616,55 @@ class GitDownloadDirectoryNamingTest(FetcherTest):
self.assertIn(self.recipe_dir, dir)
+class GitPremirrorStaleCloneTest(FetcherTest):
+
+ def test_premirror_ignores_stale_clone(self):
+ source = os.path.join(self.tempdir, "source")
+ os.mkdir(source)
+ self.git_init(cwd=source)
+
+ testfile = os.path.join(source, "testfile")
+ with open(testfile, "w") as f:
+ f.write("first\n")
+ self.git(["add", "testfile"], cwd=source)
+ self.git(["commit", "-m", "first commit"], cwd=source)
+ stale_revision = self.git(["rev-parse", "HEAD"], cwd=source).strip()
+
+ recipe_url = "git://upstream.invalid/project.git;branch=master;protocol=https"
+ self.d.setVar("SRCREV", stale_revision)
+ fetcher = bb.fetch.Fetch([recipe_url], self.d)
+ ud = fetcher.ud[recipe_url]
+ ud.setup_localpath(self.d)
+ os.makedirs(os.path.dirname(ud.clonedir), exist_ok=True)
+ self.git(["clone", "--bare", "--mirror", source, ud.clonedir], cwd=self.tempdir)
+ stale_clone = ud.clonedir
+
+ with open(testfile, "a") as f:
+ f.write("second\n")
+ self.git(["commit", "-am", "second commit"], cwd=source)
+ revision = self.git(["rev-parse", "HEAD"], cwd=source).strip()
+
+ self.d.setVar("SRCREV", revision)
+ self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
+ self.d.setVar("BB_NO_NETWORK", "1")
+ self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipe_url, source))
+
+ fetcher = bb.fetch.Fetch([recipe_url], self.d)
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+
+ self.assertFalse(os.path.islink(stale_clone))
+ self.assertEqual(stale_revision,
+ self.git(["rev-parse", "HEAD"], cwd=stale_clone).strip())
+ unpack_revision = self.git(["rev-parse", "HEAD"],
+ cwd=os.path.join(self.unpackdir, "git")).strip()
+ self.assertEqual(revision, unpack_revision)
+
+ mirrorpath = fetcher.ud[recipe_url].mirrorpath
+ fetcher.clean()
+ self.assertTrue(os.path.exists(mirrorpath))
+
+
class TarballNamingTest(FetcherTest):
def setUp(self):
super(TarballNamingTest, self).setUp()
The failure happens as follows: 1. BitBake has an old Git clone for the original URL. 2. That clone does not contain the commit being built. 3. BitBake successfully fetches the commit from a premirror into another clone. 4. BitBake tries to unpack from the old clone and fails. Remember the clone used by the successful premirror fetch and unpack from it. Leave both clones unchanged and preserve the premirror clone during clean. Add a test for this case. Signed-off-by: Fredrik Svensson <fredriks@meraki.com> --- Thanks for the feedback. This version leaves the stale clone untouched. It remembers the clone used by the successful premirror fetch and uses that clone for unpacking. It also removes update_mirror_links. A symlink is still created when the original clone path is absent, but clean no longer follows that symlink, so it does not remove the premirror clone. The updated self-test verifies that unpack uses the premirror revision, the old clone is unchanged, and clean preserves the premirror clone. Changes in v3: - do not delete or replace the stale clone; - use origud to remember the successful premirror path; - remove update_mirror_links; - preserve the premirror clone during clean; and - update the self-test for the new behavior. lib/bb/fetch/__init__.py | 48 +++++++++++++------------------- lib/bb/fetch/git.py | 60 +++++++++++++++++++--------------------- lib/bb/tests/fetch.py | 49 ++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 60 deletions(-)