@@ -133,6 +133,12 @@ class NonLocalMethod(Exception):
def __init__(self):
Exception.__init__(self)
+class MirrorException(BBFetchException):
+ """Exception raises when BB_FETCH_PREMIRRORONLY was specified but mirrors don't have SRCREV"""
+ def __init__(self, message):
+ BBFetchException.__init__(self, message)
+
+
class MissingChecksumEvent(bb.event.Event):
def __init__(self, url, **checksums):
self.url = url
@@ -1066,6 +1072,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
rename_bad_checksum(ud, e.checksum)
elif isinstance(e, NoChecksumError):
raise
+ elif isinstance(e, MirrorException):
+ logger.warning(str(e))
else:
logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url))
logger.debug(str(e))
@@ -1729,6 +1737,8 @@ class Fetch(object):
done = False
if premirroronly:
+ if not done:
+ raise MirrorException("Failed to download premirror {} and BB_FETCH_PREMIRRORONLY has been set to 1.".format(self.d.getVar("PREMIRRORS")))
self.d.setVar("BB_NO_NETWORK", "1")
firsterr = None
@@ -335,11 +335,7 @@ class Git(FetchMethod):
def try_premirror(self, ud, d):
# If we don't do this, updating an existing checkout with only premirrors
# is not possible
- if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
- return True
- if os.path.exists(ud.clonedir):
- return False
- return True
+ return bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")) or not os.path.exists(ud.clonedir)
def download(self, ud, d):
"""Fetch url"""
@@ -350,10 +346,14 @@ class Git(FetchMethod):
if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
ud.localpath = ud.fullshallow
return
- elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir):
- bb.utils.mkdirhier(ud.clonedir)
- runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
-
+ if os.path.exists(ud.fullmirror):
+ if self.try_premirror(ud, d) and self.need_update(ud, d):
+ bb.utils.mkdirhier(ud.clonedir)
+ runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
+ if self.need_update(ud, d) and \
+ bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
+ raise bb.fetch2.MirrorException("Premirror doesn't contain revisions {} and "\
+ "upstream cannot be used due to BB_FETCH_PREMIRRORONLY setting".format(ud.revisions))
repourl = self._get_repo_url(ud)
# If the repo still doesn't exist, fallback to cloning it
@@ -2776,3 +2776,73 @@ class GitSharedTest(FetcherTest):
fetcher.unpack(self.unpackdir)
alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
self.assertFalse(os.path.exists(alt))
+
+
+class FetchPremirroronlyLocalTest(FetcherTest):
+
+ def git(self, cmd, cwd=None):
+ if isinstance(cmd, str):
+ cmd = 'git ' + cmd
+ else:
+ cmd = ['git'] + cmd
+ if cwd is None:
+ cwd = self.gitdir
+ return bb.process.run(cmd, cwd=cwd)[0]
+
+ def setUp(self):
+ super(FetchPremirroronlyLocalTest, self).setUp()
+ self.mirrordir = os.path.join(self.tempdir, "mirrors")
+ os.mkdir(self.mirrordir)
+ self.reponame = "bitbake"
+ self.gitdir = os.path.join(self.tempdir, "git", self.reponame)
+ self.recipe_url = "git://git.fake.repo/bitbake"
+ self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
+ self.d.setVar("BB_NO_NETWORK", "1")
+ self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
+
+ def make_git_repo(self):
+ self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
+ recipeurl = "git:/git.fake.repo/bitbake"
+ os.makedirs(self.gitdir)
+ self.git("init", self.gitdir)
+ for i in range(0):
+ self.git_new_commit()
+ bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir)
+
+ def git_new_commit(self):
+ import random
+ testfilename = "bibake-fetch.test"
+ os.unlink(os.path.join(self.mirrordir, self.mirrorname))
+ with open(os.path.join(self.gitdir, testfilename), "w") as testfile:
+ testfile.write("Useless random data {}".format(random.random()))
+ self.git("add {}".format(testfilename), self.gitdir)
+ self.git("commit -a -m \"This random commit {}. I'm useless.\"".format(random.random()), self.gitdir)
+ bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir)
+ return self.git("rev-parse HEAD", self.gitdir).strip()
+
+ def test_mirror_commit_nonexistent(self):
+ self.make_git_repo()
+ self.d.setVar("SRCREV", "0"*40)
+ fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+ if hasattr(bb.fetch2, "MirrorException"):
+ with self.assertRaises(bb.fetch2.MirrorException):
+ fetcher.download()
+ else:
+ fetcher.download()
+
+ def test_mirror_commit_exists(self):
+ self.make_git_repo()
+ self.d.setVar("SRCREV", self.git_new_commit())
+ fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+
+ def test_mirror_tarball_nonexistent(self):
+ self.d.setVar("SRCREV", "0"*40)
+ fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+ if hasattr(bb.fetch2, "MirrorException"):
+ with self.assertRaises(bb.fetch2.MirrorException):
+ fetcher.download()
+ else:
+ fetcher.download()
+
This should fix [Yocto 13353] and related to [Yocto 13233] as well. Previously if git repo mirror has been updated in between of two builds fetcher of the second build didn't try updated mirror but switched to git clone from upstream instead. This is problem for offline builds. Fix this to raise MirrorException if BB_FETCH_PREMIRRORONLY has been specified by the mirror doesn't contain SRC_REV. Note if SRC_REV is nonSHA1 revision when it will fail without the network due to `ls-remote`. Signed-off-by: Pavel Zhukov <pavel.zhukov@huawei.com> --- lib/bb/fetch2/__init__.py | 10 ++++++ lib/bb/fetch2/git.py | 18 +++++----- lib/bb/tests/fetch.py | 70 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 9 deletions(-)