@@ -118,10 +118,15 @@ class Git(FetchMethod):
bitbake_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.join(os.path.abspath(__file__))), '..', '..', '..'))
make_shallow_path = os.path.join(bitbake_dir, 'bin', 'git-make-shallow')
+ sha1_pattern = re.compile(r'\b[0-9a-fA-F]{40}\b')
+
"""Class to fetch a module or modules from git repositories"""
def init(self, d):
pass
+ def _is_sha1(self, name):
+ return len(name) == 40 and re.match(self.sha1_pattern, name)
+
def supports(self, ud, d):
"""
Check to see if a given url can be fetched with git.
@@ -135,6 +140,8 @@ class Git(FetchMethod):
"""
init git specific variable within url data
so that the git method like latest_revision() can work
+ NOTE: This function may be called from do_unpack and in that case network is not available
+ avoid using network operations here or guard them properly.
"""
if 'protocol' in ud.parm:
ud.proto = ud.parm['protocol']
@@ -248,28 +255,33 @@ class Git(FetchMethod):
ud.setup_revisions(d)
- for name in ud.names:
- # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
- if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
- if ud.revisions[name]:
- ud.unresolvedrev[name] = ud.revisions[name]
- ud.revisions[name] = self.latest_revision(ud, d, name)
-
gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_'))
if gitsrcname.startswith('.'):
gitsrcname = gitsrcname[1:]
- # for rebaseable git repo, it is necessary to keep mirror tar ball
- # per revision, so that even the revision disappears from the
- # upstream repo in the future, the mirror will remain intact and still
- # contains the revision
- if ud.rebaseable:
- for name in ud.names:
- gitsrcname = gitsrcname + '_' + ud.revisions[name]
dl_dir = d.getVar("DL_DIR")
gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
+
+
+ ## We need clonedir to be defined before latest_revision call for unpack routine to use rev-parse instead of ls-remote
ud.clonedir = os.path.join(gitdir, gitsrcname)
+
+ for name in ud.names:
+ # Ensure anything that doesn't look like a sha1 checksum/revision is translated into one
+ if not ud.revisions[name] or not self._is_sha1(ud.revisions[name]):
+ if ud.revisions[name]:
+ ud.unresolvedrev[name] = ud.revisions[name]
+ ud.revisions[name] = self.latest_revision(ud, d, name)
+ if ud.rebaseable:
+ # for rebaseable git repo, it is necessary to keep mirror tar ball
+ # per revision, so that even the revision disappears from the
+ # upstream repo in the future, the mirror will remain intact and still
+ # contains the revision
+ gitsrcname = gitsrcname + '_' + ud.revisions[name]
+ if ud.rebaseable:
+ ud.clonedir = os.path.join(gitdir, gitsrcname)
+
ud.localfile = ud.clonedir
mirrortarball = 'git2_%s.tar.gz' % gitsrcname
@@ -697,6 +709,14 @@ class Git(FetchMethod):
slash_re = re.compile(r"/+")
return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
+ def _revparse(self, ud, d, rev):
+ """
+ Return latest revision (SHA) for the refs specified by branch/tag/etc
+ """
+ cmd = '{basecmd} rev-parse -q --verify {rev}'.format(basecmd = ud.basecmd, rev = rev)
+ output = runfetchcmd(cmd, d, quiet=True, workdir = ud.clonedir)
+ return output.strip()
+
def _lsremote(self, ud, d, search):
"""
Run git ls-remote with the specified search string
@@ -727,6 +747,20 @@ class Git(FetchMethod):
"""
Compute the HEAD revision for the url
"""
+
+ ## Network is not availble and we missed the cache. Try rev-parse
+ if d.getVar("BB_NO_NETWORK"):
+ if ud.rebaseable:
+ raise bb.fetch.UnpackError("Unable to map revision \"{name}\" to SHA1 revision without a network. " \
+ "SHA1 revision must be specified for rebaseable repos "\
+ "or \"BB_SRCREV_POLICY\" set to \"cache\". ".format(name=ud.unresolvedrev[name]),ud.host+ud.path)
+ rev = self._revparse(ud, d, ud.unresolvedrev[name])
+ if rev is None:
+ raise bb.fetch2.FetchError("Unable to resolve '%s' in local clone of repository in git rev-parse output for %s" % \
+ (ud.unresolvedrev[name], ud.host+ud.path))
+ return rev
+
+
output = self._lsremote(ud, d, "")
# Tags of the form ^{} may not work, need to fallback to other form
if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
@@ -2776,3 +2776,47 @@ class GitSharedTest(FetcherTest):
fetcher.unpack(self.unpackdir)
alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
self.assertFalse(os.path.exists(alt))
+
+class GitUnpackOfflineTest(FetcherTest):
+
+ def setUp(self):
+ super(GitUnpackOfflineTest, self).setUp()
+ self.tempdir = self.tempdir
+ self.persistdir = os.path.join(self.tempdir, "persistdata")
+
+ def setData(self):
+ self.recipe_url = "git://github.com/lz4/lz4.git;branch=release;protocol=https;"
+ self.d.setVar('SRCREV', 'v1.9.0')
+ self.d.setVar("PERSISTENT_DIR", self.persistdir)
+ self.d.setVar("DL_DIR", self.dldir)
+
+ @skipIfNoNetwork()
+ def test_unpack_offline(self):
+ # emulate do_fetch followed by do_unpack
+ self.setData()
+ bb.fetch.fetcher_init(self.d)
+ fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+
+ ## emulate do_unpack (disable network) withot previous fetch
+ self.d.setVar("BB_NO_NETWORK", "1")
+ bb.fetch.fetcher_init(self.d)
+ unpacker = bb.fetch.Fetch([self.recipe_url], self.d)
+ unpacker.unpack(self.unpackdir)
+
+ @skipIfNoNetwork()
+ def test_unpack_rebaseable(self):
+ self.setData()
+ self.recipe_url += "rebaseable=1;"
+
+ bb.fetch.fetcher_init(self.d)
+ fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+ fetcher.download()
+
+ ## emulate do_unpack (disable network)
+ self.d.setVar("BB_NO_NETWORK", "1")
+ bb.fetch.fetcher_init(self.d)
+ with self.assertRaises(bb.fetch2.UnpackError):
+ unpacker = bb.fetch.Fetch([self.recipe_url], self.d)
+ fetcher.unpack()
Sometimes Fetcher is initialized while network is disabled (unpack() is one of such cases) and Fetcher fails to map non SHA1 SRCREV to SHA1 one. In such cases git ls-remote cannot be used to determinate latest revision and git rev-parse can be used instead. This approach doesn't work with rebaseable repos so exception will occur for such repos. See [YOCTO #14707] for more details. Note: the issue doesn't occur each time in unpack() because in normal flow (fetch->unpack in one build) URI_HEADREVs are cached. Signed-off-by: Pavel Zhukov <pavel.zhukov@huawei.com> --- lib/bb/fetch2/git.py | 62 +++++++++++++++++++++++++++++++++---------- lib/bb/tests/fetch.py | 44 ++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 14 deletions(-)