@@ -148,6 +148,16 @@ If :term:`BB_CHECK_SSL_CERTS` is set to ``0`` then SSL certificate checking will
be disabled. This variable defaults to ``1`` so SSL certificates are normally
checked.
+If :term:`BB_ALLOW_URL_REMAP` is set to ``1`` then URL's will be remapped
+based on the rules defined in :term:`BB_URL_REMAP` the varriable can be set
+in the global configuration or in the recipe.
+
+Here is a examples of how to set the variable::
+
+ BB_URL_REMAP = "\
+ git://example.com/ https://example.com/ \
+ http://example.com/ https://example.com/"
+
.. _bb-the-unpack:
The Unpack
@@ -494,6 +494,15 @@ overview of their function and contents.
running builds when not connected to the Internet, and when operating
in certain kinds of firewall environments.
+ :term:`BB_ALLOW_URL_REMAP`
+ Allows the use of the :term:`BB_URL_REMAP` variable to remap URL's.
+
+ :term:`BB_URL_REMAP`
+ A table of remappings. This variable is used to remap URLs to
+ different locations. The remapping is done by the fetcher module
+ before the fetcher attempts to access the URL. The remapping is
+ performed by replacing the string with the remapped string.
+
:term:`BB_NUMBER_PARSE_THREADS`
Sets the number of threads BitBake uses when parsing. By default, the
number of threads is equal to the number of cores on the system.
@@ -222,7 +222,7 @@ class URI(object):
# We hijack the URL parameters, since the way bitbake uses
# them are not quite RFC compliant.
- uri, param_str = (uri.split(";", 1) + [None])[:2]
+ uri, param_str = self.split_uri_and_parameters(uri)
urlp = urllib.parse.urlparse(uri)
self.scheme = urlp.scheme
@@ -298,6 +298,13 @@ class URI(object):
def _param_str_join(self, dict_, elmdelim, kvdelim="="):
return elmdelim.join([kvdelim.join([k, v]) if v else k for k, v in dict_.items()])
+ @staticmethod
+ def split_uri_and_parameters(uri):
+ """
+ Splits the URI into the URI and the parameters.
+ """
+ return (uri.split(";", 1) + [None])[:2]
+
@property
def hostport(self):
if not self.port:
@@ -1319,9 +1326,9 @@ class FetchData(object):
self.mirrortarballs = []
self.basename = None
self.basepath = None
- (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url))
+ self.url = self.remap_url_if_needed(d.expand(url), self.load_url_remap_data(d))
+ (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(self.url)
self.date = self.getSRCDate(d)
- self.url = url
if not self.user and "user" in self.parm:
self.user = self.parm["user"]
if not self.pswd and "pswd" in self.parm:
@@ -1424,6 +1431,61 @@ class FetchData(object):
return d.getVar("SRCDATE") or d.getVar("DATE")
+ def parse_url_remap_entries(self, pkgname, url_remap_entries, entries_as_str):
+ """
+ Parse the URL remap entries from the given dictionary.
+ """
+ if not entries_as_str:
+ return url_remap_entries
+
+ for entry in entries_as_str.split('\n'):
+ try:
+ from_url, to_url = entry.split()
+ url_remap_entries[from_url] = to_url
+ except ValueError:
+ logger.warning("Package %s contains invalid URL remap entry: %s", entry, pkgname)
+ continue
+
+ return url_remap_entries
+
+ def load_url_remap_data(self, d):
+ """
+ Load URL remap data from the package-specific and global variables.
+ """
+ url_remap_entries = {}
+
+ allow_url_remap = d.getVar('BB_ALLOW_URL_REMAP')
+ if not allow_url_remap or not bb.utils.to_boolean(allow_url_remap):
+ logger.debug("URL remapping is disabled")
+ return url_remap_entries
+
+ pkgname = d.getVar('PN')
+ if pkgname:
+ # Check for URL remap entries in the package-specific variable
+ url_remap_entries = self.parse_url_remap_entries(pkgname,
+ url_remap_entries,
+ d.getVarFlag('BB_URL_REMAP', pkgname, False))
+
+ # Check for URL remap entries in the global variable
+ url_remap_entries = self.parse_url_remap_entries(pkgname,
+ url_remap_entries,
+ d.getVar('BB_URL_REMAP'))
+
+ return url_remap_entries
+
+ def remap_url_if_needed(self, base_url, url_remap_entries):
+ """
+ Remap the URL if needed based on the URL remap data.
+ """
+ uri, paramters = URI.split_uri_and_parameters(base_url)
+ for from_url, to_url in url_remap_entries.items():
+ if uri.startswith(from_url):
+ uri = uri.replace(from_url, to_url, 1) + ';' + paramters
+ logger.debug("Remapped URL %s to %s", base_url, uri)
+ print("Remapped URL %s to %s" % (base_url, uri))
+ return uri
+ return base_url
+
class FetchMethod(object):
"""Base class for 'fetch'ing data"""
@@ -602,13 +602,19 @@ class GitDownloadDirectoryNamingTest(FetcherTest):
@skipIfNoNetwork()
def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
- self.setup_mirror_rewrite()
fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
fetcher.download()
dir = os.listdir(self.dldir + "/git2")
self.assertIn(self.recipe_dir, dir)
+ self.assertIn(self.recipe_dir + '.done', dir)
+ self.assertNotIn(self.mirror_dir, dir)
+ self.assertNotIn(self.mirror_dir + '.done', dir)
+ self.assertEqual(len(dir), len([
+ self.recipe_dir,
+ self.recipe_dir + '.done'
+ ]))
@skipIfNoNetwork()
def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self):
@@ -620,6 +626,12 @@ class GitDownloadDirectoryNamingTest(FetcherTest):
dir = os.listdir(self.dldir + "/git2")
self.assertIn(self.mirror_dir, dir)
self.assertIn(self.recipe_dir, dir)
+ self.assertIn(self.recipe_dir + '.done', dir)
+ self.assertEqual(len(dir), len([
+ self.recipe_dir,
+ self.recipe_dir + '.done',
+ self.mirror_dir
+ ]))
@skipIfNoNetwork()
def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self):
@@ -632,7 +644,15 @@ class GitDownloadDirectoryNamingTest(FetcherTest):
dir = os.listdir(self.dldir + "/git2")
self.assertIn(self.mirror_dir, dir)
+ self.assertIn(self.mirror_dir + '.done', dir)
self.assertIn(self.recipe_dir, dir)
+ self.assertIn(self.recipe_dir + '.done', dir)
+ self.assertEqual(len(dir), len([
+ self.recipe_dir,
+ self.recipe_dir + '.done',
+ self.mirror_dir,
+ self.mirror_dir + '.done'
+ ]))
class TarballNamingTest(FetcherTest):
@@ -657,6 +677,15 @@ class TarballNamingTest(FetcherTest):
dir = os.listdir(self.dldir)
self.assertIn(self.recipe_tarball, dir)
+ self.assertIn(self.recipe_tarball + '.done', dir)
+ self.assertIn('git2', dir)
+ self.assertNotIn(self.mirror_tarball, dir)
+ self.assertNotIn(self.mirror_tarball + '.done', dir)
+ self.assertEqual(len(dir), len([
+ self.recipe_tarball,
+ self.recipe_tarball + '.done',
+ 'git2'
+ ]))
@skipIfNoNetwork()
def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
@@ -666,7 +695,16 @@ class TarballNamingTest(FetcherTest):
fetcher.download()
dir = os.listdir(self.dldir)
+ self.assertNotIn(self.recipe_tarball, dir)
+ self.assertNotIn(self.recipe_tarball + '.done', dir)
+ self.assertIn('git2', dir)
self.assertIn(self.mirror_tarball, dir)
+ self.assertIn(self.mirror_tarball + '.done', dir)
+ self.assertEqual(len(dir), len([
+ self.mirror_tarball,
+ self.mirror_tarball + '.done',
+ 'git2'
+ ]))
class GitShallowTarballNamingTest(FetcherTest):
@@ -692,6 +730,15 @@ class GitShallowTarballNamingTest(FetcherTest):
dir = os.listdir(self.dldir)
self.assertIn(self.recipe_tarball, dir)
+ self.assertIn(self.recipe_tarball + '.done', dir)
+ self.assertIn('git2', dir)
+ self.assertNotIn(self.mirror_tarball, dir)
+ self.assertNotIn(self.mirror_tarball + '.done', dir)
+ self.assertEqual(len(dir), len([
+ self.recipe_tarball,
+ self.recipe_tarball + '.done',
+ 'git2'
+ ]))
@skipIfNoNetwork()
def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
@@ -701,7 +748,62 @@ class GitShallowTarballNamingTest(FetcherTest):
fetcher.download()
dir = os.listdir(self.dldir)
+ self.assertNotIn(self.recipe_tarball, dir)
+ self.assertNotIn(self.recipe_tarball + '.done', dir)
self.assertIn(self.mirror_tarball, dir)
+ self.assertIn(self.mirror_tarball + '.done', dir)
+ self.assertIn('git2', dir)
+ self.assertEqual(len(dir), len([
+ self.mirror_tarball,
+ self.mirror_tarball + '.done',
+ 'git2'
+ ]))
+
+
+class GitSameUrl(FetcherTest):
+ def setUp(self):
+ super(GitSameUrl, self).setUp()
+ self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
+ self.recipe_tarball = "git.openembedded.org.bitbake"
+ self.recipe_url_two = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
+ self.recipe_tarball_two = "github.com.openembedded.bitbake.git"
+
+ self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+ self.d.setVar('BB_URL_REMAP', 'git://github.com/openembedded/bitbake.git git://git.openembedded.org/bitbake')
+
+ @skipIfNoNetwork()
+ def test_url_remap_allowed(self):
+ self.d.setVar('BB_ALLOW_URL_REMAP', '1')
+ fetcher = bb.fetch.Fetch([self.recipe_url, self.recipe_url_two], self.d)
+ fetcher.download()
+
+ dir = os.listdir(self.dldir + "/git2")
+ self.assertIn(self.recipe_tarball, dir)
+ self.assertIn(self.recipe_tarball + '.done', dir)
+ self.assertNotIn(self.recipe_tarball_two, dir)
+ self.assertNotIn(self.recipe_tarball_two + '.done', dir)
+ self.assertEqual(len(dir), len([
+ self.recipe_tarball,
+ self.recipe_tarball + '.done'
+ ]))
+
+ @skipIfNoNetwork()
+ def test_url_remap_not_allowed(self):
+ self.d.setVar('BB_ALLOW_URL_REMAP', '0')
+ fetcher = bb.fetch.Fetch([self.recipe_url, self.recipe_url_two], self.d)
+ fetcher.download()
+
+ dir = os.listdir(self.dldir + "/git2")
+ self.assertIn(self.recipe_tarball, dir)
+ self.assertIn(self.recipe_tarball + '.done', dir)
+ self.assertIn(self.recipe_tarball_two, dir)
+ self.assertIn(self.recipe_tarball_two + '.done', dir)
+ self.assertEqual(len(dir), len([
+ self.recipe_tarball,
+ self.recipe_tarball + '.done',
+ self.recipe_tarball_two,
+ self.recipe_tarball_two + '.done'
+ ]))
class CleanTarballTest(FetcherTest):