@@ -2112,3 +2112,4 @@ methods.append(az.Az())
methods.append(crate.Crate())
methods.append(gcp.GCP())
methods.append(gomod.GoMod())
+methods.append(gomod.GoModGit())
@@ -1,12 +1,13 @@
"""
BitBake 'Fetch' implementation for Go modules
-The gomod fetcher is used to download Go modules to the module cache from a
-module proxy.
+The gomod fetchers are used to download Go modules to the module cache from a
+module proxy or directly from a version control repository.
Example SRC_URIs:
SRC_URI = "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..."
+SRC_URI = "gomod://golang.org/x/net;version=v0.9.0;direct=git;repo=go.googlesource.com/net;srcrev=..."
Required SRC_URI parameters:
@@ -27,6 +28,27 @@ Optional SRC_URI parameters:
only the go.mod file. Alternatively, set the SRC_URI varible flag for
"module@version.sha256sum".
+- direct
+ Fetch directly from a version control repository.
+ Supported values: "git".
+
+- protocol
+ The method used when fetching directly from a version control repository.
+ The default is "https" for git.
+
+- repo
+ The URL when fetching directly from a version control repository. Required
+ when the URL is different from the module path.
+
+- srcrev
+ The revision identifier used when fetching directly from a version control
+ repository. Alternatively, set the SRCREV varible for "module@version".
+
+- subdir
+ The module subdirectory when fetching directly from a version control
+ repository. Required when the module is not located in the root of the
+ repository.
+
Related variables:
- GOMODCACHE
@@ -41,13 +63,19 @@ See the Go modules reference, https://go.dev/ref/mod, for more information
about the module cache, module proxies and version control systems.
"""
+import hashlib
import os
import re
import shutil
+import subprocess
import zipfile
import bb
-from bb.fetch2 import FetchError, MissingParameterError
+from bb.fetch2 import FetchError
+from bb.fetch2 import MissingParameterError
+from bb.fetch2 import runfetchcmd
+from bb.fetch2 import subprocess_setup
+from bb.fetch2.git import Git
from bb.fetch2.wget import Wget
@@ -61,7 +89,7 @@ class GoMod(Wget):
def supports(self, ud, d):
"""Check to see if a given URL is for this fetcher."""
- return ud.type == 'gomod'
+ return ud.type == 'gomod' and 'direct' not in ud.parm
def urldata_init(self, ud, d):
"""Set up to download the module from the module proxy."""
@@ -114,3 +142,104 @@ class GoMod(Wget):
# If the module does not have a go.mod file, synthesize
# one containing only a module statement.
mf.write(f'module {module}\n'.encode())
+
+
+class GoModGit(Git):
+ """Class to fetch Go modules directly from a git repository"""
+
+ def supports(self, ud, d):
+ """Check to see if a given URL is for this fetcher."""
+ return ud.type == 'gomod' and ud.parm['direct'] == 'git'
+
+ def urldata_init(self, ud, d):
+ """Set up to download the module from the git repository."""
+
+ mod_dir = d.getVar('GOMODCACHE')
+ if not mod_dir:
+ raise FetchError("The module cache location is not specified in the"
+ " GOMODCACHE environment variable.")
+
+ if 'version' not in ud.parm:
+ raise MissingParameterError('version', ud.url)
+
+ module = ud.host + ud.path
+ ud.parm['module'] = module
+ if 'repo' in ud.parm:
+ repo = ud.parm['repo']
+ idx = repo.find('/')
+ if idx != -1:
+ ud.host = repo[:idx]
+ ud.path = repo[idx:]
+ else:
+ ud.host = repo
+ ud.path = ''
+ if 'protocol' not in ud.parm:
+ ud.parm['protocol'] = 'https'
+ ud.parm['bareclone'] = '1'
+ if 'subdir' in ud.parm:
+ ud.parm['subpath'] = ud.parm['subdir']
+ key = f"git3:{ud.parm['protocol']}://{ud.host}{ud.path}".encode()
+ ud.parm['key'] = key
+ ud.parm['subdir'] = os.path.join(mod_dir, 'cache', 'vcs',
+ hashlib.sha256(key).hexdigest())
+ name = f"{module}@{ud.parm['version']}"
+ ud.names = [name]
+ srcrev = d.getVar('SRCREV_' + name)
+ if srcrev:
+ if 'srcrev' not in ud.parm:
+ ud.parm['srcrev'] = srcrev
+ else:
+ if 'srcrev' in ud.parm:
+ d.setVar('SRCREV_' + name, ud.parm['srcrev'])
+ if 'branch' not in ud.parm:
+ ud.parm['nobranch'] = '1'
+ super().urldata_init(ud, d)
+
+ def unpack(self, ud, rootdir, d):
+ """Unpack the module in the module cache."""
+
+ # Unpack the bare git repository
+ super().unpack(ud, rootdir, d)
+
+ # Create the info file
+ module = ud.parm['module']
+ repodir = ud.parm['subdir']
+ with open(repodir + '.info', 'wb') as f:
+ f.write(ud.parm['key'])
+
+ # Unpack the go.mod file from the repository
+ unpackdir = os.path.join(d.getVar('GOMODCACHE'), 'cache', 'download',
+ escape(module), '@v')
+ bb.utils.mkdirhier(unpackdir)
+ srcrev = ud.parm['srcrev']
+ version = ud.parm['version']
+ escaped_version = escape(version)
+ cmd = f"git ls-tree -r --name-only '{srcrev}'"
+ if 'subpath' in ud.parm:
+ cmd += f" '{ud.parm['subpath']}'"
+ files = runfetchcmd(cmd, d, workdir=repodir).split()
+ name = escaped_version + '.mod'
+ bb.note(f"Unpacking {name} to {unpackdir}/")
+ with open(os.path.join(unpackdir, name), mode='wb') as mf:
+ f = 'go.mod'
+ if 'subpath' in ud.parm:
+ f = os.path.join(ud.parm['subpath'], f)
+ if f in files:
+ cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
+ subprocess.check_call(cmd, stdout=mf, cwd=repodir,
+ preexec_fn=subprocess_setup)
+ else:
+ # If the module does not have a go.mod file, synthesize one
+ # containing only a module statement.
+ mf.write(f'module {module}\n'.encode())
+
+ # Synthesize the module zip file from the repository
+ name = escaped_version + '.zip'
+ bb.note(f"Unpacking {name} to {unpackdir}/")
+ with zipfile.ZipFile(os.path.join(unpackdir, name), mode='w') as zf:
+ prefix = module + '@' + version + '/'
+ for f in files:
+ cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
+ data = subprocess.check_output(cmd, cwd=repodir,
+ preexec_fn=subprocess_setup)
+ zf.writestr(prefix + f, data)
@@ -3463,3 +3463,100 @@ class GoModTest(FetcherTest):
downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
+
+class GoModGitTest(FetcherTest):
+
+ @skipIfNoNetwork()
+ def test_gomod_url_git_repo(self):
+ self.d.setVar('GOMODCACHE', os.path.join(self.unpackdir, 'pkg/mod'))
+
+ urls = ['gomod://golang.org/x/net;version=v0.9.0;'
+ 'direct=git;repo=go.googlesource.com/net;'
+ 'srcrev=694cff8668bac64e0864b552bffc280cd27f21b1']
+
+ fetcher = bb.fetch2.Fetch(urls, self.d)
+ ud = fetcher.ud[urls[0]]
+ self.assertEqual(ud.host, 'go.googlesource.com')
+ self.assertEqual(ud.path, '/net')
+ self.assertEqual(ud.names, ['golang.org/x/net@v0.9.0'])
+ self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1')
+
+ fetcher.download()
+ self.assertTrue(os.path.exists(ud.localpath))
+
+ fetcher.unpack(self.unpackdir)
+ vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+ self.assertTrue(os.path.exists(os.path.join(vcsdir, 'ed42bd05533fd84ae290a5d33ebd3695a0a2b06131beebd5450825bee8603aca')))
+ downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.zip')))
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod')))
+
+ @skipIfNoNetwork()
+ def test_gomod_url_git_subdir(self):
+ self.d.setVar('GOMODCACHE', os.path.join(self.unpackdir, 'pkg/mod'))
+
+ urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;'
+ 'direct=git;repo=github.com/Azure/azure-sdk-for-go;subdir=sdk/storage/azblob;'
+ 'srcrev=ec928e0ed34db682b3f783d3739d1c538142e0c3']
+
+ fetcher = bb.fetch2.Fetch(urls, self.d)
+ ud = fetcher.ud[urls[0]]
+ self.assertEqual(ud.host, 'github.com')
+ self.assertEqual(ud.path, '/Azure/azure-sdk-for-go')
+ self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob')
+ self.assertEqual(ud.names, ['github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'])
+ self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3')
+
+ fetcher.download()
+ self.assertTrue(os.path.exists(ud.localpath))
+
+ fetcher.unpack(self.unpackdir)
+ vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+ self.assertTrue(os.path.exists(os.path.join(vcsdir, 'd31d6145676ed3066ce573a8198f326dea5be45a43b3d8f41ce7787fd71d66b3')))
+ downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip')))
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
+
+ @skipIfNoNetwork()
+ def test_gomod_url_git_srcrev_var(self):
+ self.d.setVar('GOMODCACHE', os.path.join(self.unpackdir, 'pkg/mod'))
+
+ urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0;direct=git']
+ self.d.setVar('SRCREV_gopkg.in/ini.v1@v1.67.0', 'b2f570e5b5b844226bbefe6fb521d891f529a951')
+
+ fetcher = bb.fetch2.Fetch(urls, self.d)
+ ud = fetcher.ud[urls[0]]
+ self.assertEqual(ud.host, 'gopkg.in')
+ self.assertEqual(ud.path, '/ini.v1')
+ self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0'])
+ self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951')
+
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+ vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+ self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3')))
+ downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
+
+ @skipIfNoNetwork()
+ def test_gomod_url_git_no_mod_in_zip(self):
+ self.d.setVar('GOMODCACHE', os.path.join(self.unpackdir, 'pkg/mod'))
+
+ urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0;direct=git;'
+ 'srcrev=b2f570e5b5b844226bbefe6fb521d891f529a951']
+
+ fetcher = bb.fetch2.Fetch(urls, self.d)
+ ud = fetcher.ud[urls[0]]
+ self.assertEqual(ud.host, 'gopkg.in')
+ self.assertEqual(ud.path, '/ini.v1')
+ self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0'])
+ self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951')
+
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+ vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+ self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3')))
+ downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
+ self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))