@@ -11,17 +11,15 @@
from recipetool.create import RecipeHandler, handle_license_vars
-from recipetool.create import find_licenses
import bb.utils
import json
import logging
import os
import re
+import subprocess
import sys
import tempfile
-import urllib.parse
-import urllib.request
logger = logging.getLogger('recipetool')
@@ -66,97 +64,6 @@ class GoRecipeHandler(RecipeHandler):
return bindir
- @staticmethod
- def __unescape_path(path):
- """Unescape capital letters using exclamation points."""
- return re.sub(r'!([a-z])', lambda m: m.group(1).upper(), path)
-
- @staticmethod
- def __fold_uri(uri):
- """Fold URI for sorting shorter module paths before longer."""
- return uri.replace(';', ' ').replace('/', '!')
-
- @staticmethod
- def __go_run_cmd(cmd, cwd, d):
- env = dict(os.environ, PATH=d.getVar('PATH'), GOMODCACHE=d.getVar('GOMODCACHE'))
- return bb.process.run(cmd, env=env, shell=True, cwd=cwd)
-
- def __go_mod(self, go_mod, srctree, localfilesdir, extravalues, d):
- moddir = d.getVar('GOMODCACHE')
-
- # List main packages and their dependencies with the go list command.
- stdout, _ = self.__go_run_cmd(f"go list -json=Dir,Module -deps {go_mod['Module']['Path']}/...", srctree, d)
- pkgs = json.loads('[' + stdout.replace('}\n{', '},\n{') + ']')
-
- # Collect licenses for the dependencies.
- licenses = set()
- lic_files_chksum = []
- lic_files = {}
- for pkg in pkgs:
- # TODO: If the package is in a subdirectory with its own license
- # files then report those istead of the license files found in the
- # module root directory.
- mod = pkg.get('Module', None)
- if not mod or mod.get('Main', False):
- continue
- path = os.path.relpath(mod['Dir'], moddir)
- for lic in find_licenses(mod['Dir'], d):
- lic_files[os.path.join(path, lic[1])] = (lic[0], lic[2])
-
- for lic_file in lic_files:
- licenses.add(lic_files[lic_file][0])
- lic_files_chksum.append(
- f'file://pkg/mod/{lic_file};md5={lic_files[lic_file][1]}')
-
- # Collect the module cache files downloaded by the go list command as
- # the go list command knows best what the go list command needs and it
- # needs more files in the module cache than the go install command as
- # it doesn't do the dependency pruning mentioned in the Go module
- # reference, https://go.dev/ref/mod, for go 1.17 or higher.
- src_uris = []
- downloaddir = os.path.join(moddir, 'cache', 'download')
- for dirpath, _, filenames in os.walk(downloaddir):
- path, base = os.path.split(os.path.relpath(dirpath, downloaddir))
- if base != '@v':
- continue
- path = self.__unescape_path(path)
- zipver = None
- for name in filenames:
- ver, ext = os.path.splitext(name)
- if ext == '.zip':
- chksum = bb.utils.sha256_file(os.path.join(dirpath, name))
- src_uris.append(f'gomod://{path};version={ver};sha256sum={chksum}')
- zipver = ver
- break
- for name in filenames:
- ver, ext = os.path.splitext(name)
- if ext == '.mod' and ver != zipver:
- chksum = bb.utils.sha256_file(os.path.join(dirpath, name))
- src_uris.append(f'gomod://{path};version={ver};mod=1;sha256sum={chksum}')
-
- self.__go_run_cmd("go clean -modcache", srctree, d)
-
- licenses_basename = "{pn}-licenses.inc"
- licenses_filename = os.path.join(localfilesdir, licenses_basename)
- with open(licenses_filename, "w") as f:
- f.write(f'GO_MOD_LICENSES = "{" & ".join(sorted(licenses))}"\n\n')
- f.write('LIC_FILES_CHKSUM += "\\\n')
- for lic in sorted(lic_files_chksum, key=self.__fold_uri):
- f.write(' ' + lic + ' \\\n')
- f.write('"\n')
-
- extravalues['extrafiles'][f"../{licenses_basename}"] = licenses_filename
-
- go_mods_basename = "{pn}-go-mods.inc"
- go_mods_filename = os.path.join(localfilesdir, go_mods_basename)
- with open(go_mods_filename, "w") as f:
- f.write('SRC_URI += "\\\n')
- for uri in sorted(src_uris, key=self.__fold_uri):
- f.write(' ' + uri + ' \\\n')
- f.write('"\n')
-
- extravalues['extrafiles'][f"../{go_mods_basename}"] = go_mods_filename
-
def process(self, srctree, classes, lines_before,
lines_after, handled, extravalues):
@@ -167,37 +74,52 @@ class GoRecipeHandler(RecipeHandler):
if not files:
return False
- d = bb.data.createCopy(tinfoil.config_data)
go_bindir = self.__ensure_go()
if not go_bindir:
sys.exit(14)
- d.prependVar('PATH', '%s:' % go_bindir)
handled.append('buildsystem')
classes.append("go-mod")
- tmp_mod_dir = tempfile.mkdtemp(prefix='go-mod-')
- d.setVar('GOMODCACHE', tmp_mod_dir)
+ # Use go-mod-update-modules to set the full SRC_URI and LICENSE
+ classes.append("go-mod-update-modules")
+ extravalues["run_task"] = "update_modules"
- stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d)
- go_mod = json.loads(stdout)
- go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path'])
+ with tempfile.TemporaryDirectory(prefix="go-mod-") as tmp_mod_dir:
+ env = dict(os.environ)
+ env["PATH"] += f":{go_bindir}"
+ env['GOMODCACHE'] = tmp_mod_dir
- localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
- extravalues.setdefault('extrafiles', {})
+ stdout = subprocess.check_output(["go", "mod", "edit", "-json"], cwd=srctree, env=env, text=True)
+ go_mod = json.loads(stdout)
+ go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path'])
- # Write the ${BPN}-licenses.inc and ${BPN}-go-mods.inc files
- self.__go_mod(go_mod, srctree, localfilesdir, extravalues, d)
+ localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
+ extravalues.setdefault('extrafiles', {})
- # Do generic license handling
- handle_license_vars(srctree, lines_before, handled, extravalues, d)
- self.__rewrite_lic_vars(lines_before)
+ # Write the stub ${BPN}-licenses.inc and ${BPN}-go-mods.inc files
+ basename = "{pn}-licenses.inc"
+ filename = os.path.join(localfilesdir, basename)
+ with open(filename, "w") as f:
+ f.write("# FROM RECIPETOOL\n")
+ extravalues['extrafiles'][f"../{basename}"] = filename
- self.__rewrite_src_uri(lines_before)
+ basename = "{pn}-go-mods.inc"
+ filename = os.path.join(localfilesdir, basename)
+ with open(filename, "w") as f:
+ f.write("# FROM RECIPETOOL\n")
+ extravalues['extrafiles'][f"../{basename}"] = filename
- lines_before.append('require ${BPN}-licenses.inc')
- lines_before.append('require ${BPN}-go-mods.inc')
- lines_before.append(f'GO_IMPORT = "{go_import}"')
+ # Do generic license handling
+ d = bb.data.createCopy(tinfoil.config_data)
+ handle_license_vars(srctree, lines_before, handled, extravalues, d)
+ self.__rewrite_lic_vars(lines_before)
+
+ self.__rewrite_src_uri(lines_before)
+
+ lines_before.append('require ${BPN}-licenses.inc')
+ lines_before.append('require ${BPN}-go-mods.inc')
+ lines_before.append(f'GO_IMPORT = "{go_import}"')
def __update_lines_before(self, updated, newlines, lines_before):
if updated:
@@ -210,10 +132,8 @@ class GoRecipeHandler(RecipeHandler):
return updated
def __rewrite_lic_vars(self, lines_before):
-
def varfunc(varname, origvalue, op, newlines):
- if varname == 'LICENSE':
- return ' & '.join((origvalue, '${GO_MOD_LICENSES}')), None, -1, True
+ import urllib.parse
if varname == 'LIC_FILES_CHKSUM':
new_licenses = []
licenses = origvalue.split('\\')
@@ -235,7 +155,7 @@ class GoRecipeHandler(RecipeHandler):
return origvalue, None, 0, True
updated, newlines = bb.utils.edit_metadata(
- lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc)
+ lines_before, ['LIC_FILES_CHKSUM'], varfunc)
return self.__update_lines_before(updated, newlines, lines_before)
def __rewrite_src_uri(self, lines_before):
Now that the go-mod-update-modules class exists, this Go handler can create a stub recipe and then proxy the module handling to the class. Signed-off-by: Ross Burton <ross.burton@arm.com> --- scripts/lib/recipetool/create_go.py | 152 +++++++--------------------- 1 file changed, 36 insertions(+), 116 deletions(-)