| Message ID | 20231017132647.352938-2-lukas.funke-oss@weidmueller.com |
|---|---|
| State | New |
| Headers | show |
| Series | recipetool: Add handler to create go recipes | expand |
Hi Lukas, just two small typos I stumbled over. On Tue, Oct 17 2023 at 15:26 +0200, "Lukas Funke" <lukas.funke-oss@weidmueller.com> wrote: > From: Lukas Funke <lukas.funke@weidmueller.com> > > Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com> > --- > meta/classes/go-vendor.bbclass | 135 +++++++++++++++++++++++++++++++++ > 1 file changed, 135 insertions(+) > create mode 100644 meta/classes/go-vendor.bbclass > > diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass > new file mode 100644 > index 0000000000..13f1b8b2be > --- /dev/null > +++ b/meta/classes/go-vendor.bbclass > @@ -0,0 +1,135 @@ > +# > +# Copyright 2023 (C) Weidmueller GmbH & Co KG > +# Author: Lukas Funke <lukas.funke@weidmueller.com> > +# > +# Handle Go vendor support for offline builds > +# > +# When importing Go modules, Go downloads the imported module using s/imported module/imported modules/ > +# a network (proxy) connection ahead of the compile stage. This contradicts > +# the yocto build concept of fetching every source ahead of build-time > +# and supporting offline builds. > +# > +# To support offline builds, we use Go 'vendoring': module dependencies are > +# downloaded during the fetch-phase and unpacked into the modules 'vendor' > +# folder. Additinally a manifest file is generated for the 'vendor' folder s/Additinally/Additionally/ Best regards Ulrich > +# > + > +inherit go-mod > + > +def go_src_uri(repo, version, path=None, subdir=None, \ > + vcs='git', replaces=None, pathmajor=None): > + > + destsuffix = "git/src/import/vendor.fetch" > + go_module_path = repo if not path else path > + > + src_uri = "{}://{}" \ > + ";name={}" \ > + "".format(vcs, repo, \ > + go_module_path.replace('/', '.')) > + > + src_uri += ";destsuffix={}/{}@{}".format(destsuffix, \ > + go_module_path, \ > + version) > + > + if vcs == "git": > + src_uri += ";nobranch=1;protocol=https" > + if replaces: > + src_uri += ";go_module_replacement={}".format(replaces) > + if subdir: > + src_uri += ";go_subdir={}".format(subdir) > + if pathmajor: > + src_uri += ";go_pathmajor={}".format(pathmajor) > + > + return src_uri > + > + > +python do_go_vendor() { > + import shutil > + > + src_uri = (d.getVar('SRC_URI') or "").split() > + > + if len(src_uri) == 0: > + bb.error("SRC_URI is empty") > + return > + > + default_destsuffix = "git/src/import/vendor.fetch" > + fetcher = bb.fetch2.Fetch(src_uri, d) > + go_import = d.getVar('GO_IMPORT') > + source_dir = d.getVar('S') > + > + vendor_dir = os.path.join(source_dir, *['src', go_import, 'vendor']) > + import_dir = os.path.join(source_dir, *['src', 'import', 'vendor.fetch']) > + > + bb.utils.mkdirhier(vendor_dir) > + modules = {} > + > + for url in fetcher.urls: > + srcuri = fetcher.ud[url].host + fetcher.ud[url].path > + > + # Skip main module for which the recipe is actually created > + if srcuri == go_import: > + continue > + > + # Skip local files > + if fetcher.ud[url].type == 'file': > + continue > + > + destsuffix = fetcher.ud[url].parm.get('destsuffix') > + # We derive the module path / version in the following manner (exmaple): > + # > + # destsuffix = git/src/import/vendor.fetch/github.com/foo/bar@v1.2.3 > + # p = github.com/foo/bar@v1.2.3 > + # path = github.com/foo/bar > + # version = v1.2.3 > + > + p = destsuffix[len(default_destsuffix)+1:] > + path, version = p.split('@') > + > + subdir = fetcher.ud[url].parm.get('go_subdir') > + subdir = "" if not subdir else subdir > + > + pathMajor = fetcher.ud[url].parm.get('go_pathmajor') > + pathMajor = "" if not pathMajor else pathMajor > + > + base = path[:-(len(subdir)+len(pathMajor))-1] > + r = fetcher.ud[url].parm.get('go_module_replacement') > + > + if not path in modules and not r: > + modules[path] = { > + "version": version, > + "src": os.path.join(import_dir, *[p, subdir]), > + "subdir": subdir, > + "pathMajor": pathMajor, > + } > + > + for module_key in sorted(modules): > + > + # only take the version which is explicitly listed > + # as a dependency in the go.mod > + module = modules[module_key] > + src = module['src'] > + > + # If the module is released at major version 2 or higher, the module > + # path must end with a major version suffix like /v2. > + # This may or may not be part of the subdirectory name > + # > + # https://go.dev/ref/mod#modules-overview > + srcMajorVersion = os.path.join(src, module['pathMajor'].strip('/')) > + if os.path.exists(srcMajorVersion): > + src = srcMajorVersion > + dst = os.path.join(vendor_dir, module_key) > + if os.path.exists(dst): > + shutil.rmtree(dst) > + > + bb.debug(1, "cp %s --> %s" % (src, dst)) > + shutil.copytree(src, dst, symlinks=True, \ > + ignore=shutil.ignore_patterns(".git", \ > + "vendor", \ > + "*.md", \ > + "*._test.go")) > + # Copy vendor manifest > + bb.debug(1, "cp %s --> %s" % (os.path.join(d.getVar('WORKDIR'), "modules.txt"), vendor_dir)) > + shutil.copy2(os.path.join(d.getVar('WORKDIR'), "modules.txt"), vendor_dir) > +} > + > +addtask go_vendor before do_populate_lic after do_unpack
diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass new file mode 100644 index 0000000000..13f1b8b2be --- /dev/null +++ b/meta/classes/go-vendor.bbclass @@ -0,0 +1,135 @@ +# +# Copyright 2023 (C) Weidmueller GmbH & Co KG +# Author: Lukas Funke <lukas.funke@weidmueller.com> +# +# Handle Go vendor support for offline builds +# +# When importing Go modules, Go downloads the imported module using +# a network (proxy) connection ahead of the compile stage. This contradicts +# the yocto build concept of fetching every source ahead of build-time +# and supporting offline builds. +# +# To support offline builds, we use Go 'vendoring': module dependencies are +# downloaded during the fetch-phase and unpacked into the modules 'vendor' +# folder. Additinally a manifest file is generated for the 'vendor' folder +# + +inherit go-mod + +def go_src_uri(repo, version, path=None, subdir=None, \ + vcs='git', replaces=None, pathmajor=None): + + destsuffix = "git/src/import/vendor.fetch" + go_module_path = repo if not path else path + + src_uri = "{}://{}" \ + ";name={}" \ + "".format(vcs, repo, \ + go_module_path.replace('/', '.')) + + src_uri += ";destsuffix={}/{}@{}".format(destsuffix, \ + go_module_path, \ + version) + + if vcs == "git": + src_uri += ";nobranch=1;protocol=https" + if replaces: + src_uri += ";go_module_replacement={}".format(replaces) + if subdir: + src_uri += ";go_subdir={}".format(subdir) + if pathmajor: + src_uri += ";go_pathmajor={}".format(pathmajor) + + return src_uri + + +python do_go_vendor() { + import shutil + + src_uri = (d.getVar('SRC_URI') or "").split() + + if len(src_uri) == 0: + bb.error("SRC_URI is empty") + return + + default_destsuffix = "git/src/import/vendor.fetch" + fetcher = bb.fetch2.Fetch(src_uri, d) + go_import = d.getVar('GO_IMPORT') + source_dir = d.getVar('S') + + vendor_dir = os.path.join(source_dir, *['src', go_import, 'vendor']) + import_dir = os.path.join(source_dir, *['src', 'import', 'vendor.fetch']) + + bb.utils.mkdirhier(vendor_dir) + modules = {} + + for url in fetcher.urls: + srcuri = fetcher.ud[url].host + fetcher.ud[url].path + + # Skip main module for which the recipe is actually created + if srcuri == go_import: + continue + + # Skip local files + if fetcher.ud[url].type == 'file': + continue + + destsuffix = fetcher.ud[url].parm.get('destsuffix') + # We derive the module path / version in the following manner (exmaple): + # + # destsuffix = git/src/import/vendor.fetch/github.com/foo/bar@v1.2.3 + # p = github.com/foo/bar@v1.2.3 + # path = github.com/foo/bar + # version = v1.2.3 + + p = destsuffix[len(default_destsuffix)+1:] + path, version = p.split('@') + + subdir = fetcher.ud[url].parm.get('go_subdir') + subdir = "" if not subdir else subdir + + pathMajor = fetcher.ud[url].parm.get('go_pathmajor') + pathMajor = "" if not pathMajor else pathMajor + + base = path[:-(len(subdir)+len(pathMajor))-1] + r = fetcher.ud[url].parm.get('go_module_replacement') + + if not path in modules and not r: + modules[path] = { + "version": version, + "src": os.path.join(import_dir, *[p, subdir]), + "subdir": subdir, + "pathMajor": pathMajor, + } + + for module_key in sorted(modules): + + # only take the version which is explicitly listed + # as a dependency in the go.mod + module = modules[module_key] + src = module['src'] + + # If the module is released at major version 2 or higher, the module + # path must end with a major version suffix like /v2. + # This may or may not be part of the subdirectory name + # + # https://go.dev/ref/mod#modules-overview + srcMajorVersion = os.path.join(src, module['pathMajor'].strip('/')) + if os.path.exists(srcMajorVersion): + src = srcMajorVersion + dst = os.path.join(vendor_dir, module_key) + if os.path.exists(dst): + shutil.rmtree(dst) + + bb.debug(1, "cp %s --> %s" % (src, dst)) + shutil.copytree(src, dst, symlinks=True, \ + ignore=shutil.ignore_patterns(".git", \ + "vendor", \ + "*.md", \ + "*._test.go")) + # Copy vendor manifest + bb.debug(1, "cp %s --> %s" % (os.path.join(d.getVar('WORKDIR'), "modules.txt"), vendor_dir)) + shutil.copy2(os.path.join(d.getVar('WORKDIR'), "modules.txt"), vendor_dir) +} + +addtask go_vendor before do_populate_lic after do_unpack