diff mbox series

[RFC,14/21] fetch: add dependency mixin

Message ID 20241220112613.22647-15-stefan.herbrechtsmeier-oss@weidmueller.com
State New
Headers show
Series Concept for tightly coupled package manager (Node.js, Go, Rust) | expand

Commit Message

Stefan Herbrechtsmeier Dec. 20, 2024, 11:26 a.m. UTC 
From: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>

Add a dependency mixin classes and module. The mixin implements generic
methods to fetch dependencies via a dependency specification file. The
subclass must implement a resolve dependencies method. The method
resolves the dependencies from a specification file into fetcher URLs.
The module provides different fetcher subtypes:

<type> (Local)
    The fetcher uses a local specification file to fetch dependencies.

    SRC_URI = "<type>://specification.txt"

<type>+https (Wget)
    The fetcher downloads a specification file or archive with a
    specification file in the root folder and uses the specification
    file to fetch dependencies.

    SRC_URI = "<type>+http://example.com/specification.txt "
    SRC_URI = "<type>+http://example.com/${BP}.tar.gz;striplevel=1;subdir=${BP}"

<type>+git (Git)
    The fetcher checkouts a git repository with a specification file to
    fetch dependencies.

    SRC_URI = "<type>+git://example.com/${BPN}.git;protocol=https"

Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
---

 lib/bb/fetch2/dependency.py | 175 ++++++++++++++++++++++++++++++++++++
 1 file changed, 175 insertions(+)
 create mode 100644 lib/bb/fetch2/dependency.py
diff mbox series

Patch

diff --git a/lib/bb/fetch2/dependency.py b/lib/bb/fetch2/dependency.py
new file mode 100644
index 000000000..4acad8779
--- /dev/null
+++ b/lib/bb/fetch2/dependency.py
@@ -0,0 +1,175 @@ 
+# Copyright (C) 2024-2025 Weidmueller Interface GmbH & Co. KG
+# Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
+#
+# SPDX-License-Identifier: MIT
+#
+"""
+BitBake 'Fetch' mixin implementation for dependency specification files
+"""
+
+import tempfile
+import bb
+from bb.fetch2 import Fetch
+from bb.fetch2.git import Git
+from bb.fetch2.local import Local
+from bb.fetch2.wget import Wget
+from bb.utils import lockfile, unlockfile
+
+class DependencyMixin:
+    """Class to fetch all dependencies resolved via foreign function"""
+
+    def urldata_init(self, ud, d):
+        ud.type = ud.type.split("+")[-1] if "+" in ud.type else "file"
+        ud.url = ":".join((ud.type, ud.url.split(":", 1)[-1]))
+        super().urldata_init(ud, d)
+        ud.proxy = None
+
+    def _init_proxy(self, ud, d):
+        if ud.proxy:
+            return
+
+        urls = self.process_source(ud, d)
+        if urls:
+            ud.proxy = Fetch(urls, d)
+
+    @staticmethod
+    def _foreach_proxy_method(ud, handle, d):
+        """Call method for each dependency"""
+        returns = []
+        for proxy_url in ud.proxy.urls:
+            proxy_ud = ud.proxy.ud[proxy_url]
+            proxy_d = ud.proxy.d
+            proxy_ud.setup_localpath(proxy_d)
+            lf = lockfile(proxy_ud.lockfile)
+            returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
+            unlockfile(lf)
+        return returns
+
+    def verify_donestamp(self, ud, d):
+        """Verify the donestamp file"""
+        if not super().verify_donestamp(ud, d):
+            return False
+
+        self._init_proxy(ud, d)
+        def handle(m, ud, d):
+            return m.verify_donestamp(ud, d)
+        return all(self._foreach_proxy_method(ud, handle, d))
+
+    def update_donestamp(self, ud, d):
+        """Update the donestamp file"""
+        super().update_donestamp(ud, d)
+
+        self._init_proxy(ud, d)
+        def handle(m, ud, d):
+            m.update_donestamp(ud, d)
+        self._foreach_proxy_method(ud, handle, d)
+
+    def need_update(self, ud, d):
+        """Force a fetch, even if localpath exists ?"""
+        if super().need_update(ud, d):
+            return True
+
+        self._init_proxy(ud, d)
+        def handle(m, ud, d):
+            return m.need_update(ud, d)
+        return any(self._foreach_proxy_method(ud, handle, d))
+
+    def try_mirrors(self, fetch, ud, d, mirrors):
+        """Try to use a mirror"""
+        if not super().try_mirrors(fetch, ud, d, mirrors):
+            return False
+
+        self._init_proxy(ud, d)
+        def handle(m, ud, d):
+            return m.try_mirrors(fetch, ud, d, mirrors)
+        return all(self._foreach_proxy_method(ud, handle, d))
+
+    def download(self, ud, d):
+        """Fetch url"""
+        super().download(ud, d)
+        self._init_proxy(ud, d)
+        ud.proxy.download()
+
+    def unpack(self, ud, rootdir, d):
+        """Unpack the downloaded dependencies"""
+        super().unpack(ud, rootdir, d)
+        self._init_proxy(ud, d)
+        ud.proxy.unpack(ud.destdir)
+
+    def clean(self, ud, d):
+        """Clean any existing full or partial download"""
+        self._init_proxy(ud, d)
+        ud.proxy.clean()
+        super().clean(ud, d)
+
+    def done(self, ud, d):
+        """Is the download done ?"""
+        if not super().done(ud, d):
+            return False
+
+        self._init_proxy(ud, d)
+        def _handle(m, ud, d):
+            return m.done(ud, d)
+        return all(self._foreach_proxy_method(ud, _handle, d))
+
+class LocalDependency(DependencyMixin, Local):
+    """
+    Abstract class to fetch all dependencies from a local specification file
+    """
+
+    def process_source(self, ud, d):
+        return self.resolve_dependencies(ud, ud.localpath, d)
+
+class WgetDependency(DependencyMixin, Wget):
+    """
+    Abstract class to fetch all dependencies from a specification file inside an
+    archive
+    """
+
+    def process_source(self, ud, d):
+        with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir:
+            Wget.unpack(self, ud, tmpdir, d)
+            return self.resolve_dependencies(ud, ud.destdir, d)
+
+class GitDependency(DependencyMixin, Git):
+    """
+    Abstract class to fetch all dependencies from a specification file inside a
+    git repository
+    """
+
+    def process_source(self, ud, d):
+        with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir:
+            Git.unpack(self, ud, tmpdir, d)
+            return self.resolve_dependencies(ud, ud.destdir, d)
+
+def create_methods(type, mixin):
+    class SpecificLocalDependency(mixin, LocalDependency):
+        """
+        Specific class to fetch all dependencies from a local specification file
+        """
+
+        def supports(self, ud, d):
+            return ud.type == type
+
+    class SpecificWgetDependency(mixin, WgetDependency):
+        """
+        Specific class to fetch all dependencies from a specification file
+        inside an archive
+        """
+
+        def supports(self, ud, d):
+            return ud.type in [f"{type}+http", f"{type}+https"]
+
+    class SpecificGitDependency(mixin, GitDependency):
+        """
+        Specific class to fetch all dependencies from a specification file
+        inside a git repository
+        """
+
+        def supports(self, ud, d):
+            return ud.type == f"{type}+git"
+
+    return [
+        SpecificLocalDependency(),
+        SpecificWgetDependency(),
+        SpecificGitDependency()]