diff mbox series

[RFC,4/6] fetch2: add support for implicit urls

Message ID 20250902065507.35737-5-stefan.herbrechtsmeier-oss@weidmueller.com
State New
Headers show
Series fetch2: add support for implicit urls | expand

Commit Message

Stefan Herbrechtsmeier Sept. 2, 2025, 6:55 a.m. UTC 
From: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>

Add support for implicit URLs which are defined inside another source
like a version control system (git submodule) or an embedded lock file
(package-lock.json, cargo.lock or go.sum). The implicit URLs are
extracted from the source and thereby only available after the download
of the explicit source. The integration of implicit URLs beside explicit
URLs simplifies the fetcher classes and avoid bugs because of iterations
between the Fetch and FetchMethod classes.

Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
---

 lib/bb/fetch2/__init__.py | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
diff mbox series

Patch

diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 6da75008b..e7bbdac14 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -1312,6 +1312,7 @@  class FetchData(object):
         if not self.pswd and "pswd" in self.parm:
             self.pswd = self.parm["pswd"]
         self.setup = False
+        self.implicit_urls = None
 
         def configure_checksum(checksum_id):
             checksum_plain_name = "%ssum" % checksum_id
@@ -1409,6 +1410,11 @@  class FetchData(object):
 
         return d.getVar("SRCDATE") or d.getVar("DATE")
 
+    def setup_implicit_urls(self, d):
+        if self.implicit_urls is None:
+            self.setup_localpath(d)
+            self.implicit_urls = self.method.implicit_urls(self, d)
+
 class FetchMethod(object):
     """Base class for 'fetch'ing data"""
 
@@ -1727,6 +1733,9 @@  class FetchMethod(object):
         """
         return []
 
+    def implicit_urls(self, ud, d):
+        return []
+
 
 class DummyUnpackTracer(object):
     """
@@ -1843,6 +1852,9 @@  class Fetch(object):
             ud = self._get_urldata(url)
             ud.setup_localpath(self.d)
             local.append(ud.localpath)
+            ud.setup_implicit_urls(self.d)
+            for u in ud.implicit_urls:
+                localpath(u)
 
         for url in self.urls:
             localpath(url)
@@ -1960,6 +1972,16 @@  class Fetch(object):
         for url in urls:
             download(url)
 
+        def download_implicit(url):
+            ud = self._get_urldata(url)
+            ud.setup_implicit_urls(self.d)
+            for u in ud.implicit_urls:
+                download(u)
+                download_implicit(u)
+
+        for url in urls:
+            download_implicit(url)
+
         if checksum_missing_messages:
             logger.error("Missing SRC_URI checksum, please add those to the recipe: \n%s", "\n".join(checksum_missing_messages))
             raise BBFetchException("There was some missing checksums in the recipe")
@@ -2022,6 +2044,12 @@  class Fetch(object):
             if ud.lockfile:
                 bb.utils.unlockfile(lf)
 
+            ud.setup_implicit_urls(self.d)
+            for u in ud.implicit_urls:
+                unpack(u)
+            if hasattr(ud.method, "postunpack"):
+                ud.method.postunpack(ud, root, self.d)
+
         for url in urls:
             unpack(url)
 
@@ -2038,6 +2066,10 @@  class Fetch(object):
         def clean(url):
             ud = self._get_urldata(url)
             ud.setup_localpath(self.d)
+            if os.path.exists(ud.localpath):
+                ud.setup_implicit_urls(self.d)
+                for u in ud.implicit_urls:
+                    clean(u)
 
             if not ud.localfile and ud.localpath is None:
                 return
@@ -2070,6 +2102,9 @@  class Fetch(object):
             ud = self._get_urldata(url)
             urldata.append(ud)
             urldata += ud.method.implicit_urldata(ud, self.d)
+            ud.setup_implicit_urls(self.d)
+            for u in ud.implicit_urls:
+                expand_urldata(u)
 
         for url in urls:
             expand_urldata(url)