diff mbox series

[RFC,v2,6/7] fetch2: add support for implicit urls

Message ID 20250905064419.2589111-7-stefan.herbrechtsmeier-oss@weidmueller.com
State New
Headers show
Series fetch2: add support for implicit urls | expand

Commit Message

Stefan Herbrechtsmeier Sept. 5, 2025, 6:44 a.m. UTC 
From: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>

Add support for implicit URLs which are defined inside another source
like a version control system (git submodule) or an embedded lock file
(package-lock.json, cargo.lock or go.sum). The implicit URLs are
extracted from the source and thereby only available after the download
of the explicit source. The integration of implicit URLs beside explicit
URLs simplifies the fetcher classes and avoid bugs because of iterations
between the Fetch and FetchMethod classes.

Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>

---

Changes in v2:
- Remove implicit URLs from localpaths for backward compatibility

 lib/bb/fetch2/__init__.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
diff mbox series

Patch

diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index c4d7cc033..1bf16b7e6 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -1312,6 +1312,7 @@  class FetchData(object):
         if not self.pswd and "pswd" in self.parm:
             self.pswd = self.parm["pswd"]
         self.setup = False
+        self.implicit_urls = None
 
         def configure_checksum(checksum_id):
             checksum_plain_name = "%ssum" % checksum_id
@@ -1409,6 +1410,11 @@  class FetchData(object):
 
         return d.getVar("SRCDATE") or d.getVar("DATE")
 
+    def setup_implicit_urls(self, d):
+        if self.implicit_urls is None:
+            self.setup_localpath(d)
+            self.implicit_urls = self.method.implicit_urls(self, d)
+
 class FetchMethod(object):
     """Base class for 'fetch'ing data"""
 
@@ -1727,6 +1733,9 @@  class FetchMethod(object):
         """
         return []
 
+    def implicit_urls(self, ud, d):
+        return []
+
 
 class DummyUnpackTracer(object):
     """
@@ -1960,6 +1969,16 @@  class Fetch(object):
         for url in urls:
             download(url)
 
+        def download_implicit(url):
+            ud = self._get_urldata(url)
+            ud.setup_implicit_urls(self.d)
+            for u in ud.implicit_urls:
+                download(u)
+                download_implicit(u)
+
+        for url in urls:
+            download_implicit(url)
+
         if checksum_missing_messages:
             logger.error("Missing SRC_URI checksum, please add those to the recipe: \n%s", "\n".join(checksum_missing_messages))
             raise BBFetchException("There was some missing checksums in the recipe")
@@ -2022,6 +2041,12 @@  class Fetch(object):
             if ud.lockfile:
                 bb.utils.unlockfile(lf)
 
+            ud.setup_implicit_urls(self.d)
+            for u in ud.implicit_urls:
+                unpack(u)
+            if hasattr(ud.method, "postunpack"):
+                ud.method.postunpack(ud, root, self.d)
+
         for url in urls:
             unpack(url)
 
@@ -2038,6 +2063,10 @@  class Fetch(object):
         def clean(url):
             ud = self._get_urldata(url)
             ud.setup_localpath(self.d)
+            if os.path.exists(ud.localpath):
+                ud.setup_implicit_urls(self.d)
+                for u in ud.implicit_urls:
+                    clean(u)
 
             if not ud.localfile and ud.localpath is None:
                 return
@@ -2070,6 +2099,9 @@  class Fetch(object):
             ud = self._get_urldata(url)
             urldata.append(ud)
             urldata.extend(ud.method.implicit_urldata(ud, self.d))
+            ud.setup_implicit_urls(self.d)
+            for u in ud.implicit_urls:
+                expand_urldata(u)
 
         for url in urls:
             expand_urldata(url)