diff mbox series

[v2,2/2] fetch/wget: Reuse the checkstatus() urllib opener

Message ID 20260928125051.2483039-3-ecordonnier@snap.com
State New
Headers show
Series fetch/wget: Reuse the checkstatus() urllib opener | expand

Commit Message

Etienne Cordonnier Sept. 28, 2026, 12:50 p.m. UTC 
From: Etienne Cordonnier <ecordonnier@snap.com>

checkstatus() builds a new urllib opener with its redirect, proxy and
connection cache handlers on every call, although sstate.bbclass calls it
for every object in an sstate mirror check with the same settings. Keep
the opener on the FetchConnectionCache, keyed on the certificate checking
setting and the fetcher environment, and rebuild it only when those
change. A connection cache is only used by one thread at a time, so no
locking is needed.

Checking 2,114 objects on the Yocto sstate mirror with BB_NUMBER_THREADS = "8"
reduced the time to task execution from 26.9s to 25.1s on average.

AI-Generated: Uses Claude Code (Claude Opus 5.5)
Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
---
 lib/bb/fetch/__init__.py |  2 ++
 lib/bb/fetch/wget.py     | 22 +++++++++++++++-------
 2 files changed, 17 insertions(+), 7 deletions(-)
diff mbox series

Patch

diff --git a/lib/bb/fetch/__init__.py b/lib/bb/fetch/__init__.py
index 55ab710f3..0eb660685 100644
--- a/lib/bb/fetch/__init__.py
+++ b/lib/bb/fetch/__init__.py
@@ -2090,6 +2090,8 @@  class FetchConnectionCache(object):
     """
     def __init__(self):
         self.cache = {}
+        # urllib openers built by the wget fetcher, see Wget.checkstatus()
+        self.opener_cache = {}
 
     def get_connection_name(self, host, port, connection_id=None):
         if connection_id is None:
diff --git a/lib/bb/fetch/wget.py b/lib/bb/fetch/wget.py
index fe2de25d3..9cb4a4f32 100644
--- a/lib/bb/fetch/wget.py
+++ b/lib/bb/fetch/wget.py
@@ -435,13 +435,21 @@  class Wget(FetchMethod):
         # to scope the changes to the build_opener request, which is when the
         # environment lookups happen.
         with bb.utils.environment(**newenv):
-            context = self.ssl_context(d)
-            handlers = [FixedHTTPRedirectHandler,
-                        HTTPMethodFallback,
-                        urllib.request.ProxyHandler(),
-                        CacheHTTPHandler(),
-                        CacheHTTPSHandler(context=context)]
-            opener = urllib.request.build_opener(*handlers)
+            opener = None
+            if connection_cache is not None:
+                opener_key = (check_certs, tuple(sorted(newenv.items())))
+                opener = connection_cache.opener_cache.get(opener_key)
+
+            if opener is None:
+                context = self.ssl_context(d)
+                handlers = [FixedHTTPRedirectHandler,
+                            HTTPMethodFallback,
+                            urllib.request.ProxyHandler(),
+                            CacheHTTPHandler(),
+                            CacheHTTPSHandler(context=context)]
+                opener = urllib.request.build_opener(*handlers)
+                if connection_cache is not None:
+                    connection_cache.opener_cache[opener_key] = opener
 
             try:
                 parts = urllib.parse.urlparse(ud.url.split(";")[0])