diff mbox series

fetch/wget: Reuse the checkstatus() urllib opener

Message ID 20260923170846.1360600-1-ecordonnier@snap.com
State New
Headers show
Series fetch/wget: Reuse the checkstatus() urllib opener | expand

Commit Message

Etienne Cordonnier Sept. 23, 2026, 5:08 p.m. UTC 
From: Etienne Cordonnier <ecordonnier@snap.com>

checkstatus() builds a new urllib opener with its redirect, proxy and
connection cache handlers on every call, although sstate.bbclass calls it
for every object in an sstate mirror check with the same settings. Keep
the opener on the FetchConnectionCache, keyed on the certificate checking
setting and the fetcher environment, and rebuild it only when those
change. The handlers now refer to the connection cache directly rather
than through the first Fetch object, so a cached opener does not keep it
alive. A connection cache is only used by one thread at a time, so no
locking is needed.

Checking 2,114 objects on the Yocto sstate mirror with BB_NUMBER_THREADS = "8"
reduced the time to task execution from 26.9s to 25.1s on average.

AI-Generated: Uses Claude Code (Claude Opus 5.5)
Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
---
 lib/bb/fetch/__init__.py |  2 ++
 lib/bb/fetch/wget.py     | 45 ++++++++++++++++++++++++----------------
 2 files changed, 29 insertions(+), 18 deletions(-)
diff mbox series

Patch

diff --git a/lib/bb/fetch/__init__.py b/lib/bb/fetch/__init__.py
index 55ab710f3..0eb660685 100644
--- a/lib/bb/fetch/__init__.py
+++ b/lib/bb/fetch/__init__.py
@@ -2090,6 +2090,8 @@  class FetchConnectionCache(object):
     """
     def __init__(self):
         self.cache = {}
+        # urllib openers built by the wget fetcher, see Wget.checkstatus()
+        self.opener_cache = {}
 
     def get_connection_name(self, host, port, connection_id=None):
         if connection_id is None:
diff --git a/lib/bb/fetch/wget.py b/lib/bb/fetch/wget.py
index 972c84048..9cb4a4f32 100644
--- a/lib/bb/fetch/wget.py
+++ b/lib/bb/fetch/wget.py
@@ -178,23 +178,24 @@  class Wget(FetchMethod):
     def checkstatus(self, fetch, ud, d, try_again=True):
         check_certs = self.check_certs(d)
         newenv = bb.fetch.get_fetcher_environment(d)
+        connection_cache = fetch.connection_cache
 
         class HTTPConnectionCache(http.client.HTTPConnection):
             def cache_id(self):
                 return None
 
-            if fetch.connection_cache:
+            if connection_cache:
                 def connect(self):
                     """Connect to the host and port specified in __init__."""
 
-                    sock = fetch.connection_cache.get_connection(
+                    sock = connection_cache.get_connection(
                         self.host, self.port, self.cache_id())
                     if sock:
                         self.sock = sock
                     else:
                         self.sock = socket.create_connection((self.host, self.port),
                                     self.timeout, self.source_address)
-                        fetch.connection_cache.add_connection(
+                        connection_cache.add_connection(
                             self.host, self.port, self.sock, self.cache_id())
 
                     if self._tunnel_host:
@@ -206,17 +207,17 @@  class Wget(FetchMethod):
                         newenv.get("SSL_CERT_FILE"),
                         self._tunnel_host, self._tunnel_port)
 
-            if fetch.connection_cache:
+            if connection_cache:
                 def connect(self):
                     """Reuse an established TLS connection when available."""
 
-                    sock = fetch.connection_cache.get_connection(
+                    sock = connection_cache.get_connection(
                         self.host, self.port, self.cache_id())
                     if sock:
                         self.sock = sock
                     else:
                         super().connect()
-                        fetch.connection_cache.add_connection(
+                        connection_cache.add_connection(
                             self.host, self.port, self.sock, self.cache_id())
 
         class CacheHTTPHandler(urllib.request.HTTPHandler):
@@ -252,7 +253,7 @@  class Wget(FetchMethod):
                 # request.
 
                 # Don't close connection when connection_cache is enabled,
-                if fetch.connection_cache is None:
+                if connection_cache is None:
                     headers["Connection"] = "close"
                 else:
                     headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0
@@ -282,8 +283,8 @@  class Wget(FetchMethod):
                     # with the dead connection removed from the cache.
                     # If it still fails, we give up, which can happen for bad
                     # HTTP proxy settings.
-                    if fetch.connection_cache:
-                        fetch.connection_cache.remove_connection(
+                    if connection_cache:
+                        connection_cache.remove_connection(
                             h.host, h.port, h.cache_id())
                     h.close()
                     raise
@@ -316,9 +317,9 @@  class Wget(FetchMethod):
                 resp.msg = r.reason
 
                 # Close connection when server request it.
-                if fetch.connection_cache is not None:
+                if connection_cache is not None:
                     if 'Connection' in r.msg and r.msg['Connection'] == 'close':
-                        fetch.connection_cache.remove_connection(
+                        connection_cache.remove_connection(
                             h.host, h.port, h.cache_id())
 
                 return resp
@@ -434,13 +435,21 @@  class Wget(FetchMethod):
         # to scope the changes to the build_opener request, which is when the
         # environment lookups happen.
         with bb.utils.environment(**newenv):
-            context = self.ssl_context(d)
-            handlers = [FixedHTTPRedirectHandler,
-                        HTTPMethodFallback,
-                        urllib.request.ProxyHandler(),
-                        CacheHTTPHandler(),
-                        CacheHTTPSHandler(context=context)]
-            opener = urllib.request.build_opener(*handlers)
+            opener = None
+            if connection_cache is not None:
+                opener_key = (check_certs, tuple(sorted(newenv.items())))
+                opener = connection_cache.opener_cache.get(opener_key)
+
+            if opener is None:
+                context = self.ssl_context(d)
+                handlers = [FixedHTTPRedirectHandler,
+                            HTTPMethodFallback,
+                            urllib.request.ProxyHandler(),
+                            CacheHTTPHandler(),
+                            CacheHTTPSHandler(context=context)]
+                opener = urllib.request.build_opener(*handlers)
+                if connection_cache is not None:
+                    connection_cache.opener_cache[opener_key] = opener
 
             try:
                 parts = urllib.parse.urlparse(ud.url.split(";")[0])