diff mbox series

fetch2/gomod: support pipe-separated proxy list in GO_MOD_PROXY

Message ID 178367645337.2070338.3995116335029500655@qti.qualcomm.com
State New
Headers show
Series fetch2/gomod: support pipe-separated proxy list in GO_MOD_PROXY | expand

Commit Message

tengf July 10, 2026, 9:40 a.m. UTC 
From cb7d8f5176781eb441ec83241f05ab272d4611c6 Mon Sep 17 00:00:00 2001
From: tengf <tengf@qti.qualcomm.com>
Date: Fri, 10 Jul 2026 16:54:20 +0800
Subject: [PATCH] fetch2/gomod: support pipe-separated proxy list in
 GO_MOD_PROXY
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Go toolchain supports GOPROXY as a pipe-separated list of proxies
tried in order (e.g. "goproxy.cn|proxy.golang.org|direct"). The BitBake
gomod fetcher previously passed the entire string verbatim as an HTTP
hostname, causing fetch failures in environments where proxy.golang.org
is unreachable and a mirror such as goproxy.cn is required.

Parse GO_MOD_PROXY as a '|'-separated list and try each proxy in order
in a new download() override on the GoMod class. The special token
"direct" is silently skipped — direct VCS fetching is handled by the
gomodgit fetcher. If every proxy fails, a FetchError is raised listing
the module and version. A single proxy value (the existing usage) is
fully backward compatible.

Add two unit tests that exercise proxy-list parsing without requiring
network access:
- test_gomod_multi_proxy_uses_first_proxy: verifies the first non-direct
  proxy is selected for ud.url and that the full proxy list is stored.
- test_gomod_direct_only_falls_back_to_default: verifies that a
  "direct"-only value falls back to proxy.golang.org.

Signed-off-by: tengf <tengf@qti.qualcomm.com>
---
 lib/bb/fetch2/gomod.py | 39 +++++++++++++++++++++++++++++++++++----
 lib/bb/tests/fetch.py  | 22 ++++++++++++++++++++++
 2 files changed, 57 insertions(+), 4 deletions(-)
diff mbox series

Patch

diff --git a/lib/bb/fetch2/gomod.py b/lib/bb/fetch2/gomod.py
index 5cdf8f998..7bb91e2a0 100644
--- a/lib/bb/fetch2/gomod.py
+++ b/lib/bb/fetch2/gomod.py
@@ -48,7 +48,10 @@  Optional SRC_URI parameters:
 Related variables:
 
 - GO_MOD_PROXY
-    The module proxy used by the fetcher.
+    The module proxy used by the fetcher. Supports a pipe-separated list of
+    proxies tried in order, matching Go's GOPROXY syntax (e.g.
+    "goproxy.cn|proxy.golang.org|direct"). The special value "direct" is
+    skipped (direct VCS fetching is handled by the gomodgit fetcher).
 
 - GO_MOD_CACHE_DIR
     The directory where the module cache is located.
@@ -97,7 +100,7 @@  class GoMod(Wget):
         cache/download/<module>/@v/<version>.mod: The go.mod file.
         """
 
-        proxy = d.getVar('GO_MOD_PROXY') or 'proxy.golang.org'
+        proxy_setting = d.getVar('GO_MOD_PROXY') or 'proxy.golang.org'
         moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
 
         if 'version' not in ud.parm:
@@ -109,14 +112,24 @@  class GoMod(Wget):
         ud.parm['module'] = module
         version = ud.parm['version']
 
-        # Set URL and filename for wget download
+        # Parse pipe-separated proxy list (Go GOPROXY syntax); drop "direct"
+        # entries since direct VCS fetching is the job of the gomodgit fetcher.
+        proxies = [p for p in proxy_setting.split('|') if p.strip() and p.strip() != 'direct']
+        if not proxies:
+            proxies = ['proxy.golang.org']
+        ud.parm['proxies'] = proxies
+
+        # Set URL and filename for wget download.  Use the first proxy for
+        # urldata_init so that localpath / checksum keys are stable regardless
+        # of which proxy ultimately serves the file.
         if ud.parm.get('mod', '0') == '1':
             ext = '.mod'
         else:
             ext = '.zip'
         path = escape(f"{module}/@v/{version}{ext}")
+        ud.parm['mod_path'] = path
         ud.url = bb.fetch2.encodeurl(
-            ('https', proxy, '/' + path, None, None, None))
+            ('https', proxies[0], '/' + path, None, None, None))
         ud.parm['downloadfilename'] =  f"{module.replace('/', '.')}@{version}{ext}"
 
         # Set name for checksum verification
@@ -127,6 +140,24 @@  class GoMod(Wget):
 
         super().urldata_init(ud, d)
 
+    def download(self, ud, d):
+        """Try each proxy in GO_MOD_PROXY in order, falling back on failure."""
+        proxies = ud.parm.get('proxies', ['proxy.golang.org'])
+        path = ud.parm['mod_path']
+        last_exc = None
+        for proxy in proxies:
+            ud.url = bb.fetch2.encodeurl(
+                ('https', proxy, '/' + path, None, None, None))
+            try:
+                super().download(ud, d)
+                return
+            except FetchError as e:
+                bb.warn(f"GoMod: proxy {proxy} failed for {ud.parm['module']}: {e}, trying next proxy")
+                last_exc = e
+        raise FetchError(
+            f"All proxies failed for {ud.parm['module']}@{ud.parm['version']}: {last_exc}",
+            ud.url)
+
     def unpack(self, ud, rootdir, d):
         """Unpack the module in the module cache."""
 
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index d021ad786..76b2e203c 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -3815,6 +3815,28 @@  class GoModTest(FetcherTest):
         self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')),
                          '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96')
 
+    def test_gomod_multi_proxy_uses_first_proxy(self):
+        """First non-direct proxy in GO_MOD_PROXY is used to build ud.url."""
+        urls = ['gomod://golang.org/x/net;version=v0.9.0;'
+                'sha256sum=d9b70a10f88ededacb4e0e974de5d5a98e3db9c1e0e01c4dbfe5a5279b65a0c']
+        self.d.setVar('GO_MOD_PROXY', 'goproxy.cn|proxy.golang.org|direct')
+
+        fetcher = bb.fetch2.Fetch(urls, self.d)
+        ud = fetcher.ud[urls[0]]
+        self.assertIn('goproxy.cn', ud.url)
+        self.assertEqual(ud.parm['proxies'], ['goproxy.cn', 'proxy.golang.org'])
+
+    def test_gomod_direct_only_falls_back_to_default(self):
+        """A GO_MOD_PROXY of 'direct' (only) falls back to proxy.golang.org."""
+        urls = ['gomod://golang.org/x/net;version=v0.9.0;'
+                'sha256sum=d9b70a10f88ededacb4e0e974de5d5a98e3db9c1e0e01c4dbfe5a5279b65a0c']
+        self.d.setVar('GO_MOD_PROXY', 'direct')
+
+        fetcher = bb.fetch2.Fetch(urls, self.d)
+        ud = fetcher.ud[urls[0]]
+        self.assertEqual(ud.parm['proxies'], ['proxy.golang.org'])
+        self.assertIn('proxy.golang.org', ud.url)
+
 class GoModGitTest(FetcherTest):
 
     @skipIfNoNetwork()