From patchwork Thu May 14 20:05:26 2026 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: ms98.cho@gmail.com X-Patchwork-Id: 88129 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 45087CD4F39 for ; Thu, 14 May 2026 20:06:28 +0000 (UTC) Received: from Minsungs-MacBook-Air.local (Minsungs-MacBook-Air.local [24.205.142.70]) by mx.groups.io with SMTP id smtpd.msgproc01-g2.20356.1778789127566883781 for ; Thu, 14 May 2026 13:05:27 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=none, err=permanent DNS error (domain: minsungs-macbook-air.local, ip: 24.205.142.70, mailfrom: minsung@minsungs-macbook-air.local) Received: by Minsungs-MacBook-Air.local (Postfix, from userid 501) id D56525C2A00B; Thu, 14 May 2026 13:05:26 -0700 (PDT) From: ms98.cho@gmail.com To: bitbake-devel@lists.openembedded.org Cc: "minsung.cho" Subject: [PATCH v2] fetch2/crate: support configurable registry and index URLs Date: Thu, 14 May 2026 13:05:26 -0700 Message-ID: <20260514200526.98596-1-ms98.cho@gmail.com> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20260514051143.67992-1-ms98.cho@gmail.com> References: <20260514051143.67992-1-ms98.cho@gmail.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from 45-33-107-173.ip.linodeusercontent.com [45.33.107.173] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 14 May 2026 20:06:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/19525 The crate fetcher previously hardcoded the crates.io download URL and only supported a fixed /crate/versions API shape for custom registries. This prevented recipes from using private registries or mirrors with different download paths or Cargo sparse indexes. Add BB_CRATE_REGISTRY_URL[host] and BB_CRATE_INDEX_URL[host] templates with {crate}, {version}, and {index_path} placeholders. Keep the existing crates.io defaults, allow custom API-style version endpoints, and mark sparse indexes explicitly so latest-version checks parse Cargo index NDJSON instead of JSON API responses. Add non-network tests for default crates.io URLs, custom registry templates, sparse index paths, trailing slash handling, and both latest-version parser paths. Also make the fetch test cleanup chmod invocation portable by placing -R before the mode. [YOCTO #16276] Signed-off-by: minsung.cho --- lib/bb/fetch2/crate.py | 39 +++++++++++-- lib/bb/tests/fetch.py | 122 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 156 insertions(+), 5 deletions(-) diff --git a/lib/bb/fetch2/crate.py b/lib/bb/fetch2/crate.py index bb12f4e..a6dfc6b 100644 --- a/lib/bb/fetch2/crate.py +++ b/lib/bb/fetch2/crate.py @@ -77,12 +77,43 @@ class Crate(Wget): # host (this is to allow custom crate registries to be specified host = '/'.join(parts[2:-2]) - # If using crates.io use the CDN directly as per https://crates.io/data-access + # Allow overriding the registry and index URLs via bitbake variables. + # Example: + # BB_CRATE_REGISTRY_URL[my-host] = "https://my-host.com/crates/{crate}/{version}/download" + # BB_CRATE_INDEX_URL[my-host] = "https://my-host.com/index/{index_path}" + dl_url = d.getVarFlag('BB_CRATE_REGISTRY_URL', host) + index_url = d.getVarFlag('BB_CRATE_INDEX_URL', host) + if host == 'crates.io': - ud.url = "https://static.crates.io/crates/%s/%s/download" % (name, version) - ud.versionsurl = 'https://index.crates.io/' + self._generate_index_path(name) + if not dl_url: + dl_url = "https://static.crates.io/crates/{crate}/{version}/download" + if not index_url: + index_url = "https://index.crates.io/" + + if dl_url: + ud.url = dl_url.replace('{crate}', name).replace('{version}', version) else: ud.url = "https://%s/%s/%s/download" % (host, name, version) + + ud.crate_index_format = 'api' + if index_url: + index_path = self._generate_index_path(name) + # Support {index_path} for sparse index registries. If not present + # and it's crates.io, append it for backward compatibility. + if '{index_path}' in index_url: + ud.versionsurl = index_url.replace('{index_path}', index_path) + ud.crate_index_format = 'sparse' + elif host == 'crates.io' or index_url.startswith('https://index.crates.io/'): + if not index_url.endswith('/'): + index_url += '/' + ud.versionsurl = index_url + index_path + ud.crate_index_format = 'sparse' + else: + ud.versionsurl = index_url + + # Apply other placeholders + ud.versionsurl = ud.versionsurl.replace('{crate}', name).replace('{version}', version) + else: ud.versionsurl = "https://%s/%s/versions" % (host, name) ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version) @@ -160,7 +191,7 @@ class Crate(Wget): Return the latest upstream version, dispatching to the appropriate parser based on the versionsurl format. """ - if ud.versionsurl.startswith('https://index.crates.io/'): + if getattr(ud, 'crate_index_format', None) == 'sparse': return self._latest_versionstring_from_index(ud, d) return self._latest_versionstring_from_api(ud, d) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 86dd929..0dff3fc 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -424,7 +424,7 @@ class FetcherTest(unittest.TestCase): if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": print("Not cleaning up %s. Please remove manually." % self.tempdir) else: - bb.process.run('chmod u+rw -R %s' % self.tempdir) + bb.process.run('chmod -R u+rw %s' % self.tempdir) bb.utils.prunedir(self.tempdir) def git(self, cmd, cwd=None): @@ -2590,6 +2590,126 @@ class FetchLocallyMissingTagFromRemote(FetcherTest): class CrateTest(FetcherTest): + def test_crate_url_uses_crates_io_defaults(self): + ud = bb.fetch2.FetchData("crate://crates.io/glob/0.2.11", self.d) + + self.assertEqual(ud.url, + "https://static.crates.io/crates/glob/0.2.11/download") + self.assertEqual(ud.versionsurl, "https://index.crates.io/gl/ob/glob") + self.assertEqual(ud.crate_index_format, "sparse") + self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") + self.assertEqual(ud.parm["name"], "glob-0.2.11") + + def test_crate_url_supports_custom_registry_templates(self): + self.d.setVarFlag("BB_CRATE_REGISTRY_URL", "registry.example.com", + "https://registry.example.com/api/v1/crates/{crate}/{version}/download") + self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com", + "https://registry.example.com/api/v1/crates/{crate}/versions") + + ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d) + + self.assertEqual(ud.url, + "https://registry.example.com/api/v1/crates/glob/0.2.11/download") + self.assertEqual(ud.versionsurl, + "https://registry.example.com/api/v1/crates/glob/versions") + self.assertEqual(ud.crate_index_format, "api") + + def test_crate_url_supports_custom_sparse_index_templates(self): + self.d.setVarFlag("BB_CRATE_REGISTRY_URL", "registry.example.com", + "https://registry.example.com/crates/{crate}/{version}/download") + self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com", + "https://registry.example.com/index/{index_path}") + + ud = bb.fetch2.FetchData("crate://registry.example.com/aho-corasick/0.7.20", self.d) + + self.assertEqual(ud.url, + "https://registry.example.com/crates/aho-corasick/0.7.20/download") + self.assertEqual(ud.versionsurl, + "https://registry.example.com/index/ah/o-/aho-corasick") + self.assertEqual(ud.crate_index_format, "sparse") + + def test_crate_url_adds_sparse_index_path_with_missing_trailing_slash(self): + self.d.setVarFlag("BB_CRATE_INDEX_URL", "crates.io", "https://index.crates.io") + + ud = bb.fetch2.FetchData("crate://crates.io/glob/0.2.11", self.d) + + self.assertEqual(ud.versionsurl, "https://index.crates.io/gl/ob/glob") + self.assertEqual(ud.crate_index_format, "sparse") + + def test_crate_latest_versionstring_supports_custom_sparse_index(self): + self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com", + "https://registry.example.com/index/{index_path}") + ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d) + index = '\n'.join([ + '{"vers":"0.2.10","yanked":false}', + '{"vers":"0.2.11","yanked":true}', + '{"vers":"0.2.12","yanked":false}', + ]) + + with unittest.mock.patch("bb.fetch2.crate.Crate._fetch_index", return_value=index): + self.assertEqual(ud.method.latest_versionstring(ud, self.d), ("0.2.12", "")) + + def test_crate_latest_versionstring_supports_custom_api_index(self): + self.d.setVarFlag("BB_CRATE_INDEX_URL", "registry.example.com", + "https://registry.example.com/api/v1/crates/{crate}/versions") + ud = bb.fetch2.FetchData("crate://registry.example.com/glob/0.2.11", self.d) + index = '{"versions":[{"num":"0.2.10"},{"num":"0.2.12"}]}' + + with unittest.mock.patch("bb.fetch2.crate.Crate._fetch_index", return_value=index): + self.assertEqual(ud.method.latest_versionstring(ud, self.d), ("0.2.12", "")) + + def test_crate_fetches_from_local_sparse_registry(self): + registry = os.path.join(self.tempdir, "registry") + crate_name = "dummycrate" + crate_version = "1.0.0" + crate_basename = "%s-%s" % (crate_name, crate_version) + crate_path = os.path.join(registry, "crates", crate_name, + crate_version, "download") + index_path = os.path.join(registry, "index", "du", "mm", crate_name) + source_dir = os.path.join(self.tempdir, "crate-source", crate_basename) + bb.utils.mkdirhier(os.path.join(source_dir, "src")) + bb.utils.mkdirhier(os.path.dirname(crate_path)) + bb.utils.mkdirhier(os.path.dirname(index_path)) + + with open(os.path.join(source_dir, "Cargo.toml"), "w") as f: + f.write('[package]\nname = "%s"\nversion = "%s"\n' % + (crate_name, crate_version)) + with open(os.path.join(source_dir, "src", "lib.rs"), "w") as f: + f.write("pub fn answer() -> u32 { 42 }\n") + with tarfile.open(crate_path, "w:gz") as tar: + tar.add(source_dir, arcname=crate_basename) + with open(crate_path, "rb") as f: + crate_checksum = hashlib.sha256(f.read()).hexdigest() + with open(index_path, "w") as f: + f.write('{"name":"%s","vers":"%s","yanked":false}\n' % + (crate_name, crate_version)) + + server = HTTPService(registry, host="127.0.0.1") + server.start() + try: + host = "127.0.0.1:%s" % server.port + self.d.setVarFlag("BB_CRATE_REGISTRY_URL", host, + "http://%s/crates/{crate}/{version}/download" % host) + self.d.setVarFlag("BB_CRATE_INDEX_URL", host, + "http://%s/index/{index_path}" % host) + self.d.setVarFlag("SRC_URI", "%s.sha256sum" % crate_basename, + crate_checksum) + uri = "crate://%s/%s/%s" % (host, crate_name, crate_version) + + fetcher = bb.fetch2.Fetch([uri], self.d) + ud = fetcher.ud[fetcher.urls[0]] + self.assertEqual(ud.crate_index_format, "sparse") + self.assertEqual(ud.method.latest_versionstring(ud, self.d), + (crate_version, "")) + + fetcher.download() + fetcher.unpack(self.tempdir) + unpacked_file = os.path.join(self.tempdir, "cargo_home", "bitbake", + crate_basename, "src", "lib.rs") + self.assertTrue(os.path.exists(unpacked_file)) + finally: + server.stop() + @skipIfNoNetwork() def test_crate_url(self):