@@ -1,8 +1,16 @@
"""
BitBake 'Fetch' implementation for Amazon AWS S3.
-Class for fetching files from Amazon S3 using the AWS Command Line Interface.
-The aws tool must be correctly installed and configured prior to use.
+Class for fetching files from Amazon S3.
+
+By default this uses the boto3 Python SDK, which talks to S3 over HTTPS
+natively (no per-object subprocess / AWS CLI startup cost). This is a big win
+for sstate mirrors, where thousands of small objects are fetched and the AWS
+CLI's interpreter + config/credential resolution startup dominates wall time.
+
+If boto3 is not importable, the fetcher transparently falls back to shelling
+out to the AWS CLI ("aws s3 cp" / "aws s3 ls"), preserving the original
+behaviour. The AWS tool must then be correctly installed and configured.
"""
@@ -17,13 +25,22 @@ The aws tool must be correctly installed and configured prior to use.
import os
import bb
+import time
+import random
import urllib.request, urllib.parse, urllib.error
import re
-import shlex
from bb.fetch import FetchMethod
from bb.fetch import FetchError
from bb.fetch import runfetchcmd
+try:
+ import boto3
+ import botocore
+ HAVE_BOTO3 = True
+except ImportError:
+ HAVE_BOTO3 = False
+
+
def convertToBytes(value, unit):
value = float(value)
if (unit == "KiB"):
@@ -60,8 +77,105 @@ class S3ProgressHandler(bb.progress.LineFilterProgressHandler):
return True
+# Cache a single boto3 S3 client for the lifetime of the (worker) process.
+# Creating a client resolves credentials and config once; reusing it keeps the
+# underlying HTTPS connection pool warm across many small fetches.
+#
+# NOTE: bitbake runs setscene fetches in SEPARATE worker processes, so this
+# cache is per-process, not global to the build. Credentials must come from a
+# source that AUTO-REFRESHES, because a Yocto build runs for hours and any
+# static/temporary credential (e.g. `ada credentials update --once` written
+# into ~/.aws/credentials [default]) will EXPIRE partway through -- after which
+# S3 returns "400 Bad Request" on every call and all setscene tasks fail.
+#
+# The robust source is a `credential_process` profile (e.g. ada): boto3 invokes
+# it on demand and re-vends when the token nears expiry. We therefore resolve
+# AWS_PROFILE from the bitbake datastore (or the environment) and pin it on the
+# session, so the worker uses that auto-refreshing profile instead of a stale
+# static [default]. Set `AWS_PROFILE` in the build config (it is listed in the
+# bitbake-setup config's bb-env-passthrough-additions) to a profile whose
+# [profile ...] entry in ~/.aws/config uses credential_process.
+#
+# ada caches vended credentials on disk, so the per-worker credential_process
+# calls mostly hit that warm cache rather than Conduit; the backoff below still
+# covers a transient "Rate exceeded" during a cold-cache storm.
+_s3_client = None
+
+def _get_s3_client(d):
+ global _s3_client
+ if _s3_client is None:
+ # Resolve region from the usual env vars; boto3 also honours
+ # ~/.aws/config and AWS_CONFIG_FILE. (S3 handles cross-region HEAD/GET,
+ # so an exact match is not required, but pinning avoids surprises.)
+ region = (d.getVar("AWS_DEFAULT_REGION")
+ or os.environ.get("AWS_DEFAULT_REGION")
+ or os.environ.get("AWS_REGION"))
+ # Resolve the AWS profile to use. Pinning it defeats a stale/expired
+ # static [default] in ~/.aws/credentials shadowing the auto-refreshing
+ # credential_process -- the exact cause of mid-build "400 Bad Request"
+ # failures. d.getVar wins so it works even when the env is scrubbed in
+ # a worker; env is the fallback when it is passed through.
+ profile = (d.getVar("AWS_PROFILE")
+ or os.environ.get("AWS_PROFILE")
+ or os.environ.get("AWS_DEFAULT_PROFILE"))
+ # Allow more concurrent connections than the default (10) so parallel
+ # fetches within a process share a healthy pool. The 'retries' here
+ # applies to S3 API calls only, NOT to credential resolution.
+ cfg = botocore.config.Config(
+ max_pool_connections=50,
+ retries={"max_attempts": 5, "mode": "standard"},
+ )
+
+ # Credential resolution happens lazily on first API call OR eagerly in
+ # client(); either way a credential_process "Rate exceeded" surfaces as
+ # CredentialRetrievalError. Retry client creation + a cheap credential
+ # resolution with exponential backoff + jitter so a transient ada/
+ # Conduit rate limit does not fail the whole setscene task.
+ attempts = int(d.getVar("S3_CRED_MAX_ATTEMPTS") or 6)
+ last_exc = None
+ for attempt in range(attempts):
+ try:
+ session = boto3.session.Session(profile_name=profile, region_name=region)
+ client = session.client("s3", config=cfg)
+ # Force credential resolution now so we retry it here rather
+ # than failing deep inside a download/head call.
+ creds = session.get_credentials()
+ # One-time-per-worker diagnostic: which credential source won,
+ # the profile, and the region. Tells us whether the worker
+ # picked the auto-refreshing credential_process profile or fell
+ # back to a stale static default.
+ try:
+ method = getattr(creds, "method", None) if creds else "NONE"
+ bb.debug(1, "S3 fetcher: credential method=%s profile=%s region=%s"
+ % (method, profile, client.meta.region_name))
+ except Exception:
+ pass
+ _s3_client = client
+ break
+ except botocore.exceptions.CredentialRetrievalError as e:
+ last_exc = e
+ if attempt == attempts - 1:
+ break
+ # Exponential backoff with jitter: 0.5,1,2,4,8s (+/- jitter),
+ # capped. Spreads out many workers hitting ada at once.
+ delay = min(2.0 ** attempt * 0.5, 15.0)
+ delay += random.uniform(0, delay / 2.0)
+ bb.note("S3 fetcher: credential retrieval rate-limited "
+ "(attempt %d/%d), backing off %.1fs: %s"
+ % (attempt + 1, attempts, delay, e))
+ time.sleep(delay)
+ if _s3_client is None:
+ raise FetchError("S3 fetcher could not obtain AWS credentials after "
+ "%d attempts. Set AWS_PROFILE to a profile whose "
+ "~/.aws/config entry uses credential_process (which "
+ "auto-refreshes), and ensure it is in the bitbake "
+ "env passthrough. Last error: %s"
+ % (attempts, last_exc))
+ return _s3_client
+
+
class S3(FetchMethod):
- """Class to fetch urls via 'aws s3'"""
+ """Class to fetch urls via boto3 (preferred) or 'aws s3' (fallback)"""
def supports(self, ud, d):
"""
@@ -80,30 +194,104 @@ class S3(FetchMethod):
ud.localfile = ud.basename
- ud.basecmd = shlex.split(d.getVar("FETCHCMD_s3") or "") or ["aws", "s3"]
+ ud.basecmd = d.getVar("FETCHCMD_s3") or "/usr/bin/env aws s3"
- def download(self, ud, d):
- """
- Fetch urls
- Assumes localpath was called first
- """
+ # ------------------------------------------------------------------
+ # boto3-based implementation
+ # ------------------------------------------------------------------
+ def _download_boto3(self, ud, d):
+ client = _get_s3_client(d)
+ # ud.host is the bucket; ud.path starts with "/" so strip it for the
+ # key. ud.path is percent-encoded (bitbake's mirror rewrite encodes the
+ # ':' in sstate names as %3A); boto3 wants the RAW, decoded key and does
+ # its own signing-time encoding, so decode here or S3 returns 400 Bad
+ # Request from the double-encoding.
+ bucket = ud.host
+ key = urllib.parse.unquote(ud.path.lstrip('/'))
+
+ bb.fetch.check_network_access(d, "boto3 get_object s3://%s/%s" % (bucket, key), ud.url)
+
+ try:
+ client.download_file(bucket, key, ud.localpath)
+ except botocore.exceptions.ClientError as e:
+ raise FetchError("The boto3 download for s3://%s/%s failed: %s" % (bucket, key, e))
+ except botocore.exceptions.BotoCoreError as e:
+ raise FetchError("The boto3 download for s3://%s/%s failed: %s" % (bucket, key, e))
+
+ def _checkstatus_boto3(self, ud, d):
+ client = _get_s3_client(d)
+ bucket = ud.host
+ # See _download_boto3: decode the percent-encoded key for boto3.
+ key = urllib.parse.unquote(ud.path.lstrip('/'))
- cmd = ud.basecmd + ['cp', 's3://%s%s' % (ud.host, ud.path), ud.localpath]
+ bb.fetch.check_network_access(d, "boto3 head_object s3://%s/%s" % (bucket, key), ud.url)
+
+ try:
+ client.head_object(Bucket=bucket, Key=key)
+ except botocore.exceptions.ClientError as e:
+ # A 404 is an ordinary sstate miss (object not in the mirror) and is
+ # expected for any task the mirror wasn't populated for -- log it at
+ # debug level only so normal misses don't flood the console. Any
+ # other code (400/403/5xx) is a real problem: surface it as a note.
+ err = e.response.get("Error", {})
+ meta = e.response.get("ResponseMetadata", {})
+ status = meta.get("HTTPStatusCode")
+ detail = ("head_object s3://%s/%s -> code=%s status=%s msg=%r reqid=%s"
+ % (bucket, key, err.get("Code"), status,
+ err.get("Message"), meta.get("RequestId")))
+ if status == 404:
+ bb.debug(2, "S3 fetcher: miss (404) s3://%s/%s" % (bucket, key))
+ else:
+ bb.note("S3 fetcher: %s" % detail)
+ raise FetchError("The boto3 head_object for s3://%s/%s failed: %s" % (bucket, key, e))
+ except botocore.exceptions.BotoCoreError as e:
+ raise FetchError("The boto3 head_object for s3://%s/%s failed: %s" % (bucket, key, e))
+ return True
+
+ # ------------------------------------------------------------------
+ # AWS CLI fallback (original behaviour)
+ # ------------------------------------------------------------------
+ def _download_cli(self, ud, d):
+ cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath)
bb.fetch.check_network_access(d, cmd, ud.url)
progresshandler = S3ProgressHandler(d)
runfetchcmd(cmd, d, False, log=progresshandler)
+ def _checkstatus_cli(self, ud, d):
+ cmd = '%s ls s3://%s%s' % (ud.basecmd, ud.host, ud.path)
+ bb.fetch.check_network_access(d, cmd, ud.url)
+ output = runfetchcmd(cmd, d)
+
+ # "aws s3 ls s3://mybucket/foo" will exit with success even if the file
+ # is not found, so check output of the command to confirm success.
+ if not output:
+ raise FetchError("The aws ls command for s3://%s%s gave empty output" % (ud.host, ud.path))
+ return True
+
+ # ------------------------------------------------------------------
+ # Dispatch
+ # ------------------------------------------------------------------
+ def download(self, ud, d):
+ """
+ Fetch urls
+ Assumes localpath was called first
+ """
+ if HAVE_BOTO3:
+ self._download_boto3(ud, d)
+ else:
+ self._download_cli(ud, d)
+
# Additional sanity checks copied from the wget class (although there
- # are no known issues which mean these are required, treat the aws cli
- # tool with a little healthy suspicion).
+ # are no known issues which mean these are required, treat the fetch
+ # result with a little healthy suspicion).
if not os.path.exists(ud.localpath):
- raise FetchError("The aws cp command returned success for s3://%s%s but %s doesn't exist?!" % (ud.host, ud.path, ud.localpath))
+ raise FetchError("The s3 fetch returned success for s3://%s%s but %s doesn't exist?!" % (ud.host, ud.path, ud.localpath))
if os.path.getsize(ud.localpath) == 0:
os.remove(ud.localpath)
- raise FetchError("The aws cp command for s3://%s%s resulted in a zero size file?! Deleting and failing since this isn't right." % (ud.host, ud.path))
+ raise FetchError("The s3 fetch for s3://%s%s resulted in a zero size file?! Deleting and failing since this isn't right." % (ud.host, ud.path))
return True
@@ -111,15 +299,6 @@ class S3(FetchMethod):
"""
Check the status of a URL
"""
-
- cmd = ud.basecmd + ['ls', 's3://%s%s' % (ud.host, ud.path)]
- bb.fetch.check_network_access(d, cmd, ud.url)
- output = runfetchcmd(cmd, d)
-
- # "aws s3 ls s3://mybucket/foo" will exit with success even if the file
- # is not found, so check output of the command to confirm success.
-
- if not output:
- raise FetchError("The aws ls command for s3://%s%s gave empty output" % (ud.host, ud.path))
-
- return True
+ if HAVE_BOTO3:
+ return self._checkstatus_boto3(ud, d)
+ return self._checkstatus_cli(ud, d)