diff --git a/lib/bb/fetch/s3.py b/lib/bb/fetch/s3.py index ea3dc28fc..e919a4ba3 100644 --- a/lib/bb/fetch/s3.py +++ b/lib/bb/fetch/s3.py @@ -1,8 +1,16 @@ """ BitBake 'Fetch' implementation for Amazon AWS S3. -Class for fetching files from Amazon S3 using the AWS Command Line Interface. -The aws tool must be correctly installed and configured prior to use. +Class for fetching files from Amazon S3. + +By default this uses the boto3 Python SDK, which talks to S3 over HTTPS +natively (no per-object subprocess / AWS CLI startup cost). This is a big win +for sstate mirrors, where thousands of small objects are fetched and the AWS +CLI's interpreter + config/credential resolution startup dominates wall time. + +If boto3 is not importable, the fetcher transparently falls back to shelling +out to the AWS CLI ("aws s3 cp" / "aws s3 ls"), preserving the original +behaviour. The AWS tool must then be correctly installed and configured. """ @@ -17,13 +25,22 @@ The aws tool must be correctly installed and configured prior to use. import os import bb +import time +import random import urllib.request, urllib.parse, urllib.error import re -import shlex from bb.fetch import FetchMethod from bb.fetch import FetchError from bb.fetch import runfetchcmd +try: + import boto3 + import botocore + HAVE_BOTO3 = True +except ImportError: + HAVE_BOTO3 = False + + def convertToBytes(value, unit): value = float(value) if (unit == "KiB"): @@ -60,8 +77,105 @@ class S3ProgressHandler(bb.progress.LineFilterProgressHandler): return True +# Cache a single boto3 S3 client for the lifetime of the (worker) process. +# Creating a client resolves credentials and config once; reusing it keeps the +# underlying HTTPS connection pool warm across many small fetches. +# +# NOTE: bitbake runs setscene fetches in SEPARATE worker processes, so this +# cache is per-process, not global to the build. Credentials must come from a +# source that AUTO-REFRESHES, because a Yocto build runs for hours and any +# static/temporary credential (e.g. `ada credentials update --once` written +# into ~/.aws/credentials [default]) will EXPIRE partway through -- after which +# S3 returns "400 Bad Request" on every call and all setscene tasks fail. +# +# The robust source is a `credential_process` profile (e.g. ada): boto3 invokes +# it on demand and re-vends when the token nears expiry. We therefore resolve +# AWS_PROFILE from the bitbake datastore (or the environment) and pin it on the +# session, so the worker uses that auto-refreshing profile instead of a stale +# static [default]. Set `AWS_PROFILE` in the build config (it is listed in the +# bitbake-setup config's bb-env-passthrough-additions) to a profile whose +# [profile ...] entry in ~/.aws/config uses credential_process. +# +# ada caches vended credentials on disk, so the per-worker credential_process +# calls mostly hit that warm cache rather than Conduit; the backoff below still +# covers a transient "Rate exceeded" during a cold-cache storm. +_s3_client = None + +def _get_s3_client(d): + global _s3_client + if _s3_client is None: + # Resolve region from the usual env vars; boto3 also honours + # ~/.aws/config and AWS_CONFIG_FILE. (S3 handles cross-region HEAD/GET, + # so an exact match is not required, but pinning avoids surprises.) + region = (d.getVar("AWS_DEFAULT_REGION") + or os.environ.get("AWS_DEFAULT_REGION") + or os.environ.get("AWS_REGION")) + # Resolve the AWS profile to use. Pinning it defeats a stale/expired + # static [default] in ~/.aws/credentials shadowing the auto-refreshing + # credential_process -- the exact cause of mid-build "400 Bad Request" + # failures. d.getVar wins so it works even when the env is scrubbed in + # a worker; env is the fallback when it is passed through. + profile = (d.getVar("AWS_PROFILE") + or os.environ.get("AWS_PROFILE") + or os.environ.get("AWS_DEFAULT_PROFILE")) + # Allow more concurrent connections than the default (10) so parallel + # fetches within a process share a healthy pool. The 'retries' here + # applies to S3 API calls only, NOT to credential resolution. + cfg = botocore.config.Config( + max_pool_connections=50, + retries={"max_attempts": 5, "mode": "standard"}, + ) + + # Credential resolution happens lazily on first API call OR eagerly in + # client(); either way a credential_process "Rate exceeded" surfaces as + # CredentialRetrievalError. Retry client creation + a cheap credential + # resolution with exponential backoff + jitter so a transient ada/ + # Conduit rate limit does not fail the whole setscene task. + attempts = int(d.getVar("S3_CRED_MAX_ATTEMPTS") or 6) + last_exc = None + for attempt in range(attempts): + try: + session = boto3.session.Session(profile_name=profile, region_name=region) + client = session.client("s3", config=cfg) + # Force credential resolution now so we retry it here rather + # than failing deep inside a download/head call. + creds = session.get_credentials() + # One-time-per-worker diagnostic: which credential source won, + # the profile, and the region. Tells us whether the worker + # picked the auto-refreshing credential_process profile or fell + # back to a stale static default. + try: + method = getattr(creds, "method", None) if creds else "NONE" + bb.debug(1, "S3 fetcher: credential method=%s profile=%s region=%s" + % (method, profile, client.meta.region_name)) + except Exception: + pass + _s3_client = client + break + except botocore.exceptions.CredentialRetrievalError as e: + last_exc = e + if attempt == attempts - 1: + break + # Exponential backoff with jitter: 0.5,1,2,4,8s (+/- jitter), + # capped. Spreads out many workers hitting ada at once. + delay = min(2.0 ** attempt * 0.5, 15.0) + delay += random.uniform(0, delay / 2.0) + bb.note("S3 fetcher: credential retrieval rate-limited " + "(attempt %d/%d), backing off %.1fs: %s" + % (attempt + 1, attempts, delay, e)) + time.sleep(delay) + if _s3_client is None: + raise FetchError("S3 fetcher could not obtain AWS credentials after " + "%d attempts. Set AWS_PROFILE to a profile whose " + "~/.aws/config entry uses credential_process (which " + "auto-refreshes), and ensure it is in the bitbake " + "env passthrough. Last error: %s" + % (attempts, last_exc)) + return _s3_client + + class S3(FetchMethod): - """Class to fetch urls via 'aws s3'""" + """Class to fetch urls via boto3 (preferred) or 'aws s3' (fallback)""" def supports(self, ud, d): """ @@ -80,30 +194,104 @@ class S3(FetchMethod): ud.localfile = ud.basename - ud.basecmd = shlex.split(d.getVar("FETCHCMD_s3") or "") or ["aws", "s3"] + ud.basecmd = d.getVar("FETCHCMD_s3") or "/usr/bin/env aws s3" - def download(self, ud, d): - """ - Fetch urls - Assumes localpath was called first - """ + # ------------------------------------------------------------------ + # boto3-based implementation + # ------------------------------------------------------------------ + def _download_boto3(self, ud, d): + client = _get_s3_client(d) + # ud.host is the bucket; ud.path starts with "/" so strip it for the + # key. ud.path is percent-encoded (bitbake's mirror rewrite encodes the + # ':' in sstate names as %3A); boto3 wants the RAW, decoded key and does + # its own signing-time encoding, so decode here or S3 returns 400 Bad + # Request from the double-encoding. + bucket = ud.host + key = urllib.parse.unquote(ud.path.lstrip('/')) + + bb.fetch.check_network_access(d, "boto3 get_object s3://%s/%s" % (bucket, key), ud.url) + + try: + client.download_file(bucket, key, ud.localpath) + except botocore.exceptions.ClientError as e: + raise FetchError("The boto3 download for s3://%s/%s failed: %s" % (bucket, key, e)) + except botocore.exceptions.BotoCoreError as e: + raise FetchError("The boto3 download for s3://%s/%s failed: %s" % (bucket, key, e)) + + def _checkstatus_boto3(self, ud, d): + client = _get_s3_client(d) + bucket = ud.host + # See _download_boto3: decode the percent-encoded key for boto3. + key = urllib.parse.unquote(ud.path.lstrip('/')) - cmd = ud.basecmd + ['cp', 's3://%s%s' % (ud.host, ud.path), ud.localpath] + bb.fetch.check_network_access(d, "boto3 head_object s3://%s/%s" % (bucket, key), ud.url) + + try: + client.head_object(Bucket=bucket, Key=key) + except botocore.exceptions.ClientError as e: + # A 404 is an ordinary sstate miss (object not in the mirror) and is + # expected for any task the mirror wasn't populated for -- log it at + # debug level only so normal misses don't flood the console. Any + # other code (400/403/5xx) is a real problem: surface it as a note. + err = e.response.get("Error", {}) + meta = e.response.get("ResponseMetadata", {}) + status = meta.get("HTTPStatusCode") + detail = ("head_object s3://%s/%s -> code=%s status=%s msg=%r reqid=%s" + % (bucket, key, err.get("Code"), status, + err.get("Message"), meta.get("RequestId"))) + if status == 404: + bb.debug(2, "S3 fetcher: miss (404) s3://%s/%s" % (bucket, key)) + else: + bb.note("S3 fetcher: %s" % detail) + raise FetchError("The boto3 head_object for s3://%s/%s failed: %s" % (bucket, key, e)) + except botocore.exceptions.BotoCoreError as e: + raise FetchError("The boto3 head_object for s3://%s/%s failed: %s" % (bucket, key, e)) + return True + + # ------------------------------------------------------------------ + # AWS CLI fallback (original behaviour) + # ------------------------------------------------------------------ + def _download_cli(self, ud, d): + cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath) bb.fetch.check_network_access(d, cmd, ud.url) progresshandler = S3ProgressHandler(d) runfetchcmd(cmd, d, False, log=progresshandler) + def _checkstatus_cli(self, ud, d): + cmd = '%s ls s3://%s%s' % (ud.basecmd, ud.host, ud.path) + bb.fetch.check_network_access(d, cmd, ud.url) + output = runfetchcmd(cmd, d) + + # "aws s3 ls s3://mybucket/foo" will exit with success even if the file + # is not found, so check output of the command to confirm success. + if not output: + raise FetchError("The aws ls command for s3://%s%s gave empty output" % (ud.host, ud.path)) + return True + + # ------------------------------------------------------------------ + # Dispatch + # ------------------------------------------------------------------ + def download(self, ud, d): + """ + Fetch urls + Assumes localpath was called first + """ + if HAVE_BOTO3: + self._download_boto3(ud, d) + else: + self._download_cli(ud, d) + # Additional sanity checks copied from the wget class (although there - # are no known issues which mean these are required, treat the aws cli - # tool with a little healthy suspicion). + # are no known issues which mean these are required, treat the fetch + # result with a little healthy suspicion). if not os.path.exists(ud.localpath): - raise FetchError("The aws cp command returned success for s3://%s%s but %s doesn't exist?!" % (ud.host, ud.path, ud.localpath)) + raise FetchError("The s3 fetch returned success for s3://%s%s but %s doesn't exist?!" % (ud.host, ud.path, ud.localpath)) if os.path.getsize(ud.localpath) == 0: os.remove(ud.localpath) - raise FetchError("The aws cp command for s3://%s%s resulted in a zero size file?! Deleting and failing since this isn't right." % (ud.host, ud.path)) + raise FetchError("The s3 fetch for s3://%s%s resulted in a zero size file?! Deleting and failing since this isn't right." % (ud.host, ud.path)) return True @@ -111,15 +299,6 @@ class S3(FetchMethod): """ Check the status of a URL """ - - cmd = ud.basecmd + ['ls', 's3://%s%s' % (ud.host, ud.path)] - bb.fetch.check_network_access(d, cmd, ud.url) - output = runfetchcmd(cmd, d) - - # "aws s3 ls s3://mybucket/foo" will exit with success even if the file - # is not found, so check output of the command to confirm success. - - if not output: - raise FetchError("The aws ls command for s3://%s%s gave empty output" % (ud.host, ud.path)) - - return True + if HAVE_BOTO3: + return self._checkstatus_boto3(ud, d) + return self._checkstatus_cli(ud, d)