diff --git a/lib/bb/fetch/s3.py b/lib/bb/fetch/s3.py index ea3dc28fc..4df582a0b 100644 --- a/lib/bb/fetch/s3.py +++ b/lib/bb/fetch/s3.py @@ -1,8 +1,23 @@ """ BitBake 'Fetch' implementation for Amazon AWS S3. -Class for fetching files from Amazon S3 using the AWS Command Line Interface. -The aws tool must be correctly installed and configured prior to use. +Class for fetching files from Amazon S3. + +Two backends are provided and one is selected automatically when the module is +imported: + + S3Boto (preferred) + Fetch in-process with the boto3 SDK, talking to S3 over HTTPS natively + (no per-object subprocess / AWS CLI startup cost). This is a big win for + sstate mirrors, where thousands of small objects are fetched and the AWS + CLI's interpreter + config/credential resolution startup dominates wall + time. Selected when boto3 is importable. + + S3CLI (fallback) + Fetch by shelling out to the AWS CLI ("aws s3 cp" / "aws s3 ls"). This is + the original behaviour and requires no extra Python packages; the aws + tool must be installed and configured. Selected when boto3 is not + importable, so there is no new hard dependency. """ @@ -17,6 +32,8 @@ The aws tool must be correctly installed and configured prior to use. import os import bb +import time +import random import urllib.request, urllib.parse, urllib.error import re import shlex @@ -24,6 +41,13 @@ from bb.fetch import FetchMethod from bb.fetch import FetchError from bb.fetch import runfetchcmd +try: + import boto3 + import botocore +except ImportError: + boto3 = None + + def convertToBytes(value, unit): value = float(value) if (unit == "KiB"): @@ -60,8 +84,107 @@ class S3ProgressHandler(bb.progress.LineFilterProgressHandler): return True -class S3(FetchMethod): - """Class to fetch urls via 'aws s3'""" +# Cache a single boto3 S3 client for the lifetime of the (worker) process. +# Creating a client resolves credentials and config once; reusing it keeps the +# underlying HTTPS connection pool warm across many small fetches. +# +# NOTE: bitbake runs setscene fetches in SEPARATE worker processes, so this +# cache is per-process, not global to the build. Credentials must come from a +# source that AUTO-REFRESHES, because a Yocto build runs for hours and any +# static/temporary credential (e.g. a fixed access key written into +# ~/.aws/credentials [default]) can EXPIRE partway through -- after which S3 +# returns "400 Bad Request" on every call and all setscene tasks fail. +# +# The robust source is a `credential_process` profile: boto3 invokes it on +# demand and re-vends when the token nears expiry. We therefore resolve +# AWS_PROFILE from the bitbake datastore (or the environment) and pin it on the +# session, so the worker uses that auto-refreshing profile instead of a stale +# static [default]. Set AWS_PROFILE for the build (e.g. via +# BB_ENV_PASSTHROUGH_ADDITIONS so it survives bitbake's worker env scrubbing) +# to a profile whose [profile ...] entry in ~/.aws/config uses +# credential_process. +# +# A credential_process helper that caches vended credentials on disk will +# mostly serve them from that warm cache rather than from its backing service; +# the backoff below still covers a transient "Rate exceeded" during a +# cold-cache storm. +_s3_client = None + +def _get_s3_client(d): + global _s3_client + if _s3_client is None: + # Resolve region from the usual env vars; boto3 also honours + # ~/.aws/config and AWS_CONFIG_FILE. (S3 handles cross-region HEAD/GET, + # so an exact match is not required, but pinning avoids surprises.) + region = (d.getVar("AWS_DEFAULT_REGION") + or os.environ.get("AWS_DEFAULT_REGION") + or os.environ.get("AWS_REGION")) + # Resolve the AWS profile to use. Pinning it defeats a stale/expired + # static [default] in ~/.aws/credentials shadowing the auto-refreshing + # credential_process -- the exact cause of mid-build "400 Bad Request" + # failures. d.getVar wins so it works even when the env is scrubbed in + # a worker; env is the fallback when it is passed through. + profile = (d.getVar("AWS_PROFILE") + or os.environ.get("AWS_PROFILE") + or os.environ.get("AWS_DEFAULT_PROFILE")) + # Allow more concurrent connections than the default (10) so parallel + # fetches within a process share a healthy pool. The 'retries' here + # applies to S3 API calls only, NOT to credential resolution. + cfg = botocore.config.Config( + max_pool_connections=50, + retries={"max_attempts": 5, "mode": "standard"}, + ) + + # Credential resolution happens lazily on first API call OR eagerly in + # client(); either way a credential_process "Rate exceeded" surfaces as + # CredentialRetrievalError. Retry client creation + a cheap credential + # resolution with exponential backoff + jitter so a transient + # credential-service rate limit does not fail the whole setscene task. + attempts = int(d.getVar("S3_CRED_MAX_ATTEMPTS") or 6) + last_exc = None + for attempt in range(attempts): + try: + session = boto3.session.Session(profile_name=profile, region_name=region) + client = session.client("s3", config=cfg) + # Force credential resolution now so we retry it here rather + # than failing deep inside a download/head call. + creds = session.get_credentials() + # One-time-per-worker diagnostic: which credential source won, + # the profile, and the region. Tells us whether the worker + # picked the auto-refreshing credential_process profile or fell + # back to a stale static default. + try: + method = getattr(creds, "method", None) if creds else "NONE" + bb.debug(1, "S3 fetcher: credential method=%s profile=%s region=%s" + % (method, profile, client.meta.region_name)) + except Exception: + pass + _s3_client = client + break + except botocore.exceptions.CredentialRetrievalError as e: + last_exc = e + if attempt == attempts - 1: + break + # Exponential backoff with jitter: 0.5,1,2,4,8s (+/- jitter), + # capped. Spreads out many workers refreshing credentials at once. + delay = min(2.0 ** attempt * 0.5, 15.0) + delay += random.uniform(0, delay / 2.0) + bb.note("S3 fetcher: credential retrieval rate-limited " + "(attempt %d/%d), backing off %.1fs: %s" + % (attempt + 1, attempts, delay, e)) + time.sleep(delay) + if _s3_client is None: + raise FetchError("S3 fetcher could not obtain AWS credentials after " + "%d attempts. Set AWS_PROFILE to a profile whose " + "~/.aws/config entry uses credential_process (which " + "auto-refreshes), and ensure it is in the bitbake " + "env passthrough. Last error: %s" + % (attempts, last_exc)) + return _s3_client + + +class S3Common(FetchMethod): + """Behaviour shared by the S3 fetcher backends (S3Boto and S3CLI).""" def supports(self, ud, d): """ @@ -80,46 +203,114 @@ class S3(FetchMethod): ud.localfile = ud.basename - ud.basecmd = shlex.split(d.getVar("FETCHCMD_s3") or "") or ["aws", "s3"] - def download(self, ud, d): """ Fetch urls Assumes localpath was called first """ - - cmd = ud.basecmd + ['cp', 's3://%s%s' % (ud.host, ud.path), ud.localpath] - bb.fetch.check_network_access(d, cmd, ud.url) - - progresshandler = S3ProgressHandler(d) - runfetchcmd(cmd, d, False, log=progresshandler) + self._fetch(ud, d) # Additional sanity checks copied from the wget class (although there - # are no known issues which mean these are required, treat the aws cli - # tool with a little healthy suspicion). + # are no known issues which mean these are required, treat the fetch + # result with a little healthy suspicion). if not os.path.exists(ud.localpath): - raise FetchError("The aws cp command returned success for s3://%s%s but %s doesn't exist?!" % (ud.host, ud.path, ud.localpath)) + raise FetchError("The s3 fetch returned success for s3://%s%s but %s doesn't exist?!" % (ud.host, ud.path, ud.localpath)) if os.path.getsize(ud.localpath) == 0: os.remove(ud.localpath) - raise FetchError("The aws cp command for s3://%s%s resulted in a zero size file?! Deleting and failing since this isn't right." % (ud.host, ud.path)) + raise FetchError("The s3 fetch for s3://%s%s resulted in a zero size file?! Deleting and failing since this isn't right." % (ud.host, ud.path)) return True + +class S3Boto(S3Common): + """Fetch urls in-process using the boto3 SDK.""" + + def _fetch(self, ud, d): + client = _get_s3_client(d) + # ud.host is the bucket; ud.path starts with "/" so strip it for the + # key. ud.path is percent-encoded (bitbake's mirror rewrite encodes the + # ':' in sstate names as %3A); boto3 wants the RAW, decoded key and does + # its own signing-time encoding, so decode here or S3 returns 400 Bad + # Request from the double-encoding. + bucket = ud.host + key = urllib.parse.unquote(ud.path.lstrip('/')) + + bb.fetch.check_network_access(d, "boto3 get_object s3://%s/%s" % (bucket, key), ud.url) + + try: + client.download_file(bucket, key, ud.localpath) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + raise FetchError("The boto3 download for s3://%s/%s failed: %s" % (bucket, key, e)) + def checkstatus(self, fetch, ud, d): """ Check the status of a URL """ + client = _get_s3_client(d) + bucket = ud.host + # See _fetch: decode the percent-encoded key for boto3. + key = urllib.parse.unquote(ud.path.lstrip('/')) + + bb.fetch.check_network_access(d, "boto3 head_object s3://%s/%s" % (bucket, key), ud.url) + + try: + client.head_object(Bucket=bucket, Key=key) + except botocore.exceptions.ClientError as e: + # A 404 is an ordinary sstate miss (object not in the mirror) and is + # expected for any task the mirror wasn't populated for -- log it at + # debug level only so normal misses don't flood the console. Any + # other code (400/403/5xx) is a real problem: surface it as a note. + err = e.response.get("Error", {}) + meta = e.response.get("ResponseMetadata", {}) + status = meta.get("HTTPStatusCode") + detail = ("head_object s3://%s/%s -> code=%s status=%s msg=%r reqid=%s" + % (bucket, key, err.get("Code"), status, + err.get("Message"), meta.get("RequestId"))) + if status == 404: + bb.debug(2, "S3 fetcher: miss (404) s3://%s/%s" % (bucket, key)) + else: + bb.note("S3 fetcher: %s" % detail) + raise FetchError("The boto3 head_object for s3://%s/%s failed: %s" % (bucket, key, e)) + except botocore.exceptions.BotoCoreError as e: + raise FetchError("The boto3 head_object for s3://%s/%s failed: %s" % (bucket, key, e)) + return True + + +class S3CLI(S3Common): + """Fetch urls by shelling out to the AWS CLI ('aws s3').""" + + def urldata_init(self, ud, d): + super().urldata_init(ud, d) + ud.basecmd = shlex.split(d.getVar("FETCHCMD_s3") or "") or ["aws", "s3"] + + def _fetch(self, ud, d): + cmd = ud.basecmd + ['cp', 's3://%s%s' % (ud.host, ud.path), ud.localpath] + bb.fetch.check_network_access(d, cmd, ud.url) + progresshandler = S3ProgressHandler(d) + runfetchcmd(cmd, d, False, log=progresshandler) + + def checkstatus(self, fetch, ud, d): + """ + Check the status of a URL + """ cmd = ud.basecmd + ['ls', 's3://%s%s' % (ud.host, ud.path)] bb.fetch.check_network_access(d, cmd, ud.url) output = runfetchcmd(cmd, d) # "aws s3 ls s3://mybucket/foo" will exit with success even if the file # is not found, so check output of the command to confirm success. - if not output: raise FetchError("The aws ls command for s3://%s%s gave empty output" % (ud.host, ud.path)) - return True + + +# Pick the backend once, at import time: prefer boto3, fall back to the AWS CLI +# when boto3 is not importable. bb.fetch registers s3.S3, so this selection is +# transparent to the rest of bitbake. +if boto3 is not None: + S3 = S3Boto +else: + S3 = S3CLI