@@ -79,6 +79,19 @@ class FetchError(BBFetchException):
BBFetchException.__init__(self, msg)
self.args = (message, url)
+class Server5xxError(Exception):
+ """exception for 5xx server errors"""
+ def __init__(self, url, message="Server error (5xx)"):
+ self.url = url
+ self.msg = message
+ parsed = urllib.parse.urlparse(url)
+ self.mirror_host = parsed.netloc
+ super().__init__(self.msg)
+
+ def __str__(self):
+ return f"{self.msg} ({self.url})"
+
+
class ChecksumError(FetchError):
"""Exception when mismatched checksum encountered"""
def __init__(self, message, url = None, checksum = None):
@@ -1073,7 +1086,12 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
try:
if check:
found = ud.method.checkstatus(fetch, ud, ld)
- if found:
+ if found is None:
+ parsed = urllib.parse.urlparse(ud.url)
+ mirror_host = parsed.netloc
+ # Raise errors until we can reorganize the mirror polling cycle
+ raise Server5xxError(ud.url, f"Server error (5xx) for mirror {mirror_host}")
+ elif found:
return found
return False
@@ -1167,10 +1185,33 @@ def try_mirrors(fetch, d, origud, mirrors, check = False):
uris, uds = build_mirroruris(origud, mirrors, ld)
+ '''
+ TBD: The issue with this loop is that if we don't interrupt it, there will be no effect, and we will iterate through all the addresses from build_mirror_uris.
+ If we abruptly interrupt the loop, the progress of previous polls will be lost. We need to preserve the progress, for example, by restructuring it into a while loop.
+ In this loop, we can iterate and, upon encountering an error, filter out only those addresses that haven't been processed yet.
+ '''
for index, uri in enumerate(uris):
- ret = try_mirror_url(fetch, origud, uds[index], ld, check)
- if ret:
- return ret
+ try:
+ ret = try_mirror_url(fetch, origud, uds[index], ld, check)
+ if ret:
+ return ret
+ except Exception as e:
+ if isinstance(e, Server5xxError):
+ global_d = fetch.d
+ sstate_mirrors = global_d.getVar('SSTATE_MIRRORS') or ""
+ new_mirrors = []
+ '''
+ Add mirror filtering — eliminate those that caused errors (.mirror_host) !!!
+ '''
+ new_value = " ".join(new_mirrors)
+ global_d.setVar('SSTATE_MIRRORS', new_value)
+ logger.warning("Removed mirror %s from SSTATE_MIRRORS due to server error", e.mirror_host)
+ continue
+ else:
+ raise
+ '''
+ After this, it is necessary to change d via setVar so that when calling try_mirrors, mirrors that return a server error do not appear.
+ '''
return None
def trusted_network(d, url):
@@ -401,6 +401,25 @@ class Wget(FetchMethod):
with opener.open(r, timeout=100) as response:
pass
except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e:
+ errno_code = None
+ if hasattr(e, 'errno') and e.errno:
+ errno_code = e.errno
+ elif hasattr(e, 'reason') and hasattr(e.reason, 'errno') and e.reason.errno:
+ errno_code = e.reason.errno
+ # Server (network) errors
+ server_side_errors = {
+ errno.ECONNREFUSED,
+ errno.ETIMEDOUT,
+ errno.EHOSTUNREACH,
+ errno.ENETUNREACH,
+ errno.ECONNABORTED,
+ errno.ECONNRESET,
+ }
+
+ if errno_code in server_side_errors:
+ # Return None to have information one level higher about why it was not possible
+ # to get the file -- if there was a server error, then None
+ return None
if try_again:
logger.debug2("checkstatus: trying again")
return self.checkstatus(fetch, ud, d, False)
I am sending the second idea regarding the filtering of unavailable servers in SSTATE_MIRRORS. The first version of the patch was left unattended; it is available at the link: https://lists.openembedded.org/g/bitbake-devel/message/17595 In the first version, a simple availability check is performed at the stage of reading local.conf. This might slightly disrupt the logic, so I created a second version of the patch (the current version does not work—it's just an idea of where the logic can be implemented; I want to hear your opinion on this—whether it was worth dealing with). This version globally performs the same task—preventing additional load when specifying partially non-working servers in SSTATE_MIRRORS. The main idea behind the changes is to check what error occurred when trying to retrieve a file from a remote server. If the error is one of the server-related errors (for example, Service Unavailable) or another error that makes it impossible to use the server causing the problem, then requests should not be sent to such a "non-working" server. As I understand it, for each sstate-cache artifact, three requests are sent to the server, which creates a large unnecessary load if the server is generally unavailable. Please consider this idea and, if possible, the previous version as well. Thank you, and have a good day! Signed-off-by: KonstantinKondratenko <sider123456789101112131415@gmail.com> --- bitbake/lib/bb/fetch2/__init__.py | 49 ++++++++++++++++++++++++++++--- bitbake/lib/bb/fetch2/wget.py | 19 ++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-)