From patchwork Fri Jul 5 14:46:37 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 46057 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 9673EC38150 for ; Fri, 5 Jul 2024 14:47:28 +0000 (UTC) Received: from relay1-d.mail.gandi.net (relay1-d.mail.gandi.net [217.70.183.193]) by mx.groups.io with SMTP id smtpd.web11.18861.1720190846017143836 for ; Fri, 05 Jul 2024 07:47:26 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=HiFVz5+k; spf=pass (domain: bootlin.com, ip: 217.70.183.193, mailfrom: alexis.lothore@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 523D3240007; Fri, 5 Jul 2024 14:47:24 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1720190844; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=8fbFzla7cMRg7Y6cQp5b1+Dg2yG0K17NDLgH1NqhM70=; b=HiFVz5+kMX9df1wMFxU7jCtg03c2pQNQZoi6rubPApE8zA+adoeX/0gI4fqCcvqa5fxnPN fAv8O33tQ0NQ/XczaWU7ZZF8vxHBBoUNI8ea7yKlrUF2BbZ6rjy6mkU1AM+0TmtTCLvTAi PLenwsCpzA9t0cWQahbIWb6aMnPHNdTRjWKfVJUc/N1XZ1xTAZo39WzUPI+D4hOQgbYp+z NGGlmTWNuGk+p7PKE6oqxxIMlBtNi9EzbdKebyzcHoIyu61n9jQVnaVe7K/g62yGOpqIDW MH+Igw3TgQwJYqbXkb3GAMrrFmTAKNRqPAHVqRltr1UyB9VvHtSYX0djVJZLcg== From: =?utf-8?q?Alexis_Lothor=C3=A9?= To: Cc: Thomas Petazzoni , Alexandre Belloni Subject: [PATCH 1/2] oeqa/ssh: allow to retrieve raw, unformatted ouput Date: Fri, 5 Jul 2024 16:46:37 +0200 Message-ID: <20240705144638.441976-2-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.45.2 In-Reply-To: <20240705144638.441976-1-alexis.lothore@bootlin.com> References: <20240705144638.441976-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: alexis.lothore@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 05 Jul 2024 14:47:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/201611 From: Alexis Lothoré The ssh target is currently well tailored to easily retrieve textual output from a command run on a remote target. It could also be used to retrieve raw data from a command run onto a remote target (for example, to feed this data directly to another program), but it currently suffers two minor issues preventing such use case: - stderr is piped to stdout, so any error log will be mixed in the program output - the final output is decoded as utf-8 and stripped Allow to return the raw, unmodified output by adding an optional "raw" parameter. Keep it to False by default to preserve the current behavior. When enabled, do not return a string but the raw output as bytes. Signed-off-by: Alexis Lothoré --- meta/lib/oeqa/core/target/ssh.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/meta/lib/oeqa/core/target/ssh.py b/meta/lib/oeqa/core/target/ssh.py index 09cdd14c759d..d4734693848e 100644 --- a/meta/lib/oeqa/core/target/ssh.py +++ b/meta/lib/oeqa/core/target/ssh.py @@ -55,14 +55,14 @@ class OESSHTarget(OETarget): def stop(self, **kwargs): pass - def _run(self, command, timeout=None, ignore_status=True): + def _run(self, command, timeout=None, ignore_status=True, raw=False): """ Runs command in target using SSHProcess. """ self.logger.debug("[Running]$ %s" % " ".join(command)) starttime = time.time() - status, output = SSHCall(command, self.logger, timeout) + status, output = SSHCall(command, self.logger, timeout, raw) self.logger.debug("[Command returned '%d' after %.2f seconds]" "" % (status, time.time() - starttime)) @@ -72,7 +72,7 @@ class OESSHTarget(OETarget): return (status, output) - def run(self, command, timeout=None, ignore_status=True): + def run(self, command, timeout=None, ignore_status=True, raw=False): """ Runs command in target. @@ -91,7 +91,7 @@ class OESSHTarget(OETarget): else: processTimeout = self.timeout - status, output = self._run(sshCmd, processTimeout, ignore_status) + status, output = self._run(sshCmd, processTimeout, ignore_status, raw) self.logger.debug('Command: %s\nStatus: %d Output: %s\n' % (command, status, output)) return (status, output) @@ -206,7 +206,7 @@ class OESSHTarget(OETarget): remoteDir = os.path.join(remotePath, tmpDir.lstrip("/")) self.deleteDir(remoteDir) -def SSHCall(command, logger, timeout=None, **opts): +def SSHCall(command, logger, timeout=None, raw=False, **opts): def run(): nonlocal output @@ -265,7 +265,7 @@ def SSHCall(command, logger, timeout=None, **opts): else: output_raw = process.communicate()[0] - output = output_raw.decode('utf-8', errors='ignore') + output = output_raw if raw else output_raw.decode('utf-8', errors='ignore') logger.debug('Data from SSH call:\n%s' % output.rstrip()) # timout or not, make sure process exits and is not hanging @@ -292,7 +292,7 @@ def SSHCall(command, logger, timeout=None, **opts): options = { "stdout": subprocess.PIPE, - "stderr": subprocess.STDOUT, + "stderr": subprocess.STDOUT if not raw else None, "stdin": None, "shell": False, "bufsize": -1, @@ -320,4 +320,4 @@ def SSHCall(command, logger, timeout=None, **opts): logger.debug('Something went wrong, killing SSH process') raise - return (process.returncode, output.rstrip()) + return (process.returncode, output if raw else output.rstrip()) From patchwork Fri Jul 5 14:46:38 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 46058 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A2CEDC41513 for ; Fri, 5 Jul 2024 14:47:28 +0000 (UTC) Received: from relay1-d.mail.gandi.net (relay1-d.mail.gandi.net [217.70.183.193]) by mx.groups.io with SMTP id smtpd.web11.18862.1720190846026453559 for ; Fri, 05 Jul 2024 07:47:26 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=ZT1GvDe+; spf=pass (domain: bootlin.com, ip: 217.70.183.193, mailfrom: alexis.lothore@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 95288240008; Fri, 5 Jul 2024 14:47:24 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1720190844; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=0oMkU1sM+0WVZ3+ZrW8MOJy9gTT8/wCbYp8QCMdI6Ds=; b=ZT1GvDe+xL5Led+WVum8JLeRuIhPhqGY7KrqMw8ILgf2b5MchqPMA3YC1tE2zbd5bYa1ZX 2cS4nTgpkUQZ6/g978sVdccZHa5agMKV7dR3kG8hCiUx9qfWb2EzBgriqSr9MKdIYTOtok vMbl5PwBTStislzHr6ODMTiV0/dK3acOrK4zrS/J5iIshlBEaVi/GRjjk+IBcEw/ATr2Ym 4dRu9B87Gx0+LQasU4OTGwrFF+qGiDJpUJOBd7jRK6A0UeD+CDPz7XCcOFiP+BE7It4Hog VFctOUi40m7xoMq5rJsK7bheFCcRmRUzZOVjGMG7eHcdlaCPWPQVC/bf5MJkiA== From: =?utf-8?q?Alexis_Lothor=C3=A9?= To: Cc: Thomas Petazzoni , Alexandre Belloni Subject: [PATCH 2/2] oeqa/utils/postactions: transfer whole archive over ssh instead of doing individual copies Date: Fri, 5 Jul 2024 16:46:38 +0200 Message-ID: <20240705144638.441976-3-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.45.2 In-Reply-To: <20240705144638.441976-1-alexis.lothore@bootlin.com> References: <20240705144638.441976-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: alexis.lothore@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 05 Jul 2024 14:47:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/201610 From: Alexis Lothoré Fixes [YOCTO 15536] The postactions retrieval actions currently rely on scp executed individually on any file or directory expanded from TESTIMAGE_FAILED_QA_ARTIFACTS. Unfortunately, symlinks are not preserved with this mechanism, which lead to big storage space consumption. Things may go even worse if those symlinks create some circular chains. This mechanism then needs to be updated to preserve symlinks instead of following them during copy. There are multiple ways to do it: - create a local archive on the target and execute scp on this file - use rsync instead of scp for all files - create an archive and pipe it to ssh instead of storing it onto the target The first solution may create pressure on targets storage space, while the second assumes that rsync is installed on the target, which may not be true. So the third one is a compromise: tar is very likely present, at least through busybox, and no disk space is used on the target. Replace the current per-file scp call by a single call to tar run on the target. Retrieve the generated compressed archive directly from SSH output, and feed it to another tar process but on host, to uncompress and extract it at the same place as before. Signed-off-by: Alexis Lothoré --- meta/lib/oeqa/utils/postactions.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/meta/lib/oeqa/utils/postactions.py b/meta/lib/oeqa/utils/postactions.py index ecdddd2d40e3..f7ce25f2af8e 100644 --- a/meta/lib/oeqa/utils/postactions.py +++ b/meta/lib/oeqa/utils/postactions.py @@ -62,17 +62,16 @@ def get_artifacts_list(target, raw_list): return result def retrieve_test_artifacts(target, artifacts_list, target_dir): + import io, subprocess local_artifacts_dir = os.path.join(target_dir, "artifacts") - for artifact_path in artifacts_list: - if not os.path.isabs(artifact_path): - bb.warn(f"{artifact_path} is not an absolute path") - continue - try: - dest_dir = os.path.join(local_artifacts_dir, os.path.dirname(artifact_path[1:])) - os.makedirs(dest_dir, exist_ok=True) - target.copyFrom(artifact_path, dest_dir) - except Exception as e: - bb.warn(f"Can not retrieve {artifact_path} from test target: {e}") + try: + cmd = f"tar zcf - {" ".join(artifacts_list)}" + (status, output) = target.run(cmd, raw = True) + if status != 0 or not output: + raise Exception("Error while fetching compressed artifacts") + p = subprocess.run(["tar", "zxf", "-", "-C", local_artifacts_dir], input=output) + except Exception as e: + bb.warn(f"Can not retrieve {artifact_path} from test target: {e}") def list_and_fetch_failed_tests_artifacts(d, tc): artifacts_list = get_artifacts_list(tc.target, d.getVar("TESTIMAGE_FAILED_QA_ARTIFACTS"))