diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py index 32a7bc0098..9d227e2032 100644 --- a/scripts/lib/devtool/__init__.py +++ b/scripts/lib/devtool/__init__.py @@ -77,6 +77,18 @@ def exec_watch(cmd, **options): return buf, None +def build_fakeroot_env_no_d(fakerootenv, path, env_overrides=None): + """Build the environment for running a command under pseudo (see exec_fakeroot_no_d).""" + newenv = dict(os.environ) + newenv['PATH'] = path + for varvalue in fakerootenv.split(): + if '=' in varvalue: + splitval = varvalue.split('=', 1) + newenv[splitval[0]] = splitval[1] + if env_overrides: + newenv.update(env_overrides) + return newenv + def exec_fakeroot_no_d(fakerootcmd, fakerootenv, path, cmd, env_overrides=None, **kwargs): """Run cmd under pseudo using a recipe's own FAKEROOTCMD/FAKEROOTENV. @@ -88,15 +100,7 @@ def exec_fakeroot_no_d(fakerootcmd, fakerootenv, path, cmd, env_overrides=None, if not os.path.exists(fakerootcmd): logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built') return 2 - # Set up the appropriate environment - newenv = dict(os.environ) - newenv['PATH'] = path - for varvalue in fakerootenv.split(): - if '=' in varvalue: - splitval = varvalue.split('=', 1) - newenv[splitval[0]] = splitval[1] - if env_overrides: - newenv.update(env_overrides) + newenv = build_fakeroot_env_no_d(fakerootenv, path, env_overrides) return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs) def setup_tinfoil(config_only=False, basepath=None, tracking=False): diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py index ffca1269ef..e3890a0113 100644 --- a/scripts/lib/devtool/deploy.py +++ b/scripts/lib/devtool/deploy.py @@ -19,7 +19,7 @@ import argparse_oe import oe.types import oe.package -from devtool import exec_fakeroot_no_d, setup_tinfoil, check_workspace_recipe, DevtoolError +from devtool import exec_fakeroot_no_d, build_fakeroot_env_no_d, setup_tinfoil, check_workspace_recipe, DevtoolError logger = logging.getLogger('devtool') @@ -257,14 +257,18 @@ def deploy(args, config, basepath, workspace): def deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args, file_globs=None, packages_files=None): import math - try: - host, destdir = args.target.split(':') - except ValueError: - destdir = '/' + if os.path.isabs(args.target): + # A local pseudo-managed rootfs directory (e.g. NFS-exported) + destdir = os.path.realpath(args.target) else: - args.target = host - if not destdir.endswith('/'): - destdir += '/' + try: + host, destdir = args.target.split(':') + except ValueError: + destdir = '/' + else: + args.target = host + # Canonical form used throughout: no trailing slash (except root itself). + destdir = destdir.rstrip('/') or '/' recipe_outdir = srcdir if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): @@ -353,9 +357,107 @@ def deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_proce print(' %s' % item) return 0 + if os.path.isabs(args.target): + # A local directory (e.g. an NFS-exported rootfs) rather than a + # user@host ssh target: copy the files in directly, no network needed. + return _deploy_local(args, destdir, filelist, ftotalsize, tar_relpaths, + allowed_files, fakerootcmd, fakerootenv, path, recipe_outdir) + return _deploy_ssh(args, destdir, filelist, ftotalsize, tar_relpaths, allowed_files, fakerootcmd, fakerootenv, path, recipe_outdir) +def _deploy_local(args, destdir, filelist, ftotalsize, tar_relpaths, allowed_files, + fakerootcmd, fakerootenv, path, recipe_outdir): + """Copy files directly into destdir instead of over ssh/scp. + + destdir is the local pseudo-managed rootfs directory itself (no trailing + slash), not the real filesystem root. + """ + if not os.path.isdir(destdir): + raise DevtoolError('Target directory %s does not exist' % destdir) + state_dir = destdir + '.pseudo_state' + if not os.path.isdir(state_dir): + raise DevtoolError( + '%s does not exist - %s does not look like a pseudo-managed rootfs ' + '(e.g. one extracted by runqemu-extract-sdk).' % (state_dir, destdir)) + + if not args.no_check_space: + freespace = shutil.disk_usage(destdir).free // 1024 + if ftotalsize > freespace: + raise DevtoolError('Deploy failed - insufficient space on target ' + '(available %d, needed %d)' % (freespace, ftotalsize)) + + shellscript = _prepare_remote_script(deploy=True, + destdir=destdir, + verbose=args.show_status, + nopreserve=args.no_preserve, + nocheckspace=True) + + tmpdir = tempfile.mkdtemp(prefix='devtool') + tar_send_filelist_path = None + try: + script_path = os.path.join(tmpdir, 'devtool_deploy.sh') + with open(script_path, 'w') as f: + f.write(shellscript) + filelist_path = os.path.join(tmpdir, 'devtool_deploy.list') + with open(filelist_path, 'w') as f: + f.write('%d\n' % ftotalsize) + for fpath, fsize in filelist: + f.write('%s %d\n' % (fpath, fsize)) + + # tar_send_* builds up the sending side of the pipe: the plain tar + # invocation, then wrapped to capture its own exit status (only the + # last stage of a shell pipeline is visible to subprocess), then + # wrapped again to run under its own pseudo instance. + if allowed_files is not None: + tar_send_fd, tar_send_filelist_path = tempfile.mkstemp(prefix='devtool-deploy-filelist-') + with os.fdopen(tar_send_fd, 'w') as f: + for relpath in tar_relpaths: + f.write('./' + relpath + '\n') + tar_send_argv = 'tar cf - -T %s' % shlex.quote(tar_send_filelist_path) + else: + tar_send_argv = 'tar cf - .' + + tar_send_status_path = os.path.join(tmpdir, 'devtool_deploy.tar_status') + tar_send_script = 'sh -c %s' % shlex.quote( + '%s; echo $? > %s' % (tar_send_argv, shlex.quote(tar_send_status_path))) + tar_send_cmd = 'PSEUDO_INCLUDE_PATHS=%s %s %s' % ( + shlex.quote(recipe_outdir), shlex.quote(fakerootcmd), tar_send_script) + + # tar_receive_cmd is the other side of the pipe: extracts into destdir + # under the target rootfs's own pseudo database (state_dir/destdir, + # not the recipe's). + # $2 needs a trailing slash: the script's manifest substitution + # (sed 's!^./!$2!') turns tar's './relative' entries into absolute paths. + tar_receive_cmd = 'PSEUDO_LOCALSTATEDIR=%s PSEUDO_INCLUDE_PATHS=%s %s sh %s %s %s %s' % ( + shlex.quote(state_dir), shlex.quote(destdir), shlex.quote(fakerootcmd), + shlex.quote(script_path), shlex.quote(args.recipename), + shlex.quote(destdir.rstrip('/') + '/'), shlex.quote(filelist_path)) + + if not os.path.exists(fakerootcmd): + logger.error('pseudo executable %s could not be found - have you run a build ' + 'yet? pseudo-native should install this and if you have run any ' + 'build then that should have been built' % fakerootcmd) + ret = 2 + else: + shell_env = build_fakeroot_env_no_d(fakerootenv, path) + ret = subprocess.call('%s | %s' % (tar_send_cmd, tar_receive_cmd), env=shell_env, + cwd=recipe_outdir, shell=True) + if ret == 0 and os.path.exists(tar_send_status_path): + with open(tar_send_status_path) as f: + ret = int(f.read().strip() or 0) + finally: + if tar_send_filelist_path: + os.remove(tar_send_filelist_path) + shutil.rmtree(tmpdir) + + if ret != 0: + raise DevtoolError('Deploy failed - rerun with -s to get a complete ' + 'error message') + + logger.info('Successfully deployed %s to %s' % (recipe_outdir, destdir)) + return 0 + def _deploy_ssh(args, destdir, filelist, ftotalsize, tar_relpaths, allowed_files, fakerootcmd, fakerootenv, path, recipe_outdir): """Copy files to target_dir over ssh/scp (user@hostname[:destdir]).""" @@ -423,9 +525,11 @@ def _deploy_ssh(args, destdir, filelist, ftotalsize, tar_relpaths, allowed_files tar_cmd = 'tar cf - -T %s' % shlex.quote(tar_filelist_path) else: tar_cmd = 'tar cf - .' + # $2 needs a trailing slash: the script's manifest substitution + # (sed 's!^./!$2!') turns tar's './relative' entries into absolute paths. remote_cmd = '%s | %s %s %s %s \'sh %s %s %s %s\'' % ( tar_cmd, ssh_sshexec, ssh_port, extraoptions, args.target, - tmpscript, args.recipename, destdir, tmpfilelist) + tmpscript, args.recipename, destdir.rstrip('/') + '/', tmpfilelist) ret = exec_fakeroot_no_d(fakerootcmd, fakerootenv, path, remote_cmd, cwd=recipe_outdir, env_overrides={'PSEUDO_INCLUDE_PATHS': recipe_outdir}, shell=True) finally: @@ -446,8 +550,64 @@ def undeploy(args, config, basepath, workspace): elif not args.recipename and not args.all: raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target') + if os.path.isabs(args.target): + # A local directory (e.g. an NFS-exported rootfs) rather than a + # user@host ssh target: remove the files in directly, no network needed. + tinfoil = setup_tinfoil(config_only=True, basepath=basepath) + try: + fakerootcmd = tinfoil.config_data.getVar('FAKEROOTCMD') + fakerootenv = tinfoil.config_data.getVar('FAKEROOTENV') + path = tinfoil.config_data.getVar('PATH') + finally: + tinfoil.shutdown() + return _undeploy_local(args, os.path.realpath(args.target), fakerootcmd, fakerootenv, path) + return _undeploy_ssh(args) +def _undeploy_local(args, target_dir, fakerootcmd, fakerootenv, path): + """Remove files directly from target_dir instead of over ssh.""" + if not os.path.isdir(target_dir): + raise DevtoolError('Target directory %s does not exist' % target_dir) + state_dir = target_dir + '.pseudo_state' + if not os.path.isdir(state_dir): + raise DevtoolError( + '%s does not exist - %s does not look like a pseudo-managed rootfs ' + '(e.g. one extracted by runqemu-extract-sdk).' % (state_dir, target_dir)) + + # deploy=False here: the generated script never touches $2, so target_dir + # doesn't need the trailing slash _with_trailing_slash() adds for deploy. + shellscript = _prepare_remote_script(deploy=False, destdir=target_dir, dryrun=args.dry_run, undeployall=args.all) + + tmpdir = tempfile.mkdtemp(prefix='devtool') + try: + script_path = os.path.join(tmpdir, 'devtool_undeploy.sh') + with open(script_path, 'w') as f: + f.write(shellscript) + + environment = dict(os.environ) + environment['PATH'] = path + for varvalue in (fakerootenv or '').split(): + if '=' in varvalue: + key, value = varvalue.split('=', 1) + environment[key] = value + # Use target_dir's own pseudo database, not the ambient one from FAKEROOTENV, + # so file removals stay consistent with what was recorded on deploy/extract. + environment['PSEUDO_LOCALSTATEDIR'] = state_dir + environment['PSEUDO_INCLUDE_PATHS'] = target_dir + command = [fakerootcmd, 'sh', script_path, args.recipename or ''] + ret = subprocess.call(command, env=environment) + finally: + shutil.rmtree(tmpdir) + + if ret != 0: + # Unlike the ssh case there is nothing -s could add here, the script + # runs locally and its output is already on the console. + raise DevtoolError('Undeploy failed - see the output above for details') + + if not args.all and not args.dry_run: + logger.info('Successfully undeployed %s' % args.recipename) + return 0 + def _undeploy_ssh(args): """Run the undeploy script on the target over ssh/scp (user@hostname[:destdir]).""" extraoptions = '' @@ -473,8 +633,6 @@ def _undeploy_ssh(args): destdir = '/' else: args.target = host - if not destdir.endswith('/'): - destdir += '/' tmpdir = tempfile.mkdtemp(prefix='devtool') try: @@ -508,17 +666,20 @@ def register_commands(subparsers, context): parser_deploy = subparsers.add_parser('deploy-target', help='Deploy recipe output files to live target machine', description='Deploys a recipe\'s build output (i.e. the output of ' - 'the do_install task) to a live target machine over ssh. ' - 'By default, any existing files will be preserved instead ' - 'of being overwritten and will be restored if you run ' - 'devtool undeploy-target. Note: this only deploys the ' - 'recipe itself and not any runtime dependencies, so it is ' - 'assumed that those have been installed on the target ' - 'beforehand. Use --package/--file-glob to deploy only a ' - 'subset of the recipe\'s installed files.', + 'the do_install task) to a live target machine over ssh, ' + 'or directly into a local pseudo-managed rootfs directory ' + '(e.g. one extracted for NFS booting). Existing files are ' + 'preserved by default and restored by devtool ' + 'undeploy-target. Only the recipe itself is deployed, not ' + 'its runtime dependencies. Use --package/--file-glob to ' + 'deploy only a subset of the recipe\'s installed files.', group='testbuild') parser_deploy.add_argument('recipename', help='Recipe to deploy') - parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]') + parser_deploy.add_argument('target', + help='Either a live target machine running an ssh server: ' + 'user@hostname[:destdir]; or an absolute path to a local ' + 'pseudo-managed rootfs directory (e.g. one extracted by ' + 'runqemu-extract-sdk) to copy the files into directly, without ssh.') parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true') @@ -552,10 +713,15 @@ def register_commands(subparsers, context): parser_undeploy = subparsers.add_parser('undeploy-target', help='Undeploy recipe output files in live target machine', - description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.', + description='Un-deploys recipe output files previously deployed to a live target machine or local ' + 'pseudo-managed rootfs directory by devtool deploy-target.', group='testbuild') parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?') - parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname') + parser_undeploy.add_argument('target', + help='Either a live target machine running an ssh server: ' + 'user@hostname; or an absolute path to the local pseudo-managed ' + 'rootfs directory previously used with deploy-target, to remove ' + 'the files directly, without ssh.') parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true')