@@ -32,7 +32,7 @@ def create_index(arg):
def opkg_query(cmd_output):
"""
- This method parse the output from the package managerand return
+ This method parse the output from the package manager and return
a dictionary with the information of the packages. This is used
when the packages are in deb or ipk format.
"""
@@ -369,40 +369,48 @@ class PackageManager(object, metaclass=ABCMeta):
if globs:
# we need to write the list of installed packages to a file because the
# oe-pkgdata-util reads it from a file
- with tempfile.NamedTemporaryFile(mode="w+", prefix="installed-pkgs") as installed_pkgs:
- pkgs = self.list_installed()
-
- provided_pkgs = set()
- for pkg in pkgs.values():
- provided_pkgs |= set(pkg.get('provs', []))
-
- output = oe.utils.format_pkg_list(pkgs, "arch")
- installed_pkgs.write(output)
- installed_pkgs.flush()
-
- cmd = ["oe-pkgdata-util",
- "-p", self.d.getVar('PKGDATA_DIR'), "glob", installed_pkgs.name,
- globs]
- exclude = self.d.getVar('PACKAGE_EXCLUDE_COMPLEMENTARY')
- if exclude:
- cmd.extend(['--exclude=' + '|'.join(exclude.split())])
- try:
- bb.note('Running %s' % cmd)
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- stdout, stderr = proc.communicate()
- if stderr: bb.note(stderr.decode("utf-8"))
- complementary_pkgs = stdout.decode("utf-8")
- complementary_pkgs = set(complementary_pkgs.split())
- skip_pkgs = sorted(complementary_pkgs & provided_pkgs)
- install_pkgs = sorted(complementary_pkgs - provided_pkgs)
- bb.note("Installing complementary packages ... %s (skipped already provided packages %s)" % (
- ' '.join(install_pkgs),
- ' '.join(skip_pkgs)))
- self.install(install_pkgs, hard_depends_only=True)
- except subprocess.CalledProcessError as e:
- bb.fatal("Could not compute complementary packages list. Command "
- "'%s' returned %d:\n%s" %
- (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
+ with tempfile.NamedTemporaryFile(mode="w+", prefix="all-pkgs") as all_pkgs:
+ with tempfile.NamedTemporaryFile(mode="w+", prefix="installed-pkgs") as installed_pkgs:
+ pkgs = self.list_installed()
+
+ provided_pkgs = set()
+ for pkg in pkgs.values():
+ provided_pkgs |= set(pkg.get('provs', []))
+
+ output = oe.utils.format_pkg_list(pkgs, "arch")
+ installed_pkgs.write(output)
+ installed_pkgs.flush()
+
+ cmd = ["oe-pkgdata-util",
+ "-p", self.d.getVar('PKGDATA_DIR'), "glob",
+ installed_pkgs.name, globs]
+
+ if hasattr(self, "list_all"):
+ output_allpkg = self.list_all()
+ all_pkgs.write(output_allpkg)
+ all_pkgs.flush()
+ cmd.extend(["--allpkgs=%s" % all_pkgs.name])
+
+ exclude = self.d.getVar('PACKAGE_EXCLUDE_COMPLEMENTARY')
+ if exclude:
+ cmd.extend(['--exclude=' + '|'.join(exclude.split())])
+ try:
+ bb.note('Running %s' % cmd)
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = proc.communicate()
+ if stderr: bb.note(stderr.decode("utf-8"))
+ complementary_pkgs = stdout.decode("utf-8")
+ complementary_pkgs = set(complementary_pkgs.split())
+ skip_pkgs = sorted(complementary_pkgs & provided_pkgs)
+ install_pkgs = sorted(complementary_pkgs - provided_pkgs)
+ bb.note("Installing complementary packages ... %s (skipped already provided packages %s)" % (
+ ' '.join(install_pkgs),
+ ' '.join(skip_pkgs)))
+ self.install(install_pkgs, hard_depends_only=True)
+ except subprocess.CalledProcessError as e:
+ bb.fatal("Could not compute complementary packages list. Command "
+ "'%s' returned %d:\n%s" %
+ (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
if self.d.getVar('IMAGE_LOCALES_ARCHIVE') == '1':
target_arch = self.d.getVar('TARGET_ARCH')
@@ -112,6 +112,29 @@ class PMPkgsList(PkgsList):
return opkg_query(cmd_output)
+ def list_all_pkgs(self, apt_conf_file=None):
+ if not apt_conf_file:
+ apt_conf_file = self.d.expand("${APTCONF_TARGET}/apt/apt.conf")
+ os.environ['APT_CONFIG'] = apt_conf_file
+
+ cmd = [bb.utils.which(os.getenv('PATH'), "apt"), "list"]
+
+ try:
+ cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8")
+ except subprocess.CalledProcessError as e:
+ bb.fatal("Cannot get the all packages list. Command '%s' "
+ "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
+
+ all_pkgs_lines = []
+ for line in cmd_output.splitlines():
+ line_parts = line.split()
+ # the valid lines takes the format of something like "findutils-locale-ga/unknown 4.10.0-r0 amd64"
+ if len(line_parts) != 3:
+ continue
+ line_parts[0] = line_parts[0].split('/')[0]
+ new_line = ' '.join(line_parts)
+ all_pkgs_lines.append(new_line)
+ return "\n".join(all_pkgs_lines)
class DpkgPM(OpkgDpkgPM):
def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None, deb_repo_workdir="oe-rootfs-repo", filterbydependencies=True):
@@ -436,6 +459,9 @@ class DpkgPM(OpkgDpkgPM):
def list_installed(self):
return PMPkgsList(self.d, self.target_rootfs).list_pkgs()
+ def list_all(self):
+ return PMPkgsList(self.d, self.target_rootfs).list_all_pkgs(apt_conf_file=self.apt_conf_file)
+
def package_info(self, pkg):
"""
Returns a dictionary with the package info.
@@ -90,6 +90,23 @@ class PMPkgsList(PkgsList):
return opkg_query(cmd_output)
+ def list_all_pkgs(self, format=None):
+ cmd = "%s %s list" % (self.opkg_cmd, self.opkg_args)
+
+ # opkg returns success even when it printed some
+ # "Collected errors:" report to stderr. Mixing stderr into
+ # stdout then leads to random failures later on when
+ # parsing the output. To avoid this we need to collect both
+ # output streams separately and check for empty stderr.
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ cmd_output, cmd_stderr = p.communicate()
+ cmd_output = cmd_output.decode("utf-8")
+ cmd_stderr = cmd_stderr.decode("utf-8")
+ if p.returncode or cmd_stderr:
+ bb.fatal("Cannot get all packages list. Command '%s' "
+ "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr))
+
+ return cmd_output
class OpkgPM(OpkgDpkgPM):
def __init__(self, d, target_rootfs, config_file, archs, task_name='target', ipk_repo_workdir="oe-rootfs-repo", filterbydependencies=True, prepare_index=True):
@@ -364,6 +381,9 @@ class OpkgPM(OpkgDpkgPM):
def list_installed(self):
return PMPkgsList(self.d, self.target_rootfs).list_pkgs()
+ def list_all(self):
+ return PMPkgsList(self.d, self.target_rootfs).list_all_pkgs()
+
def dummy_install(self, pkgs):
"""
The following function dummy installs pkgs and returns the log of output.
@@ -275,6 +275,14 @@ class RpmPM(PackageManager):
elif os.path.isfile(source_dir):
shutil.copy2(source_dir, target_dir)
+ def list_all(self):
+ output = self._invoke_dnf(["repoquery", "--all", "--queryformat", "Packages: %{name} %{arch} %{version}"], print_output = False)
+ all_pkgs_lines = []
+ for line in output.splitlines():
+ if line.startswith("Packages: "):
+ all_pkgs_lines.append(line.replace("Packages: ", ""))
+ return "\n".join(all_pkgs_lines)
+
def list_installed(self):
output = self._invoke_dnf(["repoquery", "--installed", "--queryformat", "Package: %{name} %{arch} %{version} %{name}-%{version}-%{release}.%{arch}.rpm\nDependencies:\n%{requires}\nRecommendations:\n%{recommends}\nDependenciesEndHere:\n"],
print_output = False)
@@ -51,6 +51,15 @@ def glob(args):
skippedpkgs = set()
mappedpkgs = set()
+ allpkgs = set()
+ if args.allpkgs:
+ with open(args.allpkgs, 'r') as f:
+ for line in f:
+ fields = line.rstrip().split()
+ if not fields:
+ continue
+ else:
+ allpkgs.add(fields[0])
with open(args.pkglistfile, 'r') as f:
for line in f:
fields = line.rstrip().split()
@@ -136,6 +145,10 @@ def glob(args):
logger.debug("%s is not a valid package!" % (pkg))
break
+ if args.allpkgs:
+ if mappedpkg not in allpkgs:
+ continue
+
if mappedpkg:
logger.debug("%s (%s) -> %s" % (pkg, g, mappedpkg))
mappedpkgs.add(mappedpkg)
@@ -592,6 +605,7 @@ def main():
parser_glob.add_argument('pkglistfile', help='File listing packages (one package name per line)')
parser_glob.add_argument('glob', nargs="+", help='Glob expression for package names, e.g. *-dev')
parser_glob.add_argument('-x', '--exclude', help='Exclude packages matching specified regex from the glob operation')
+ parser_glob.add_argument('-a', '--allpkgs', help='File listing all available packages (one package name per line)')
parser_glob.set_defaults(func=glob)