Message ID | 20240513015242.1233919-1-seungkyun.kim@lge.com |
---|---|
State | Rejected |
Delegated to: | Steve Sakoman |
Headers | show |
Series | [kirkstone] ipk/rootfs: run sanity test of multilib in parallel | expand |
This patch would need to be submitted (and merged) for master before it can be considered for kirkstone. And since it is a significant change in behaviour, even then it might not be suitable for an LTS branch. Thanks, Steve On Sun, May 12, 2024 at 6:52 PM Seungkyun Kim via lists.openembedded.org <seungkyun.kim=lge.com@lists.openembedded.org> wrote: > > From: "seungkyun.kim" <seungkyun.kim@lge.com> > > For multilib type packages, there is an additional temporary > installation before the actual installation. It makes almost doubles > the do_rootfs time if having many multilib type packages. > To avoid this overhead, run sanity test in parallel. > Installing package groups through opkg takes much more time than > copying directory. > > Signed-off-by: seungkyun.kim <seungkyun.kim@lge.com> > --- > meta/lib/oe/package_manager/ipk/rootfs.py | 21 +++++++++++++++++---- > 1 file changed, 17 insertions(+), 4 deletions(-) > > diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py > index 10a831994e..88108bb312 100644 > --- a/meta/lib/oe/package_manager/ipk/rootfs.py > +++ b/meta/lib/oe/package_manager/ipk/rootfs.py > @@ -4,6 +4,7 @@ > > import re > import filecmp > +import multiprocessing > import shutil > from oe.rootfs import Rootfs > from oe.manifest import Manifest > @@ -196,10 +197,16 @@ class PkgRootfs(DpkgOpkgRootfs): > > def _multilib_test_install(self, pkgs): > ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS") > + rootfs_temp = os.path.join(ml_temp, "rootfs") > bb.utils.mkdirhier(ml_temp) > > - dirs = [self.image_rootfs] > + bb.utils.remove(rootfs_temp, True) > + shutil.copytree(self.image_rootfs, rootfs_temp) > + dirs = [rootfs_temp] > + return multiprocessing.Process(target=self._multilib_test_pkg_install, \ > + args=(pkgs, ml_temp, dirs, False)) > > + def _multilib_test_pkg_install(self, pkgs, ml_temp, dirs): > for variant in self.d.getVar("MULTILIB_VARIANTS").split(): > ml_target_rootfs = os.path.join(ml_temp, variant) > > @@ -216,6 +223,8 @@ class PkgRootfs(DpkgOpkgRootfs): > dirs.append(ml_target_rootfs) > > self._multilib_sanity_test(dirs) > + rootfs_temp = os.path.join(ml_temp, "rootfs") > + bb.utils.remove(rootfs_temp) > > ''' > While ipk incremental image generation is enabled, it will remove the > @@ -298,14 +307,18 @@ class PkgRootfs(DpkgOpkgRootfs): > > for pkg_type in self.install_order: > if pkg_type in pkgs_to_install: > + sanity_test = None > # For multilib, we perform a sanity test before final install > # If sanity test fails, it will automatically do a bb.fatal() > # and the installation will stop > if pkg_type == Manifest.PKG_TYPE_MULTILIB: > - self._multilib_test_install(pkgs_to_install[pkg_type]) > + sanity_test= self._multilib_test_install(pkgs_to_install[pkg_type]) > + sanity_test.start() > > - self.pm.install(pkgs_to_install[pkg_type], > - [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) > + self.pm.install(pkgs_to_install[pkg_type], [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) > + > + if sanity_test is not None: > + sanity_test.join() > > if self.progress_reporter: > self.progress_reporter.next_stage() > -- > 2.43.0 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#199222): https://lists.openembedded.org/g/openembedded-core/message/199222 > Mute This Topic: https://lists.openembedded.org/mt/106065500/3620601 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [steve@sakoman.com] > -=-=-=-=-=-=-=-=-=-=-=- >
diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py index 10a831994e..88108bb312 100644 --- a/meta/lib/oe/package_manager/ipk/rootfs.py +++ b/meta/lib/oe/package_manager/ipk/rootfs.py @@ -4,6 +4,7 @@ import re import filecmp +import multiprocessing import shutil from oe.rootfs import Rootfs from oe.manifest import Manifest @@ -196,10 +197,16 @@ class PkgRootfs(DpkgOpkgRootfs): def _multilib_test_install(self, pkgs): ml_temp = self.d.getVar("MULTILIB_TEMP_ROOTFS") + rootfs_temp = os.path.join(ml_temp, "rootfs") bb.utils.mkdirhier(ml_temp) - dirs = [self.image_rootfs] + bb.utils.remove(rootfs_temp, True) + shutil.copytree(self.image_rootfs, rootfs_temp) + dirs = [rootfs_temp] + return multiprocessing.Process(target=self._multilib_test_pkg_install, \ + args=(pkgs, ml_temp, dirs, False)) + def _multilib_test_pkg_install(self, pkgs, ml_temp, dirs): for variant in self.d.getVar("MULTILIB_VARIANTS").split(): ml_target_rootfs = os.path.join(ml_temp, variant) @@ -216,6 +223,8 @@ class PkgRootfs(DpkgOpkgRootfs): dirs.append(ml_target_rootfs) self._multilib_sanity_test(dirs) + rootfs_temp = os.path.join(ml_temp, "rootfs") + bb.utils.remove(rootfs_temp) ''' While ipk incremental image generation is enabled, it will remove the @@ -298,14 +307,18 @@ class PkgRootfs(DpkgOpkgRootfs): for pkg_type in self.install_order: if pkg_type in pkgs_to_install: + sanity_test = None # For multilib, we perform a sanity test before final install # If sanity test fails, it will automatically do a bb.fatal() # and the installation will stop if pkg_type == Manifest.PKG_TYPE_MULTILIB: - self._multilib_test_install(pkgs_to_install[pkg_type]) + sanity_test= self._multilib_test_install(pkgs_to_install[pkg_type]) + sanity_test.start() - self.pm.install(pkgs_to_install[pkg_type], - [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + self.pm.install(pkgs_to_install[pkg_type], [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY]) + + if sanity_test is not None: + sanity_test.join() if self.progress_reporter: self.progress_reporter.next_stage()