[1/2] package_manager/ipk: skip checksums for unsigned local feeds

From: Etienne Cordonnier <ecordonnier@snap.com>

Computing checksums in the opkg Packages index requires reading every
.ipk file in full.  For a large image with 7000+ packages (including
multi-gigabyte debug packages), this adds 150-300s to every do_rootfs run.

Checksums in the Packages index are only meaningful for signed feeds
(PACKAGE_FEED_SIGN=1): the GPG signature covers the Packages index
which contains the SHA256Sum of each .ipk, forming a chain of trust
that prevents tampered packages being swapped on a remote feed.

For unsigned local file:// feeds the packages are installed directly
from the build host filesystem where there is no tampering risk.  Skip
all checksum generation in that case by passing no --checksum flags to
opkg-make-index (the tool's default behaviour when given no flags).
Pass --force-checksum to opkg so it does not error on the absent
checksum fields.

On a test image with 7000+ packages (including a 2.3 GB debug
package): write_index time reduced from ~180s to ~22s (8x speedup)
when opkg-make-index is configured to produce no checksums by default.

See https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796
("Enable sha256 checksums in opkg indexer", 2019) for the original rationale.

AI-Generated: Claude Sonnet 4.6
Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
---
 meta/lib/oe/package_manager/ipk/__init__.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

Etienne Cordonnier via lists.openembedded.org schrieb am Mi 29. Jul, 17:29 (+0200):
> From: Etienne Cordonnier <ecordonnier@snap.com>
> 
> Computing checksums in the opkg Packages index requires reading every
> .ipk file in full.  For a large image with 7000+ packages (including
> multi-gigabyte debug packages), this adds 150-300s to every do_rootfs run.
> 
> Checksums in the Packages index are only meaningful for signed feeds
> (PACKAGE_FEED_SIGN=1): the GPG signature covers the Packages index
> which contains the SHA256Sum of each .ipk, forming a chain of trust
> that prevents tampered packages being swapped on a remote feed.
> 
> For unsigned local file:// feeds the packages are installed directly
> from the build host filesystem where there is no tampering risk.  Skip
> all checksum generation in that case by passing no --checksum flags to
> opkg-make-index (the tool's default behaviour when given no flags).
> Pass --force-checksum to opkg so it does not error on the absent
> checksum fields.
> 
> On a test image with 7000+ packages (including a 2.3 GB debug
> package): write_index time reduced from ~180s to ~22s (8x speedup)
> when opkg-make-index is configured to produce no checksums by default.
> 
> See https://git.openembedded.org/openembedded-core/commit/?id=e462f47489f35902b6972f9837d9adfa542fc796
> ("Enable sha256 checksums in opkg indexer", 2019) for the original rationale.
> 
> AI-Generated: Claude Sonnet 4.6
> Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
> ---
>  meta/lib/oe/package_manager/ipk/__init__.py | 14 +++++++++-----
>  1 file changed, 9 insertions(+), 5 deletions(-)
> 
> diff --git a/meta/lib/oe/package_manager/ipk/__init__.py b/meta/lib/oe/package_manager/ipk/__init__.py
> index 344e0852177..6c0b896a086 100644
> --- a/meta/lib/oe/package_manager/ipk/__init__.py
> +++ b/meta/lib/oe/package_manager/ipk/__init__.py
> @@ -12,7 +12,7 @@ from oe.package_manager import *
>  from oe.package_manager.common_deb_ipk import OpkgDpkgPM
>  
>  class OpkgIndexer(Indexer):
> -    def write_index(self):
> +    def write_index(self, skip_checksums=False):

Suggestion: I would use the positive case (like with_checksums or
use_checksums or build_checksums), because reasoning about/reading code with
the inverse is more difficult.

>          arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
>                       "SDK_PACKAGE_ARCHS",
>                       ]
> @@ -44,8 +44,9 @@ class OpkgIndexer(Indexer):
>                  if not os.path.exists(pkgs_file):
>                      open(pkgs_file, "w").close()
>  
> -                index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s %s' %
> -                                  (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params))
> +                checksum_args = '' if skip_checksums else '--checksum md5 --checksum sha256 '
> +                index_cmds.add('%s %s-r %s -p %s -m %s %s' %
> +                                  (opkg_index_cmd, checksum_args, pkgs_file, pkgs_file, pkgs_dir, opkg_index_cmd_extra_params))
>  
>                  index_sign_files.add(pkgs_file)
>  
> @@ -103,8 +104,12 @@ class OpkgPM(OpkgDpkgPM):
>          self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir)
>          self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
>          self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
> +        self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"

Or: bb.utils.to_boolean(...)

> +        self._skip_checksums = self.d.getVar('PACKAGE_FEED_SIGN') != '1' and not self.from_feeds

I would swap them because evaluation of the second one is cheaper.

>          self.opkg_args = ['--volatile-cache', '-f', config_file, '-t', self.d.expand('${T}/ipktemp/'), '-o', target_rootfs]
>          self.opkg_args.extend(shlex.split(self.d.getVar("OPKG_ARGS")))
> +        if self._skip_checksums:
> +            self.opkg_args.append('--force-checksum')
>  
>          if prepare_index:
>              create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)
> @@ -116,7 +121,6 @@ class OpkgPM(OpkgDpkgPM):
>          if not os.path.exists(self.d.expand('${T}/saved')):
>              bb.utils.mkdirhier(self.d.expand('${T}/saved'))
>  
> -        self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"
>          if self.from_feeds:
>              self._create_custom_config()
>          else:
> @@ -346,7 +350,7 @@ class OpkgPM(OpkgDpkgPM):
>      def write_index(self):
>          self.deploy_dir_lock()
>  
> -        result = self.indexer.write_index()
> +        result = self.indexer.write_index(skip_checksums=self._skip_checksums)
>  
>          self.deploy_dir_unlock()


Jörg