[v3,0/4] Use the kernel from sstate when building fitImages

Changes in comparison to v2:
The commit "kernel: refactor fitimage" dropped the task dependency from
kernel:do_deploy -> kernel:do_install. This is now fixed by adding "after
do_install" to the addtask deploy.

Note, there is also a dependendency like
do_deploy --> do_populate_sysroot --> do_install.
However since the do_populate_sysroot task is sstate cached, the build
folder is not available for the do_deploy task if the do_populate_sysroot
setscene tasks runs. This is what most probably caused the errors on the
AB with v2.

Additionally the test script got more non firImage related
configurations.

Summary:
If the KERNEL_IMAGETYPES(S) contains fitImage, the kernel is always
rebuilt when something changes in the initramfs, which happens
frequently.
This is even worse if the build runs from an empty TMPDIR. The kernel
re-build starts by fetching the large kernel git repository and gets
recompiled from scratch.

This cannot be improved if INITRAMFS_IMAGE_BUNDLE = "1". If the kernel
Makefile is needed to generate the initramfs bundle the kernel build
folder is required.
But for a build configuration with INITRAMFS_IMAGE_BUNDLE = "" the
build folder is not needed. Creating the initramfs bundle requires:
linux.bin, DTBs and the initramfs which are available in the deploy
directory. That means creating the fitImage can be done with artifacts
which are already cached by the sstate.

Testing:
- oe-selftest -a --skip-tests distrodata.Distrodata.test_checkpkg \
     reproducible -T yocto-mirrors -T machine -T toolchain-user \
     -T toolchain-system
- Build linux-yocto with empty TMPDIR and empty sstate, followed by a build
  with populated sstate.
  - Clean build config:
    KERNEL_IMAGETYPE = "Image"
    KERNEL_IMAGETYPES += " fitImage "
    KERNEL_CLASSES = " kernel-fitimage "
    IMAGE_FSTYPES += "cpio.gz"
    INITRAMFS_IMAGE = "core-image-minimal"
    IMAGE_NAME_SUFFIX:pn-core-image-minimal = ""
    UBOOT_RD_LOADADDRESS = "0x88000000"
    UBOOT_RD_ENTRYPOINT = "0x88000000"
    UBOOT_LOADADDRESS = "0x80080000"
    UBOOT_ENTRYPOINT = "0x80080000"
    FIT_DESC = "A model description"
    FOO_VAR = "1"
    INHERIT += "image-buildinfo"
    IMAGE_BUILDINFO_VARS:append = " FOO_VAR"
  - bitbake linux-yocto
  - Append the following and rebuild with sstate:
    FOO_VAR = "2"
    TMPDIR = "${TOPDIR}/tmp-2"
  - bitbake linux-yocto
  - Check the log files:
    - tmp/log/cooker/qemux86-64 contains:
      linux-yocto-6.6.43+git-r0: task do_deploy: Succeeded
      linux-yocto-6.6.43+git-r0: task do_deploy_fitimage_unbundled: Succeeded
    - tmp-2/log/cooker/qemux86-64 contains:
      linux-yocto-6.6.43+git-r0: task do_deploy_setscene: Succeeded
      linux-yocto-6.6.43+git-r0: task do_deploy_fitimage_unbundled: Succeeded
- To re-view the changes in the task dependencies, the bash script bellow
  has been used. It confirms:
  - For builds with fitImage and unbundled initramfs:
    - the do_deploy_fitimage_unbundled task runs after do_deploy
    - do_assemble_fitimage_initramfs is not executed
    - do_bundle_initramfs is not executed
    - It works for fitImage in KERNEL_IMAGETYPE as well as for fitImage
      in KERNEL_IMAGETYPES
  - For builds with fitImage and bundled initramfs: No changes
  - For builds with fitImage, without initramfs:
    - do_assemble_fitimage_initramfs is not executed
    - do_bundle_initramfs is not executed

OUTPUT_FILE=task-depends.md
GIT_BRANCH=master
GIT_BRANCH_NEXT=adrianf/kernel-fitimage-sstate

echo "# Task dependeny changes" > "$OUTPUT_FILE"
echo "" >> "$OUTPUT_FILE"

run_bitbake(){
    echo "$1" >> "$OUTPUT_FILE"
    echo "" >> "$OUTPUT_FILE"

    bitbake core-image-minimal -g
    grep -E '(digraph depends|linux-yocto|\})' task-depends.dot \
    | grep -v -E '(spdx|do_kernel_configcheck|do_prepare_recipe_sysroot|do_populate_lic|native.do_populate_sysroot)' \
    | grep -v -E '(depmodwrapper-cross.do_populate_sysroot|binutils-cross-x86_64.do_populate_sysroot|do_package)' \
    | grep -v -E '(do_validate_branches|do_unpack|do_recipe_qa|do_patch|do_kernel_checkout|do_kernel_configme|do_kernel_metadata)' \
    | grep -v -E '(do_sizecheck|do_strip|do_compile_kernelmodules|do_shared_workdir|do_transform_kernel|do_kernel_link_images)' \
    | grep -v -E '(do_kernel_version_sanity_check|do_symlink_kernsrc|do_deploy_source_date_epoch|do_configure|do_fetch)' \
    | sed -e 's;\\n\:.*.bb;;g' -e 's;linux-yocto[. ];;g' > task-depends-filtered.dot

    echo '```plantuml' >> "$OUTPUT_FILE"
    cat task-depends-filtered.dot >> "$OUTPUT_FILE"
    echo '```' >> "$OUTPUT_FILE"
    echo "" >> "$OUTPUT_FILE"
}

run_bitbake_per_branch(){
    echo "## Configuration: $1" >> "$OUTPUT_FILE"
    echo "" >> "$OUTPUT_FILE"
    echo '```raw' >> "$OUTPUT_FILE"
    cat build/conf/auto.conf >> "$OUTPUT_FILE"
    echo '```' >> "$OUTPUT_FILE"
    echo "" >> "$OUTPUT_FILE"

    git checkout $GIT_BRANCH
    run_bitbake "### branch: $GIT_BRANCH"
    mv -f task-depends-filtered.dot task-depends-filtered-old.dot

    git checkout $GIT_BRANCH_NEXT
    run_bitbake "### branch: $GIT_BRANCH_NEXT"

    echo "## Diff" >> "$OUTPUT_FILE"
    echo "" >> "$OUTPUT_FILE"
    echo '```patch' >> "$OUTPUT_FILE"
    diff task-depends-filtered-old.dot task-depends-filtered.dot >> "$OUTPUT_FILE"
    echo '```' >> "$OUTPUT_FILE"
    echo "" >> "$OUTPUT_FILE"
}

function cleanup {
  rm -f build/conf/auto.conf task-depends-filtered-old.dot task-depends-filtered.dot task-depends.dot
}
trap cleanup EXIT

cat << EOF > build/conf/auto.conf
KERNEL_IMAGETYPE = "Image"
KERNEL_IMAGETYPES += " fitImage "
KERNEL_CLASSES = " kernel-fitimage "
IMAGE_FSTYPES += "cpio.gz"
INITRAMFS_IMAGE = "core-image-minimal-initramfs"
EOF
run_bitbake_per_branch "image, fitimage, initramfs, unbundled"

cat << EOF > build/conf/auto.conf
KERNEL_IMAGETYPE:forcevariable = "fitImage"
KERNEL_CLASSES = " kernel-fitimage "
IMAGE_FSTYPES += "cpio.gz"
INITRAMFS_IMAGE = "core-image-minimal-initramfs"
EOF
run_bitbake_per_branch "fitimage, initramfs, unbundled"

cat << EOF > build/conf/auto.conf
KERNEL_IMAGETYPE = "Image"
KERNEL_IMAGETYPES += " fitImage "
KERNEL_CLASSES = " kernel-fitimage "
IMAGE_FSTYPES += "cpio.gz"
INITRAMFS_IMAGE = "core-image-minimal-initramfs"
INITRAMFS_IMAGE_BUNDLE = "1"
EOF
run_bitbake_per_branch "image, fitimage, initramfs, bundled"

cat << EOF > build/conf/auto.conf
KERNEL_IMAGETYPE = "Image"
KERNEL_IMAGETYPES += " fitImage "
KERNEL_CLASSES = " kernel-fitimage "
EOF
run_bitbake_per_branch "image, fitimage"

cat << EOF > build/conf/auto.conf
KERNEL_IMAGETYPE = "Image"
EOF
run_bitbake_per_branch "image, no fitimage"

cat << EOF > build/conf/auto.conf
KERNEL_IMAGETYPE = "Image"
IMAGE_FSTYPES += "cpio.gz"
INITRAMFS_IMAGE = "core-image-minimal-initramfs"
INITRAMFS_IMAGE_BUNDLE = "1"
EOF
run_bitbake_per_branch "image, no fitimage, initramfs, bundled"

cat << EOF > build/conf/auto.conf
KERNEL_IMAGETYPE = "Image"
IMAGE_FSTYPES += "cpio.gz"
INITRAMFS_IMAGE = "core-image-minimal-initramfs"
INITRAMFS_IMAGE_BUNDLE = ""
EOF
run_bitbake_per_branch "image, no fitimage, initramfs, not bundled"

Adrian Freihofer (4):
  kernel: refactor linux compression
  kernel-fitimage: refactor fitimage_assemble
  kernel: refactor fitimage
  kernel-fitimage: run unbundled fitimage after deploy

 meta/classes-recipe/image.bbclass           |  12 +-
 meta/classes-recipe/kernel-fitimage.bbclass | 152 +++++++++++++-------
 meta/classes-recipe/kernel-uboot.bbclass    |   1 +
 meta/classes-recipe/kernel.bbclass          |  32 +++--
 4 files changed, 130 insertions(+), 67 deletions(-)