[2/2] pip_install_wheel: use installer instead of pip

Message ID 20220310171656.3106028-2-ross.burton@arm.com
State New
Headers show
Series [1/2] python3-installer: add installer module | expand

Commit Message

Ross Burton March 10, 2022, 5:16 p.m. UTC
Instead of battling pip to install a wheel, use installer. Installer
does one thing, so it's faster and easier to work with.

This means setuptools, pip, and wheel are no longer part of the
bootstrap phase, so they can be built normally.  To avoid sysroot file
conflicts these three recipes can't install .pyc files to the native
sysroot.

Signed-off-by: Ross Burton <ross.burton@arm.com>
---
 meta/classes/pip_install_wheel.bbclass        | 16 +++------
 .../python/python3-flit-core_3.7.1.bb         |  2 +-
 .../python/python3-pip_22.0.3.bb              | 35 +++----------------
 .../python/python3-setuptools_59.5.0.bb       | 12 +++----
 .../python/python3-wheel_0.37.1.bb            | 26 +++-----------
 5 files changed, 18 insertions(+), 73 deletions(-)

Comments

Richard Purdie March 10, 2022, 9:19 p.m. UTC | #1
On Thu, 2022-03-10 at 17:16 +0000, Ross Burton wrote:
> Instead of battling pip to install a wheel, use installer. Installer
> does one thing, so it's faster and easier to work with.
> 
> This means setuptools, pip, and wheel are no longer part of the
> bootstrap phase, so they can be built normally.  To avoid sysroot file
> conflicts these three recipes can't install .pyc files to the native
> sysroot.
> 
> Signed-off-by: Ross Burton <ross.burton@arm.com>
> ---
>  meta/classes/pip_install_wheel.bbclass        | 16 +++------
>  .../python/python3-flit-core_3.7.1.bb         |  2 +-
>  .../python/python3-pip_22.0.3.bb              | 35 +++----------------
>  .../python/python3-setuptools_59.5.0.bb       | 12 +++----
>  .../python/python3-wheel_0.37.1.bb            | 26 +++-----------
>  5 files changed, 18 insertions(+), 73 deletions(-)

I suspect pip is doing some interpreter patch manipulation we're now missing out
on:

https://autobuilder.yoctoproject.org/typhoon/#/builders/62/builds/4865/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/48/builds/4871/steps/13/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/37/builds/4868/steps/12/logs/stdio

and many more :/

including meta-arm:

https://autobuilder.yoctoproject.org/typhoon/#/builders/37/builds/4868/steps/12/logs/stdio

Cheers,

Richard

Patch

diff --git a/meta/classes/pip_install_wheel.bbclass b/meta/classes/pip_install_wheel.bbclass
index 29cd544aa3..497f69260e 100644
--- a/meta/classes/pip_install_wheel.bbclass
+++ b/meta/classes/pip_install_wheel.bbclass
@@ -1,23 +1,15 @@ 
-DEPENDS:append = " python3-pip-native"
+DEPENDS:append = " python3-installer-native"
 
 # The directory where wheels should be written too. Build classes
 # will ideally [cleandirs] this but we don't do that here in case
 # a recipe wants to install prebuilt wheels.
 PIP_INSTALL_DIST_PATH ?= "${WORKDIR}/dist"
 
-PIP_INSTALL_ARGS = "\
-    -vvvv \
-    --ignore-installed \
-    --no-cache \
-    --no-deps \
-    --no-index \
-    --root=${D} \
-    --prefix=${prefix} \
-"
-
 PIP_INSTALL_PYTHON = "python3"
 PIP_INSTALL_PYTHON:class-native = "nativepython3"
 
+INSTALL_WHEEL_COMPILE_BYTECODE ?= "--compile-bytecode=0"
+
 pip_install_wheel_do_install () {
     COUNT=$(find ${PIP_INSTALL_DIST_PATH} -name '*.whl' | wc -l)
     if test $COUNT -eq 0; then
@@ -26,7 +18,7 @@  pip_install_wheel_do_install () {
         bbfatal More than one wheel found in ${PIP_INSTALL_DIST_PATH}, this should not happen
     fi
 
-    nativepython3 -m pip install ${PIP_INSTALL_ARGS} ${PIP_INSTALL_DIST_PATH}/*.whl
+    nativepython3 -m installer ${INSTALL_WHEEL_COMPILE_BYTECODE} --destdir=${D} ${PIP_INSTALL_DIST_PATH}/*.whl
 
     cd ${D}
     for i in ${D}${bindir}/* ${D}${sbindir}/*; do
diff --git a/meta/recipes-devtools/python/python3-flit-core_3.7.1.bb b/meta/recipes-devtools/python/python3-flit-core_3.7.1.bb
index 1f14345d50..d4993fee69 100644
--- a/meta/recipes-devtools/python/python3-flit-core_3.7.1.bb
+++ b/meta/recipes-devtools/python/python3-flit-core_3.7.1.bb
@@ -13,7 +13,7 @@  SRC_URI[sha256sum] = "3c9bd9c140515bfe62dd938c6610d10d6efb9e35cc647fc614fe5fb3a5
 inherit pypi flit_core
 
 # Need to install by hand as there's a dependency loop
-DEPENDS:remove:class-native = " python3-pip-native"
+DEPENDS:remove:class-native = " python3-installer-native"
 DEPENDS:append:class-native = " unzip-native"
 
 # We need the full flit tarball
diff --git a/meta/recipes-devtools/python/python3-pip_22.0.3.bb b/meta/recipes-devtools/python/python3-pip_22.0.3.bb
index bce3b68861..9ca8fbc1e5 100644
--- a/meta/recipes-devtools/python/python3-pip_22.0.3.bb
+++ b/meta/recipes-devtools/python/python3-pip_22.0.3.bb
@@ -6,43 +6,14 @@  LIC_FILES_CHKSUM = "file://LICENSE.txt;md5=63ec52baf95163b597008bb46db68030"
 
 inherit pypi setuptools_build_meta
 
-DEPENDS += "python3"
-
-# To avoid a dependency loop; we bootstrap -native
-DEPENDS:remove:class-native = "python3-pip-native"
-DEPENDS:append:class-native = " unzip-native"
-
 SRC_URI += "file://0001-change-shebang-to-python3.patch"
 SRC_URI += "file://no_shebang_mangling.patch"
 SRC_URI += "file://reproducible.patch"
 
 SRC_URI[sha256sum] = "f29d589df8c8ab99c060e68ad294c4a9ed896624f6368c5349d70aa581b333d0"
 
-do_install:class-native() {
-    pip_install_wheel_do_bootstrap_install
-
-    # pip install would normally generate [console_scripts] in ${bindir}
-    install -d ${D}/${bindir}
-    # We will skip the ${bindir}/pip variant as we would just remove it in the do_install:append
-    cat << EOF >> ${D}/${bindir}/pip3 | tee ${D}/${bindir}/pip${PYTHON_BASEVERSION}
-#!/bin/sh
-'''exec' ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} "\$0" "\$@"
-' '''
-# -*- coding: utf-8 -*-
-import re
-import sys
-from pip._internal.cli.main import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
-    sys.exit(main())
-EOF
-    chmod 0755 ${D}${bindir}/pip3 ${D}${bindir}/pip${PYTHON_BASEVERSION}
-}
-
 do_install:append() {
-    if [ -e ${D}/${bindir}/pip ]; then
-        rm ${D}/${bindir}/pip
-    fi
+    rm -f ${D}/${bindir}/pip
 }
 
 RDEPENDS:${PN} = "\
@@ -59,3 +30,7 @@  RDEPENDS:${PN} = "\
 "
 
 BBCLASSEXTEND = "native nativesdk"
+
+# This used to use the bootstrap install which didn't compile. Until we bump the
+# tmpdir version we can't compile the native otherwise the sysroot unpack fails
+INSTALL_WHEEL_COMPILE_BYTECODE:class-native = "--no-compile-bytecode"
diff --git a/meta/recipes-devtools/python/python3-setuptools_59.5.0.bb b/meta/recipes-devtools/python/python3-setuptools_59.5.0.bb
index 7cd67b85f9..35bec19e33 100644
--- a/meta/recipes-devtools/python/python3-setuptools_59.5.0.bb
+++ b/meta/recipes-devtools/python/python3-setuptools_59.5.0.bb
@@ -17,14 +17,6 @@  SRC_URI[sha256sum] = "d144f85102f999444d06f9c0e8c737fd0194f10f2f7e5fdb77573f6e2f
 
 DEPENDS += "${PYTHON_PN}"
 
-# Avoid dependency loop; we bootstrap -native
-DEPENDS:remove:class-native = "python3-pip-native python3-setuptools-native"
-DEPENDS:append:class-native = " unzip-native"
-
-do_install:class-native() {
-    pip_install_wheel_do_bootstrap_install
-}
-
 RDEPENDS:${PN} = "\
     ${PYTHON_PN}-2to3 \
     ${PYTHON_PN}-compile \
@@ -59,3 +51,7 @@  RDEPENDS:${PYTHON_PN}-pkg-resources = "\
     ${PYTHON_PN}-plistlib \
     ${PYTHON_PN}-pprint \
 "
+
+# This used to use the bootstrap install which didn't compile. Until we bump the
+# tmpdir version we can't compile the native otherwise the sysroot unpack fails
+INSTALL_WHEEL_COMPILE_BYTECODE:class-native = "--no-compile-bytecode"
diff --git a/meta/recipes-devtools/python/python3-wheel_0.37.1.bb b/meta/recipes-devtools/python/python3-wheel_0.37.1.bb
index cf0d8191b4..efd6c2f968 100644
--- a/meta/recipes-devtools/python/python3-wheel_0.37.1.bb
+++ b/meta/recipes-devtools/python/python3-wheel_0.37.1.bb
@@ -10,26 +10,8 @@  inherit flit_core pypi
 
 SRC_URI += " file://0001-Backport-pyproject.toml-from-flit-backend-branch.patch"
 
-DEPENDS:remove:class-native = "python3-pip-native"
-
-do_install:class-native () {
-    pip_install_wheel_do_bootstrap_install
-
-    # pip install would normally generate [project.scripts] in ${bindir}
-    install -d ${D}/${bindir}
-    cat << EOF >> ${D}/${bindir}/wheel
-#!/bin/sh
-'''exec' ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} "\$0" "\$@"
-' '''
-# -*- coding: utf-8 -*-
-import re
-import sys
-from wheel.cli import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
-    sys.exit(main())
-EOF
-    chmod 0755 ${D}${bindir}/wheel
-}
-
 BBCLASSEXTEND = "native nativesdk"
+
+# This used to use the bootstrap install which didn't compile. Until we bump the
+# tmpdir version we can't compile the native otherwise the sysroot unpack fails
+INSTALL_WHEEL_COMPILE_BYTECODE:class-native = "--no-compile-bytecode"