diff mbox series

python3-numpy: Fix reproducibility issue

Message ID 20230929215615.2543438-1-richard.purdie@linuxfoundation.org
State Accepted, archived
Commit c4d1dc5e33734b94835e7f5e0e1746d4a6542b55
Headers show
Series python3-numpy: Fix reproducibility issue | expand

Commit Message

Richard Purdie Sept. 29, 2023, 9:56 p.m. UTC
Add a patch to fix the reproducibility issue being seen during testing. The issue
is from a poor choice of regex against the compiler flags, being triggered by
different path names.

Drop the previous attempt at fixing this as it wasn't that.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 .../python/python3-numpy/disable_blas.patch   | 37 -------------------
 .../python3-numpy/fix_reproducibility.patch   | 33 +++++++++++++++++
 .../python/python3-numpy_1.26.0.bb            |  2 +-
 3 files changed, 34 insertions(+), 38 deletions(-)
 delete mode 100644 meta/recipes-devtools/python/python3-numpy/disable_blas.patch
 create mode 100644 meta/recipes-devtools/python/python3-numpy/fix_reproducibility.patch
diff mbox series

Patch

diff --git a/meta/recipes-devtools/python/python3-numpy/disable_blas.patch b/meta/recipes-devtools/python/python3-numpy/disable_blas.patch
deleted file mode 100644
index 09d65e84190..00000000000
--- a/meta/recipes-devtools/python/python3-numpy/disable_blas.patch
+++ /dev/null
@@ -1,37 +0,0 @@ 
-We've seen reproducibility failures where it appears an extra -O3 compiler flag ends
-up in the multiarray library compilation. This can only really have come through
-extra_info since it only affects just this library. Rather than try and track
-down exactly where this came from in a rabbit warren of code, just disable
-this since we don't have any of the dependencies.
-
-Upstream-Status: Inappropriate [OE specific config hack]
-Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-
-Index: numpy-1.25.2/numpy/core/setup.py
-===================================================================
---- numpy-1.25.2.orig/numpy/core/setup.py
-+++ numpy-1.25.2/numpy/core/setup.py
-@@ -755,22 +755,7 @@ def configuration(parent_package='',top_
-             join('src', 'common', 'npy_cpu_features.c'),
-             ]
- 
--    if os.environ.get('NPY_USE_BLAS_ILP64', "0") != "0":
--        blas_info = get_info('blas_ilp64_opt', 2)
--    else:
--        blas_info = get_info('blas_opt', 0)
--
--    have_blas = blas_info and ('HAVE_CBLAS', None) in blas_info.get('define_macros', [])
--
--    if have_blas:
--        extra_info = blas_info
--        # These files are also in MANIFEST.in so that they are always in
--        # the source distribution independently of HAVE_CBLAS.
--        common_src.extend([join('src', 'common', 'cblasfuncs.c'),
--                           join('src', 'common', 'python_xerbla.c'),
--                          ])
--    else:
--        extra_info = {}
-+    extra_info = {}
- 
-     #######################################################################
-     #             _multiarray_umath module - multiarray part              #
diff --git a/meta/recipes-devtools/python/python3-numpy/fix_reproducibility.patch b/meta/recipes-devtools/python/python3-numpy/fix_reproducibility.patch
new file mode 100644
index 00000000000..d952aed00c6
--- /dev/null
+++ b/meta/recipes-devtools/python/python3-numpy/fix_reproducibility.patch
@@ -0,0 +1,33 @@ 
+This regex decides whether to use O3 opimisation on numpy or not.
+
+It includes "od", which happens to be a substring of "reproducible"
+but not "qemux86-world". 
+
+The regex will run against all compiler options including things like:
+
+-fmacro-prefix-map=/XXX/build/tmp/work/core2-64-poky-linux/python3-numpy/1.26.0/numpy-1.26.0=/usr/src/debug/python3-numpy/1.26.0-r0
+
+i.e. including build paths.
+
+Reduce the regex to something deterministic for our builds, assuming
+nobody builds in /home/debug:full/
+
+The autobuilder race depended upon whether qemux86-world or the
+reproducible target ran first and won the race to populate sstate.
+
+Upstream-Status: Inappropriate [upstream have dropped distutils and switched to meson]
+Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
+
+Index: numpy-1.26.0/numpy/distutils/ccompiler_opt.py
+===================================================================
+--- numpy-1.26.0.orig/numpy/distutils/ccompiler_opt.py
++++ numpy-1.26.0/numpy/distutils/ccompiler_opt.py
+@@ -990,7 +990,7 @@ class _CCompiler:
+             ("cc_is_nocc",     "", ""),
+         )
+         detect_args = (
+-           ("cc_has_debug",  ".*(O0|Od|ggdb|coverage|debug:full).*", ""),
++           ("cc_has_debug",  ".*debug:full.*", ""),
+            ("cc_has_native",
+                 ".*(-march=native|-xHost|/QxHost|-mcpu=a64fx).*", ""),
+            # in case if the class run with -DNPY_DISABLE_OPTIMIZATION
diff --git a/meta/recipes-devtools/python/python3-numpy_1.26.0.bb b/meta/recipes-devtools/python/python3-numpy_1.26.0.bb
index a1f398338ee..e6aaafe7dd4 100644
--- a/meta/recipes-devtools/python/python3-numpy_1.26.0.bb
+++ b/meta/recipes-devtools/python/python3-numpy_1.26.0.bb
@@ -10,7 +10,7 @@  SRCNAME = "numpy"
 SRC_URI = "${GITHUB_BASE_URI}/download/v${PV}/${SRCNAME}-${PV}.tar.gz \
            file://0001-Don-t-search-usr-and-so-on-for-libraries-by-default-.patch \
            file://0001-numpy-core-Define-RISCV-32-support.patch \
-           file://disable_blas.patch \
+           file://fix_reproducibility.patch \
            file://run-ptest \
            "
 SRC_URI[sha256sum] = "f93fc78fe8bf15afe2b8d6b6499f1c73953169fad1e9a8dd086cdff3190e7fdf"