diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index c213a9a3ca6..8ec00b96b69 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -1010,7 +1010,15 @@ def copydebugsources(debugsrcdir, sources, d):
         bb.utils.mkdirhier(basepath)
         cpath.updatecache(basepath)
 
-        for pmap in prefixmap:
+        # Re-order the prefix list so that we sweep ${B} first, as the sstate creation will
+        # clamp mtimes of hardlinks in the -src package, and that will change the source.
+        # (see #15491)
+        b = d.getVar("B")
+        prefixmap_keys = list(prefixmap)
+        prefixmap_keys.remove(b)
+        prefixmap_keys.insert(0, b)
+
+        for pmap in prefixmap_keys:
             # Ignore files from the recipe sysroots (target and native)
             cmd =  "LC_ALL=C ; sort -z -u '%s' | egrep -v -z '((<internal>|<built-in>)$|/.*recipe-sysroot.*/)' | " % sourcefile
             # We need to ignore files that are not actually ours