diff mbox series

[v2,1/3] archiver.bbclass: archive shallow git sources which have no clone

Message ID 20261030174536.2724008-2-ecordonnier@snap.com
State New
Headers show
Series *** archiver.bbclass: archive shallow git sources which have no clone *** | expand

Commit Message

Etienne Cordonnier Oct. 30, 2026, 5:45 p.m. UTC 
From: Etienne Cordonnier <ecordonnier@snap.com>

In the original mode, do_ar_original archives a source if the fetcher's
localpath is a file, or a directory (which is unpacked and tarred). For git,
localpath is the clone in DL_DIR/git2. A shallow fetch (BB_GIT_SHALLOW) can
use the gitshallow tarball (ud.fullshallow) without creating the clone: since
bitbake commit 457288b2f ("fetch2/git: Add support for fast initial shallow
fetch") this is what a shallow fetch does by default, and it also happens when
the shallow tarball is fetched from a mirror. In this case, the source was
silently skipped and not archived at all.

Treat this case like a directory source, as unpack() populates the
directory from the shallow tarball. Use the same check as unpack()
(clonedir_need_update()), so that the case of an out of date clone, where
unpack() also uses the shallow tarball, is handled as well.

AI-Generated: Claude Sonnet 5.5
Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
---
 meta/classes/archiver.bbclass            |  8 +++-
 meta/lib/oeqa/selftest/cases/archiver.py | 48 ++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 1 deletion(-)
diff mbox series

Patch

diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass
index 0400d6e373..f6cca35186 100644
--- a/meta/classes/archiver.bbclass
+++ b/meta/classes/archiver.bbclass
@@ -204,10 +204,16 @@  python do_ar_original() {
     fetch = bb.fetch.Fetch(urls, ld)
     tarball_suffix = {}
     for url in fetch.urls:
+        ud = fetch.ud[url]
         local = fetch.localpath(url).rstrip("/");
+        # A shallow git fetch which used the shallow tarball has no up to date
+        # clone, which is what localpath() points to. unpack() then uses the
+        # shallow tarball, using the same check as here.
+        shallow = getattr(ud, 'shallow', False) and os.path.isfile(ud.fullshallow) \
+                  and ud.method.clonedir_need_update(ud, ld)
         if os.path.isfile(local):
             shutil.copy(local, ar_outdir)
-        elif os.path.isdir(local):
+        elif os.path.isdir(local) or shallow:
             tmpdir = tempfile.mkdtemp(dir=d.getVar('ARCHIVER_WORKDIR'))
             fetch.unpack(tmpdir, (url,))
             # To handle recipes with more than one source, we add the "name"
diff --git a/meta/lib/oeqa/selftest/cases/archiver.py b/meta/lib/oeqa/selftest/cases/archiver.py
index 8c5aa75e9b..84f13839c9 100644
--- a/meta/lib/oeqa/selftest/cases/archiver.py
+++ b/meta/lib/oeqa/selftest/cases/archiver.py
@@ -7,6 +7,8 @@ 
 import os
 import glob
 import re
+import shutil
+import tarfile
 from oeqa.utils.commands import bitbake, get_bb_vars
 from oeqa.selftest.case import OESelftestTestCase
 
@@ -176,6 +178,52 @@  class Archiver(OESelftestTestCase):
 
         self._test_archiver_mode('original', 'ed-1.21.1.tar.lz')
 
+    def test_archiver_mode_original_git_shallow(self):
+        """
+        Test that the archiver archives a shallow git source with
+        `ARCHIVER_MODE[src] = "original"` when only the shallow tarball is
+        available, but not the clone. This is what happens with the fast
+        initial shallow fetch, or when the shallow tarball is fetched from a
+        mirror.
+        """
+        target = 'gitrepotest'
+
+        features = 'INHERIT += "archiver"\n'
+        features += 'ARCHIVER_MODE[src] = "original"\n'
+        features += 'BB_GIT_SHALLOW = "1"\n'
+        features += 'BB_GENERATE_SHALLOW_TARBALLS = "1"\n'
+        features += 'DL_DIR = "${TOPDIR}/downloads-shallow"\n'
+        self.write_config(features)
+
+        # Fetch to create the shallow tarball, then remove the clone if the
+        # fetcher fell back to a full clone, so that the second fetch has to
+        # use the shallow tarball. cleansstate makes sure that the archive is
+        # created again and not restored from sstate.
+        bitbake('-c cleansstate %s' % target)
+        bitbake('-c fetch %s' % target)
+        bb_vars = get_bb_vars(['DL_DIR'])
+        self.track_for_cleanup(bb_vars['DL_DIR'])
+        self.assertTrue(glob.glob(os.path.join(bb_vars['DL_DIR'], 'gitshallow_*.tar.gz')),
+                        'Missing shallow tarball in DL_DIR')
+        shutil.rmtree(os.path.join(bb_vars['DL_DIR'], 'git2'))
+        bitbake('-c cleansstate %s' % target)
+
+        bitbake('-c deploy_archives %s' % target)
+
+        names = self._get_original_git_archive_names(target)
+        self.assertTrue([n for n in names if n.endswith('/COPYING')], 'Archive has no source files: %s' % names[:20])
+
+    def _get_original_git_archive_names(self, target):
+        """
+        Return the names of the files in the original source archive of the
+        git source of target.
+        """
+        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC'], target)
+        archives = glob.glob(os.path.join(bb_vars['DEPLOY_DIR_SRC'], '*', '%s-*' % target, '%s-*.tar.*' % target))
+        self.assertEqual(len(archives), 1, 'Expected one source archive for %s, found %s' % (target, archives))
+        with tarfile.open(archives[0]) as tar:
+            return tar.getnames()
+
     def test_archiver_mode_patched(self):
         """
         Test that the archiver works with `ARCHIVER_MODE[src] = "patched"`.