@@ -494,11 +494,7 @@ def main():
layerrecipes_add = []
# Check if any paths should be ignored because there are layers within this layer
- removedirs = []
- for root, dirs, files in os.walk(layerdir):
- for diritem in dirs:
- if os.path.exists(os.path.join(root, diritem, 'conf', 'layer.conf')):
- removedirs.append(os.path.join(root, diritem) + os.sep)
+ removedirs = utils.nested_layer_dirs(layerdir)
if diff:
# Apply git changes to existing recipe list
@@ -515,19 +511,17 @@ def main():
for diffitem in diff.iter_change_type('R'):
oldpath = diffitem.a_blob.path
newpath = diffitem.b_blob.path
- skip = False
- for removedir in removedirs:
- # FIXME what about files moved into removedirs?
- if oldpath.startswith(removedir):
- skip = True
- break
- if skip:
+ if utils.in_nested_layer(oldpath, removedirs, repodir):
continue
if oldpath.startswith(subdir_start):
if not newpath.startswith(subdir_start):
logger.debug("Treating rename of %s to %s as a delete since new path is outside layer" % (oldpath, newpath))
other_deletes.append(diffitem)
continue
+ if utils.in_nested_layer(newpath, removedirs, repodir):
+ logger.debug("Treating rename of %s to %s as a delete since new path is in a layer within this layer" % (oldpath, newpath))
+ other_deletes.append(diffitem)
+ continue
(oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
(newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
if oldtypename != newtypename:
@@ -606,12 +600,7 @@ def main():
for diffitem in itertools.chain(diff.iter_change_type('D'), other_deletes):
path = diffitem.a_blob.path
if path.startswith(subdir_start):
- skip = False
- for removedir in removedirs:
- if path.startswith(removedir):
- skip = True
- break
- if skip:
+ if utils.in_nested_layer(path, removedirs, repodir):
continue
(typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
if typename == 'recipe':
@@ -636,12 +625,7 @@ def main():
for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds):
path = diffitem.b_blob.path
if path.startswith(subdir_start):
- skip = False
- for removedir in removedirs:
- if path.startswith(removedir):
- skip = True
- break
- if skip:
+ if utils.in_nested_layer(path, removedirs, repodir):
continue
(typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
if typename == 'recipe':
@@ -685,12 +669,7 @@ def main():
for diffitem in diff.iter_change_type('M'):
path = diffitem.b_blob.path
if path.startswith(subdir_start):
- skip = False
- for removedir in removedirs:
- if path.startswith(removedir):
- skip = True
- break
- if skip:
+ if utils.in_nested_layer(path, removedirs, repodir):
continue
(typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
if typename == 'recipe':
@@ -57,6 +57,28 @@ def get_layer_var(config_data, var, logger):
value = config_data.getVar(var, True)
return value or ''
+def nested_layer_dirs(layerdir):
+ """
+ Return the directories below layerdir that contain a conf/layer.conf of
+ their own, i.e. other layers within this layer, each with a trailing
+ separator.
+ """
+ dirs = []
+ for root, subdirs, _ in os.walk(layerdir):
+ for subdir in subdirs:
+ if os.path.exists(os.path.join(root, subdir, 'conf', 'layer.conf')):
+ dirs.append(os.path.join(root, subdir) + os.sep)
+ return dirs
+
+def in_nested_layer(path, nested_dirs, repodir):
+ """
+ Check whether path is within one of nested_dirs (as returned by
+ nested_layer_dirs()). path may be relative to repodir, as git diff paths
+ are, or absolute.
+ """
+ fullpath = os.path.join(repodir, path)
+ return any(fullpath.startswith(nested_dir) for nested_dir in nested_dirs)
+
def is_deps_satisfied(req_col, req_ver, collections):
""" Check whether required collection and version are in collections"""
for existed_col, existed_ver in collections:
@@ -5,6 +5,7 @@
# SPDX-License-Identifier: MIT
import operator
+import os
import sys
import types
@@ -40,3 +41,47 @@ def fake_bb(monkeypatch):
])
def test_is_deps_satisfied(fake_bb, req_ver, collections, expected):
assert utils.is_deps_satisfied('core', req_ver, collections) is expected
+
+
+def make_layer(path):
+ (path / 'conf').mkdir(parents=True)
+ (path / 'conf' / 'layer.conf').write_text('')
+
+
+@pytest.fixture
+def repo_with_nested_layers(tmp_path):
+ """A repository whose layer (in subdirectory 'meta-top') contains other
+ layers, as when layers are vendored with git subtree."""
+ repodir = tmp_path / 'repo'
+ layerdir = repodir / 'meta-top'
+ make_layer(layerdir)
+ (layerdir / 'recipes-core' / 'foo').mkdir(parents=True)
+ make_layer(layerdir / 'meta-sub')
+ make_layer(layerdir / 'meta-openembedded' / 'meta-oe')
+ # A directory whose name starts like a nested layer's, but isn't one
+ (layerdir / 'meta-subsidiary' / 'recipes-x').mkdir(parents=True)
+ return str(repodir), str(layerdir)
+
+
+def test_nested_layer_dirs(repo_with_nested_layers):
+ repodir, layerdir = repo_with_nested_layers
+ assert sorted(utils.nested_layer_dirs(layerdir)) == [
+ os.path.join(layerdir, 'meta-openembedded', 'meta-oe') + os.sep,
+ os.path.join(layerdir, 'meta-sub') + os.sep,
+ ]
+
+
+@pytest.mark.parametrize('path, expected', [
+ # Paths relative to the repository, as git diff reports them
+ ('meta-top/meta-openembedded/meta-oe/recipes-a/a/a_1.0.bb', True),
+ ('meta-top/meta-sub/recipes-b/b/b_1.0.bb', True),
+ ('meta-top/recipes-core/foo/foo_1.0.bb', False),
+ ('meta-top/meta-subsidiary/recipes-x/x/x_1.0.bb', False),
+ ('meta-top/meta-openembedded/README', False),
+])
+def test_in_nested_layer(repo_with_nested_layers, path, expected):
+ repodir, layerdir = repo_with_nested_layers
+ nested = utils.nested_layer_dirs(layerdir)
+ assert utils.in_nested_layer(path, nested, repodir) is expected
+ # Absolute paths give the same answer
+ assert utils.in_nested_layer(os.path.join(repodir, path), nested, repodir) is expected
On incremental updates the check for layers within a layer compared repository-relative git diff paths with absolute directory paths, so it never matched, and files in nested layers (e.g. meta-voltumna's vendored copies of oe-core and meta-openembedded) were indexed as part of the outer layer. Move the check into utils.nested_layer_dirs() and utils.in_nested_layer(), use it for all change types, and treat renames into a nested layer as deletes. Records already indexed this way are removed by a full rescan (update.py -r). AI-Generated: Claude Opus 5.5 Signed-off-by: Tim Orling <tim.orling@konsulko.com> --- layerindex/update_layer.py | 39 ++++++++------------------------- layerindex/utils.py | 22 +++++++++++++++++++ tests/test_utils.py | 45 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 30 deletions(-)