diff --git a/bin/bitbake-selftest b/bin/bitbake-selftest index fb7c57dd8..1b09815cd 100755 --- a/bin/bitbake-selftest +++ b/bin/bitbake-selftest @@ -21,6 +21,7 @@ except RuntimeError as exc: sys.exit(str(exc)) tests = ["bb.tests.codeparser", + "bb.tests.cache_extra", "bb.tests.color", "bb.tests.cooker", "bb.tests.cow", diff --git a/lib/bb/cache.py b/lib/bb/cache.py index 2361c5684..33fa61936 100644 --- a/lib/bb/cache.py +++ b/lib/bb/cache.py @@ -28,7 +28,7 @@ import shutil logger = logging.getLogger("BitBake.Cache") -__cache_version__ = "156" +__cache_version__ = "157" def getCacheFile(path, filename, mc, data_hash): mcspec = '' diff --git a/lib/bb/cache_extra.py b/lib/bb/cache_extra.py index bf4226d16..78ef6b638 100644 --- a/lib/bb/cache_extra.py +++ b/lib/bb/cache_extra.py @@ -13,7 +13,65 @@ # SPDX-License-Identifier: GPL-2.0-only # +import logging +import json +import zlib + from bb.cache import RecipeInfoCommon +logger = logging.getLogger("BitBake.CacheExtra") + + +def _winning_override(metadata, variable_name): + active = {} + metadata.need_overrides() + for override_variable, override in metadata.overridedata.get(variable_name, ()): + if (override in metadata.overridesset or + ':' in override and + set(override.split(':')).issubset(metadata.overridesset)): + active[override] = override_variable + + match = None + modified = True + while modified: + modified = False + for override in metadata.overrides: + for candidate in active.copy(): + if candidate.endswith(':' + override): + active[candidate.removesuffix(':' + override)] = active.pop(candidate) + modified = True + elif candidate == override: + match = active.pop(candidate) + return match + + +def _recipe_includes(metadata, filename): + includes = set() + + def collect(node): + for child in node.children: + if child.filename.endswith('.bbclass'): + continue + if child.filename.endswith('.inc'): + includes.add(child.filename) + collect(child) + + for child in metadata.inchistory.children: + if child.filename == filename: + collect(child) + return includes + + +def _recipe_value_event(metadata, event, recipe_includes): + event_file = event.get('file', '') + if (not event_file.endswith(('.bb', '.bbappend')) and + event_file not in recipe_includes): + return False + operation = event.get('op', '') + if '[' not in operation: + return True + override = operation.rsplit('[', 1)[1].removesuffix(']') + return set(override.split(':')).issubset(metadata.overridesset) + class HobRecipeInfo(RecipeInfoCommon): __slots__ = () @@ -27,7 +85,7 @@ class HobRecipeInfo(RecipeInfoCommon): # that this class will provide cachefields = ['summary', 'license', 'section', 'description', 'homepage', 'bugtracker', - 'prevision', 'files_info'] + 'prevision', 'files_info', 'recipe_variables'] def __init__(self, filename, metadata): @@ -39,6 +97,45 @@ class HobRecipeInfo(RecipeInfoCommon): self.bugtracker = self.getvar('BUGTRACKER', metadata) self.prevision = self.getvar('PR', metadata) self.files_info = self.getvar('FILES_INFO', metadata) + recipe_variables = {} + recipe_includes = _recipe_includes(metadata, filename) + for variable_name in metadata: + try: + if ':' in variable_name: + continue + winning_override = _winning_override(metadata, variable_name) + history = metadata.varhistory.variable(variable_name) + if winning_override: + history += metadata.varhistory.variable(winning_override) + if not any(_recipe_value_event(metadata, event, recipe_includes) + for event in history): + # Keep this snapshot limited to recipe metadata. + continue + if (metadata.getVarFlag(variable_name, 'func', False) or + winning_override and + metadata.getVarFlag(winning_override, 'func', False)): + continue + value = metadata.getVar(variable_name, True) + recipe_variables[variable_name] = '' if value is None else str(value) + flag_sources = [variable_name] + if winning_override and winning_override != variable_name: + flag_sources.append(winning_override) + for flag_source in flag_sources: + for flag in metadata.getVarFlags(flag_source) or {}: + if flag == 'func': + continue + try: + flag_value = metadata.getVarFlag(flag_source, flag, True) + if flag_value is not None: + recipe_variables['%s[%s]' % (variable_name, flag)] = str(flag_value) + except Exception as exc: + logger.debug("Omitting recipe variable flag %s[%s] from %s after %s", + variable_name, flag, filename, type(exc).__name__) + except Exception as exc: + logger.debug("Omitting recipe variable %s from %s after %s", + variable_name, filename, type(exc).__name__) + self.recipe_variables = zlib.compress( + json.dumps(recipe_variables, separators=(',', ':')).encode('utf-8')) @classmethod def init_cacheData(cls, cachedata): @@ -51,6 +148,7 @@ class HobRecipeInfo(RecipeInfoCommon): cachedata.bugtracker = {} cachedata.prevision = {} cachedata.files_info = {} + cachedata.recipe_variables = {} def add_cacheData(self, cachedata, fn): cachedata.summary[fn] = self.summary @@ -61,3 +159,4 @@ class HobRecipeInfo(RecipeInfoCommon): cachedata.bugtracker[fn] = self.bugtracker cachedata.prevision[fn] = self.prevision cachedata.files_info[fn] = self.files_info + cachedata.recipe_variables[fn] = self.recipe_variables diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py index 59f808c96..e68829813 100644 --- a/lib/bb/cookerdata.py +++ b/lib/bb/cookerdata.py @@ -507,8 +507,9 @@ class CookerDataBuilder(object): return data - @staticmethod - def _parse_recipe(bb_data, bbfile, appends, mc, layername): + def _parse_recipe(self, bb_data, bbfile, appends, mc, layername): + if self.tracking: + bb_data.enableTracking() bb_data.setVar("__BBMULTICONFIG", mc) bb_data.setVar("FILE_LAYERNAME", layername) diff --git a/lib/bb/tests/cache_extra.py b/lib/bb/tests/cache_extra.py new file mode 100644 index 000000000..c695bb185 --- /dev/null +++ b/lib/bb/tests/cache_extra.py @@ -0,0 +1,235 @@ +# +# BitBake Tests for extra cache data +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import unittest +import json +import os +import tempfile +import zlib + +import bb.data +import bb.parse +import bb.siggen +from bb.cookerdata import CookerDataBuilder +from bb.cache_extra import HobRecipeInfo + + +class HobRecipeInfoTest(unittest.TestCase): + + @staticmethod + def metadata(): + metadata = bb.data.init() + metadata.enableTracking() + return metadata + + def test_recipe_variables(self): + metadata = self.metadata() + metadata.setVar('TEXT', '${VALUE}', file='test.bb', line=1) + metadata.setVar('VALUE', 'expanded', file='test.bb', line=2) + metadata.setVar('EMPTY', '', file='test.bb', line=3) + metadata.setVar('NUMBER', 7, file='test.bbappend', line=1) + metadata.setVar('SPECIAL', 'café\nline\x00end', + file='test.bb', line=4) + metadata.setVar('INHERITED', 'global', file='test.inc', line=1) + metadata.setVar('__INTERNAL', 'internal', file='test.bbclass', line=1) + metadata.setVar('FROM_CONFIG', 'config', file='conf/local.conf', line=1) + metadata.setVar('do_function', 'echo test', file='test.bb', line=5) + metadata.setVarFlag('do_function', 'func', True) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['TEXT'], 'expanded') + self.assertEqual(variables['EMPTY'], '') + self.assertEqual(variables['NUMBER'], '7') + self.assertEqual(variables['SPECIAL'], 'café\nline\x00end') + self.assertNotIn('INHERITED', variables) + self.assertNotIn('__INTERNAL', variables) + self.assertNotIn('FROM_CONFIG', variables) + self.assertNotIn('do_function', variables) + + def test_parsed_recipe_variables_include_recipe_inc_but_not_class(self): + metadata = self.metadata() + metadata.setVar('__bbclasstype', 'recipe') + bb.parse.siggen = bb.siggen.init(metadata) + metadata.disableTracking() + builder = object.__new__(CookerDataBuilder) + builder.tracking = True + + with tempfile.TemporaryDirectory() as tempdir: + recipe = os.path.join(tempdir, 'test.bb') + include = os.path.join(tempdir, 'test.inc') + classes = os.path.join(tempdir, 'classes') + os.mkdir(classes) + metadata.setVar('BBPATH', tempdir) + with open(include, 'w') as handle: + handle.write('FROM_INC = "inc"\n') + with open(os.path.join(tempdir, 'class.inc'), 'w') as handle: + handle.write('FROM_CLASS_INC = "class-inc"\n') + with open(os.path.join(classes, 'testclass.bbclass'), 'w') as handle: + handle.write('require class.inc\nFROM_CLASS = "class"\n') + with open(recipe, 'w') as handle: + handle.write('require test.inc\ninherit testclass\nFROM_RECIPE = "recipe"\n') + + parsed = builder._parse_recipe(metadata, recipe, [], '', '')[''] + info = HobRecipeInfo(recipe, parsed) + + variables = json.loads(zlib.decompress(info.recipe_variables)) + self.assertEqual(variables['FROM_RECIPE'], 'recipe') + self.assertEqual(variables['FROM_INC'], 'inc') + self.assertNotIn('FROM_CLASS', variables) + self.assertNotIn('FROM_CLASS_INC', variables) + + def test_recipe_variable_operations_are_included(self): + metadata = self.metadata() + metadata.setVar('TEXT', 'global', file='conf/bitbake.conf', line=1) + metadata.setVar('TEXT:append', ' recipe', file='test.bb', line=1) + metadata.setVar('TEXT:remove', 'global', file='test.bbappend', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['TEXT'], ' recipe') + + def test_only_effective_overrides_are_included(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'machine', file='conf/bitbake.conf', line=1) + metadata.setVar('ACTIVE:machine', 'recipe', file='test.bb', line=1) + metadata.setVar('INACTIVE:other', 'recipe', file='test.bb', line=2) + metadata.setVar('GLOBAL', 'global', file='conf/bitbake.conf', line=2) + metadata.setVar('GLOBAL:append:other', ' recipe', file='test.bb', line=3) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['ACTIVE'], 'recipe') + self.assertNotIn('ACTIVE:machine', variables) + self.assertNotIn('INACTIVE', variables) + self.assertNotIn('INACTIVE:other', variables) + self.assertNotIn('GLOBAL', variables) + + def test_weak_default_is_included(self): + metadata = self.metadata() + metadata.setVarFlag('PACKAGECONFIG', '_defaultval', 'feature', + file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['PACKAGECONFIG'], 'feature') + + def test_variable_flags_are_included(self): + metadata = self.metadata() + metadata.setVar('PACKAGECONFIG', 'feature', file='test.bb', line=1) + metadata.setVarFlag('PACKAGECONFIG', 'feature', '--enable-feature', + file='test.inc', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['PACKAGECONFIG[feature]'], '--enable-feature') + + def test_active_override_flags_are_included(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'machine', file='test.inc', line=1) + metadata.setVar('PACKAGECONFIG:machine', 'feature', + file='test.bbappend', line=2) + metadata.setVarFlag('PACKAGECONFIG:machine', 'feature', + '--enable-feature', file='test.bbappend', line=3) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['PACKAGECONFIG'], 'feature') + self.assertEqual(variables['PACKAGECONFIG[feature]'], '--enable-feature') + + def test_active_override_flag_wins_over_base_flag(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'machine', file='conf/local.conf', line=1) + metadata.setVar('PACKAGECONFIG:machine', 'feature', + file='test.bb', line=2) + metadata.setVarFlag('PACKAGECONFIG', 'feature', '--base', + file='test.inc', line=3) + metadata.setVarFlag('PACKAGECONFIG:machine', 'feature', '--machine', + file='test.bbappend', line=4) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['PACKAGECONFIG[feature]'], '--machine') + + def test_inherited_only_values_are_excluded(self): + metadata = self.metadata() + metadata.setVar('FROM_INC', 'inc', file='test.inc', line=1) + metadata.setVar('FROM_CLASS', 'class', file='test.bbclass', line=2) + metadata.setVar('FROM_CONFIG', 'config', file='conf/local.conf', line=3) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertNotIn('FROM_INC', variables) + self.assertNotIn('FROM_CLASS', variables) + self.assertNotIn('FROM_CONFIG', variables) + + def test_combined_override_is_included_under_logical_name(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'foo:bar:local', + file='conf/bitbake.conf', line=1) + metadata.setVar('COMBINED:local:foo:bar', 'recipe', + file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['COMBINED'], 'recipe') + self.assertNotIn('OVERRIDES', variables) + + def test_active_override_function_is_omitted(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'machine', file='conf/bitbake.conf', line=1) + metadata.setVar('do_work:machine', 'echo test', file='test.bb', line=1) + metadata.setVarFlag('do_work:machine', 'func', True, + file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertNotIn('do_work', variables) + + def test_recipe_variables_survive_extra_cache_mapping(self): + metadata = self.metadata() + metadata.setVar('EMPTY', '', file='test.bb', line=1) + info = HobRecipeInfo('test.bb', metadata) + cachedata = type('CacheData', (), {})() + + HobRecipeInfo.init_cacheData(cachedata) + info.add_cacheData(cachedata, 'test.bb') + + self.assertIn('recipe_variables', HobRecipeInfo.cachefields) + variables = json.loads(zlib.decompress( + cachedata.recipe_variables['test.bb'])) + self.assertEqual(variables['EMPTY'], '') + + def test_recipe_variables_are_compressed_before_caching(self): + metadata = self.metadata() + metadata.setVar('LARGE', 'repeated-value-' * 1000, + file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + + self.assertLess(len(info.recipe_variables), len(metadata.getVar('LARGE'))) + + def test_unexpandable_variable_is_omitted(self): + metadata = self.metadata() + metadata.setVar('BROKEN', '${BROKEN}', file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + + variables = json.loads(zlib.decompress(info.recipe_variables)) + self.assertNotIn('BROKEN', variables) + +if __name__ == '__main__': + unittest.main()