From patchwork Fri Aug 14 15:35:54 2026 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Paolo Wattebled X-Patchwork-Id: 95280 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A965AC5CFC1 for ; Fri, 14 Aug 2026 15:36:23 +0000 (UTC) Received: from mail.savoirfairelinux.com (mail.savoirfairelinux.com [208.88.110.44]) by mx.groups.io with SMTP id smtpd.msgproc02-g2.15785.1786721773449096641 for ; Fri, 14 Aug 2026 08:36:13 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@savoirfairelinux.com header.s=DFC430D2-D198-11EC-948E-34200CB392D2 header.b=lJwik8tl; spf=pass (domain: savoirfairelinux.com, ip: 208.88.110.44, mailfrom: paolo.wattebled@savoirfairelinux.com) Received: from localhost (localhost [127.0.0.1]) by mail.savoirfairelinux.com (Postfix) with ESMTP id 458783D8185E for ; Fri, 14 Aug 2026 11:36:12 -0400 (EDT) Received: from mail.savoirfairelinux.com ([127.0.0.1]) by localhost (mail.savoirfairelinux.com [127.0.0.1]) (amavis, port 10032) with ESMTP id 1saMAWK8khzD; Fri, 14 Aug 2026 11:36:08 -0400 (EDT) Received: from localhost (localhost [127.0.0.1]) by mail.savoirfairelinux.com (Postfix) with ESMTP id 04CC43D818E9; Fri, 14 Aug 2026 11:36:08 -0400 (EDT) DKIM-Filter: OpenDKIM Filter v2.10.3 mail.savoirfairelinux.com 04CC43D818E9 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=savoirfairelinux.com; s=DFC430D2-D198-11EC-948E-34200CB392D2; t=1786721768; bh=fJVLWX8G1x8kU0shDbYZJQHJi2eKHblfJIgxyftUwWc=; h=From:To:Date:Message-ID:MIME-Version; b=lJwik8tlu2G+6xk1fAPYdNq5V8vt3RujDLB4XtzekkqJo3+qNfKHBYXSnFzrbvo7Q Tb6LjhxuKG9OACMrZNLIOdJbF9Mtf9avzacsH/fprc75ff/hJb4jlC+mzrODtYp2Bk q02HHRELR2duOkS3107AsgsQpnCcvSKw/pXeCjQubJorQ1VAT0//4MhgheqKKxsCPe 4nI23MDOUzOq5mz2hBpTEuCUu2Y1c8siHiAoWyLwpoQyyixv1y2a3uRSNujpKR9YTn PJearvhfDnfexkd9GZF+N7jjv9QHyj0Ve50FvU8/hAZzmxWPQ/DnyieP0Ht7CQUEkj HxUCDE3dnDtqg== X-Virus-Scanned: amavis at mail.savoirfairelinux.com Received: from mail.savoirfairelinux.com ([127.0.0.1]) by localhost (mail.savoirfairelinux.com [127.0.0.1]) (amavis, port 10026) with ESMTP id tU3ZQll3BR9I; Fri, 14 Aug 2026 11:36:07 -0400 (EDT) Received: from pwattebled.mtl.sfl (unknown [192.168.51.254]) by mail.savoirfairelinux.com (Postfix) with ESMTPSA id CED623D818A9; Fri, 14 Aug 2026 11:36:07 -0400 (EDT) From: Paolo Wattebled To: bitbake-devel@lists.openembedded.org Cc: Paolo Wattebled Subject: [PATCH v2 1/5] cache_extra: Collect recipe variable snapshots Date: Fri, 14 Aug 2026 11:35:54 -0400 Message-ID: <20260814153602.2107764-2-paolo.wattebled@savoirfairelinux.com> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260814153602.2107764-1-paolo.wattebled@savoirfairelinux.com> References: <20260814153602.2107764-1-paolo.wattebled@savoirfairelinux.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from 45-33-107-173.ip.linodeusercontent.com [45.33.107.173] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 14 Aug 2026 15:36:23 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/19944 Collect final values defined or modified by recipes, matching bbappends and recipe include files. Preserve active override and variable flag values while excluding inherited-only metadata and functions. Enable variable tracking while recipe metadata is parsed, add focused extraction tests and invalidate stale recipe information caches. AI-Generated: Uses GitHub Copilot and OpenCode with GPT-5.6 Sol Signed-off-by: Paolo Wattebled --- bin/bitbake-selftest | 1 + lib/bb/cache.py | 2 +- lib/bb/cache_extra.py | 101 +++++++++++++++- lib/bb/cookerdata.py | 5 +- lib/bb/tests/cache_extra.py | 235 ++++++++++++++++++++++++++++++++++++ 5 files changed, 340 insertions(+), 4 deletions(-) create mode 100644 lib/bb/tests/cache_extra.py -- 2.55.0 diff --git a/bin/bitbake-selftest b/bin/bitbake-selftest index fb7c57dd8..1b09815cd 100755 --- a/bin/bitbake-selftest +++ b/bin/bitbake-selftest @@ -21,6 +21,7 @@ except RuntimeError as exc: sys.exit(str(exc)) tests = ["bb.tests.codeparser", + "bb.tests.cache_extra", "bb.tests.color", "bb.tests.cooker", "bb.tests.cow", diff --git a/lib/bb/cache.py b/lib/bb/cache.py index 2361c5684..33fa61936 100644 --- a/lib/bb/cache.py +++ b/lib/bb/cache.py @@ -28,7 +28,7 @@ import shutil logger = logging.getLogger("BitBake.Cache") -__cache_version__ = "156" +__cache_version__ = "157" def getCacheFile(path, filename, mc, data_hash): mcspec = '' diff --git a/lib/bb/cache_extra.py b/lib/bb/cache_extra.py index bf4226d16..78ef6b638 100644 --- a/lib/bb/cache_extra.py +++ b/lib/bb/cache_extra.py @@ -13,7 +13,65 @@ # SPDX-License-Identifier: GPL-2.0-only # +import logging +import json +import zlib + from bb.cache import RecipeInfoCommon +logger = logging.getLogger("BitBake.CacheExtra") + + +def _winning_override(metadata, variable_name): + active = {} + metadata.need_overrides() + for override_variable, override in metadata.overridedata.get(variable_name, ()): + if (override in metadata.overridesset or + ':' in override and + set(override.split(':')).issubset(metadata.overridesset)): + active[override] = override_variable + + match = None + modified = True + while modified: + modified = False + for override in metadata.overrides: + for candidate in active.copy(): + if candidate.endswith(':' + override): + active[candidate.removesuffix(':' + override)] = active.pop(candidate) + modified = True + elif candidate == override: + match = active.pop(candidate) + return match + + +def _recipe_includes(metadata, filename): + includes = set() + + def collect(node): + for child in node.children: + if child.filename.endswith('.bbclass'): + continue + if child.filename.endswith('.inc'): + includes.add(child.filename) + collect(child) + + for child in metadata.inchistory.children: + if child.filename == filename: + collect(child) + return includes + + +def _recipe_value_event(metadata, event, recipe_includes): + event_file = event.get('file', '') + if (not event_file.endswith(('.bb', '.bbappend')) and + event_file not in recipe_includes): + return False + operation = event.get('op', '') + if '[' not in operation: + return True + override = operation.rsplit('[', 1)[1].removesuffix(']') + return set(override.split(':')).issubset(metadata.overridesset) + class HobRecipeInfo(RecipeInfoCommon): __slots__ = () @@ -27,7 +85,7 @@ class HobRecipeInfo(RecipeInfoCommon): # that this class will provide cachefields = ['summary', 'license', 'section', 'description', 'homepage', 'bugtracker', - 'prevision', 'files_info'] + 'prevision', 'files_info', 'recipe_variables'] def __init__(self, filename, metadata): @@ -39,6 +97,45 @@ class HobRecipeInfo(RecipeInfoCommon): self.bugtracker = self.getvar('BUGTRACKER', metadata) self.prevision = self.getvar('PR', metadata) self.files_info = self.getvar('FILES_INFO', metadata) + recipe_variables = {} + recipe_includes = _recipe_includes(metadata, filename) + for variable_name in metadata: + try: + if ':' in variable_name: + continue + winning_override = _winning_override(metadata, variable_name) + history = metadata.varhistory.variable(variable_name) + if winning_override: + history += metadata.varhistory.variable(winning_override) + if not any(_recipe_value_event(metadata, event, recipe_includes) + for event in history): + # Keep this snapshot limited to recipe metadata. + continue + if (metadata.getVarFlag(variable_name, 'func', False) or + winning_override and + metadata.getVarFlag(winning_override, 'func', False)): + continue + value = metadata.getVar(variable_name, True) + recipe_variables[variable_name] = '' if value is None else str(value) + flag_sources = [variable_name] + if winning_override and winning_override != variable_name: + flag_sources.append(winning_override) + for flag_source in flag_sources: + for flag in metadata.getVarFlags(flag_source) or {}: + if flag == 'func': + continue + try: + flag_value = metadata.getVarFlag(flag_source, flag, True) + if flag_value is not None: + recipe_variables['%s[%s]' % (variable_name, flag)] = str(flag_value) + except Exception as exc: + logger.debug("Omitting recipe variable flag %s[%s] from %s after %s", + variable_name, flag, filename, type(exc).__name__) + except Exception as exc: + logger.debug("Omitting recipe variable %s from %s after %s", + variable_name, filename, type(exc).__name__) + self.recipe_variables = zlib.compress( + json.dumps(recipe_variables, separators=(',', ':')).encode('utf-8')) @classmethod def init_cacheData(cls, cachedata): @@ -51,6 +148,7 @@ class HobRecipeInfo(RecipeInfoCommon): cachedata.bugtracker = {} cachedata.prevision = {} cachedata.files_info = {} + cachedata.recipe_variables = {} def add_cacheData(self, cachedata, fn): cachedata.summary[fn] = self.summary @@ -61,3 +159,4 @@ class HobRecipeInfo(RecipeInfoCommon): cachedata.bugtracker[fn] = self.bugtracker cachedata.prevision[fn] = self.prevision cachedata.files_info[fn] = self.files_info + cachedata.recipe_variables[fn] = self.recipe_variables diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py index 59f808c96..e68829813 100644 --- a/lib/bb/cookerdata.py +++ b/lib/bb/cookerdata.py @@ -507,8 +507,9 @@ class CookerDataBuilder(object): return data - @staticmethod - def _parse_recipe(bb_data, bbfile, appends, mc, layername): + def _parse_recipe(self, bb_data, bbfile, appends, mc, layername): + if self.tracking: + bb_data.enableTracking() bb_data.setVar("__BBMULTICONFIG", mc) bb_data.setVar("FILE_LAYERNAME", layername) diff --git a/lib/bb/tests/cache_extra.py b/lib/bb/tests/cache_extra.py new file mode 100644 index 000000000..c695bb185 --- /dev/null +++ b/lib/bb/tests/cache_extra.py @@ -0,0 +1,235 @@ +# +# BitBake Tests for extra cache data +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import unittest +import json +import os +import tempfile +import zlib + +import bb.data +import bb.parse +import bb.siggen +from bb.cookerdata import CookerDataBuilder +from bb.cache_extra import HobRecipeInfo + + +class HobRecipeInfoTest(unittest.TestCase): + + @staticmethod + def metadata(): + metadata = bb.data.init() + metadata.enableTracking() + return metadata + + def test_recipe_variables(self): + metadata = self.metadata() + metadata.setVar('TEXT', '${VALUE}', file='test.bb', line=1) + metadata.setVar('VALUE', 'expanded', file='test.bb', line=2) + metadata.setVar('EMPTY', '', file='test.bb', line=3) + metadata.setVar('NUMBER', 7, file='test.bbappend', line=1) + metadata.setVar('SPECIAL', 'café\nline\x00end', + file='test.bb', line=4) + metadata.setVar('INHERITED', 'global', file='test.inc', line=1) + metadata.setVar('__INTERNAL', 'internal', file='test.bbclass', line=1) + metadata.setVar('FROM_CONFIG', 'config', file='conf/local.conf', line=1) + metadata.setVar('do_function', 'echo test', file='test.bb', line=5) + metadata.setVarFlag('do_function', 'func', True) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['TEXT'], 'expanded') + self.assertEqual(variables['EMPTY'], '') + self.assertEqual(variables['NUMBER'], '7') + self.assertEqual(variables['SPECIAL'], 'café\nline\x00end') + self.assertNotIn('INHERITED', variables) + self.assertNotIn('__INTERNAL', variables) + self.assertNotIn('FROM_CONFIG', variables) + self.assertNotIn('do_function', variables) + + def test_parsed_recipe_variables_include_recipe_inc_but_not_class(self): + metadata = self.metadata() + metadata.setVar('__bbclasstype', 'recipe') + bb.parse.siggen = bb.siggen.init(metadata) + metadata.disableTracking() + builder = object.__new__(CookerDataBuilder) + builder.tracking = True + + with tempfile.TemporaryDirectory() as tempdir: + recipe = os.path.join(tempdir, 'test.bb') + include = os.path.join(tempdir, 'test.inc') + classes = os.path.join(tempdir, 'classes') + os.mkdir(classes) + metadata.setVar('BBPATH', tempdir) + with open(include, 'w') as handle: + handle.write('FROM_INC = "inc"\n') + with open(os.path.join(tempdir, 'class.inc'), 'w') as handle: + handle.write('FROM_CLASS_INC = "class-inc"\n') + with open(os.path.join(classes, 'testclass.bbclass'), 'w') as handle: + handle.write('require class.inc\nFROM_CLASS = "class"\n') + with open(recipe, 'w') as handle: + handle.write('require test.inc\ninherit testclass\nFROM_RECIPE = "recipe"\n') + + parsed = builder._parse_recipe(metadata, recipe, [], '', '')[''] + info = HobRecipeInfo(recipe, parsed) + + variables = json.loads(zlib.decompress(info.recipe_variables)) + self.assertEqual(variables['FROM_RECIPE'], 'recipe') + self.assertEqual(variables['FROM_INC'], 'inc') + self.assertNotIn('FROM_CLASS', variables) + self.assertNotIn('FROM_CLASS_INC', variables) + + def test_recipe_variable_operations_are_included(self): + metadata = self.metadata() + metadata.setVar('TEXT', 'global', file='conf/bitbake.conf', line=1) + metadata.setVar('TEXT:append', ' recipe', file='test.bb', line=1) + metadata.setVar('TEXT:remove', 'global', file='test.bbappend', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['TEXT'], ' recipe') + + def test_only_effective_overrides_are_included(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'machine', file='conf/bitbake.conf', line=1) + metadata.setVar('ACTIVE:machine', 'recipe', file='test.bb', line=1) + metadata.setVar('INACTIVE:other', 'recipe', file='test.bb', line=2) + metadata.setVar('GLOBAL', 'global', file='conf/bitbake.conf', line=2) + metadata.setVar('GLOBAL:append:other', ' recipe', file='test.bb', line=3) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['ACTIVE'], 'recipe') + self.assertNotIn('ACTIVE:machine', variables) + self.assertNotIn('INACTIVE', variables) + self.assertNotIn('INACTIVE:other', variables) + self.assertNotIn('GLOBAL', variables) + + def test_weak_default_is_included(self): + metadata = self.metadata() + metadata.setVarFlag('PACKAGECONFIG', '_defaultval', 'feature', + file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['PACKAGECONFIG'], 'feature') + + def test_variable_flags_are_included(self): + metadata = self.metadata() + metadata.setVar('PACKAGECONFIG', 'feature', file='test.bb', line=1) + metadata.setVarFlag('PACKAGECONFIG', 'feature', '--enable-feature', + file='test.inc', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['PACKAGECONFIG[feature]'], '--enable-feature') + + def test_active_override_flags_are_included(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'machine', file='test.inc', line=1) + metadata.setVar('PACKAGECONFIG:machine', 'feature', + file='test.bbappend', line=2) + metadata.setVarFlag('PACKAGECONFIG:machine', 'feature', + '--enable-feature', file='test.bbappend', line=3) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['PACKAGECONFIG'], 'feature') + self.assertEqual(variables['PACKAGECONFIG[feature]'], '--enable-feature') + + def test_active_override_flag_wins_over_base_flag(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'machine', file='conf/local.conf', line=1) + metadata.setVar('PACKAGECONFIG:machine', 'feature', + file='test.bb', line=2) + metadata.setVarFlag('PACKAGECONFIG', 'feature', '--base', + file='test.inc', line=3) + metadata.setVarFlag('PACKAGECONFIG:machine', 'feature', '--machine', + file='test.bbappend', line=4) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['PACKAGECONFIG[feature]'], '--machine') + + def test_inherited_only_values_are_excluded(self): + metadata = self.metadata() + metadata.setVar('FROM_INC', 'inc', file='test.inc', line=1) + metadata.setVar('FROM_CLASS', 'class', file='test.bbclass', line=2) + metadata.setVar('FROM_CONFIG', 'config', file='conf/local.conf', line=3) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertNotIn('FROM_INC', variables) + self.assertNotIn('FROM_CLASS', variables) + self.assertNotIn('FROM_CONFIG', variables) + + def test_combined_override_is_included_under_logical_name(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'foo:bar:local', + file='conf/bitbake.conf', line=1) + metadata.setVar('COMBINED:local:foo:bar', 'recipe', + file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['COMBINED'], 'recipe') + self.assertNotIn('OVERRIDES', variables) + + def test_active_override_function_is_omitted(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'machine', file='conf/bitbake.conf', line=1) + metadata.setVar('do_work:machine', 'echo test', file='test.bb', line=1) + metadata.setVarFlag('do_work:machine', 'func', True, + file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertNotIn('do_work', variables) + + def test_recipe_variables_survive_extra_cache_mapping(self): + metadata = self.metadata() + metadata.setVar('EMPTY', '', file='test.bb', line=1) + info = HobRecipeInfo('test.bb', metadata) + cachedata = type('CacheData', (), {})() + + HobRecipeInfo.init_cacheData(cachedata) + info.add_cacheData(cachedata, 'test.bb') + + self.assertIn('recipe_variables', HobRecipeInfo.cachefields) + variables = json.loads(zlib.decompress( + cachedata.recipe_variables['test.bb'])) + self.assertEqual(variables['EMPTY'], '') + + def test_recipe_variables_are_compressed_before_caching(self): + metadata = self.metadata() + metadata.setVar('LARGE', 'repeated-value-' * 1000, + file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + + self.assertLess(len(info.recipe_variables), len(metadata.getVar('LARGE'))) + + def test_unexpandable_variable_is_omitted(self): + metadata = self.metadata() + metadata.setVar('BROKEN', '${BROKEN}', file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + + variables = json.loads(zlib.decompress(info.recipe_variables)) + self.assertNotIn('BROKEN', variables) + +if __name__ == '__main__': + unittest.main()