[v2,1/5] cache_extra: Collect recipe variable snapshots

diff --git a/bin/bitbake-selftest b/bin/bitbake-selftest
index fb7c57dd8..1b09815cd 100755
--- a/bin/bitbake-selftest
+++ b/bin/bitbake-selftest
@@ -21,6 +21,7 @@  except RuntimeError as exc:
     sys.exit(str(exc))

 tests = ["bb.tests.codeparser",
+         "bb.tests.cache_extra",
          "bb.tests.color",
          "bb.tests.cooker",
          "bb.tests.cow",
diff --git a/lib/bb/cache.py b/lib/bb/cache.py
index 2361c5684..33fa61936 100644
--- a/lib/bb/cache.py
+++ b/lib/bb/cache.py
@@ -28,7 +28,7 @@  import shutil

 logger = logging.getLogger("BitBake.Cache")

-__cache_version__ = "156"
+__cache_version__ = "157"

 def getCacheFile(path, filename, mc, data_hash):
     mcspec = ''
diff --git a/lib/bb/cache_extra.py b/lib/bb/cache_extra.py
index bf4226d16..78ef6b638 100644
--- a/lib/bb/cache_extra.py
+++ b/lib/bb/cache_extra.py
@@ -13,7 +13,65 @@ 
 # SPDX-License-Identifier: GPL-2.0-only
 #

+import logging
+import json
+import zlib
+
 from bb.cache import RecipeInfoCommon
+logger = logging.getLogger("BitBake.CacheExtra")
+
+
+def _winning_override(metadata, variable_name):
+    active = {}
+    metadata.need_overrides()
+    for override_variable, override in metadata.overridedata.get(variable_name, ()):
+        if (override in metadata.overridesset or
+                ':' in override and
+                set(override.split(':')).issubset(metadata.overridesset)):
+            active[override] = override_variable
+
+    match = None
+    modified = True
+    while modified:
+        modified = False
+        for override in metadata.overrides:
+            for candidate in active.copy():
+                if candidate.endswith(':' + override):
+                    active[candidate.removesuffix(':' + override)] = active.pop(candidate)
+                    modified = True
+                elif candidate == override:
+                    match = active.pop(candidate)
+    return match
+
+
+def _recipe_includes(metadata, filename):
+    includes = set()
+
+    def collect(node):
+        for child in node.children:
+            if child.filename.endswith('.bbclass'):
+                continue
+            if child.filename.endswith('.inc'):
+                includes.add(child.filename)
+            collect(child)
+
+    for child in metadata.inchistory.children:
+        if child.filename == filename:
+            collect(child)
+    return includes
+
+
+def _recipe_value_event(metadata, event, recipe_includes):
+    event_file = event.get('file', '')
+    if (not event_file.endswith(('.bb', '.bbappend')) and
+            event_file not in recipe_includes):
+        return False
+    operation = event.get('op', '')
+    if '[' not in operation:
+        return True
+    override = operation.rsplit('[', 1)[1].removesuffix(']')
+    return set(override.split(':')).issubset(metadata.overridesset)
+

 class HobRecipeInfo(RecipeInfoCommon):
     __slots__ = ()
@@ -27,7 +85,7 @@  class HobRecipeInfo(RecipeInfoCommon):
     # that this class will provide
     cachefields = ['summary', 'license', 'section',
             'description', 'homepage', 'bugtracker',
-            'prevision', 'files_info']
+            'prevision', 'files_info', 'recipe_variables']

     def __init__(self, filename, metadata):

@@ -39,6 +97,45 @@  class HobRecipeInfo(RecipeInfoCommon):
         self.bugtracker = self.getvar('BUGTRACKER', metadata)
         self.prevision = self.getvar('PR', metadata)
         self.files_info = self.getvar('FILES_INFO', metadata)
+        recipe_variables = {}
+        recipe_includes = _recipe_includes(metadata, filename)
+        for variable_name in metadata:
+            try:
+                if ':' in variable_name:
+                    continue
+                winning_override = _winning_override(metadata, variable_name)
+                history = metadata.varhistory.variable(variable_name)
+                if winning_override:
+                    history += metadata.varhistory.variable(winning_override)
+                if not any(_recipe_value_event(metadata, event, recipe_includes)
+                           for event in history):
+                    # Keep this snapshot limited to recipe metadata.
+                    continue
+                if (metadata.getVarFlag(variable_name, 'func', False) or
+                        winning_override and
+                        metadata.getVarFlag(winning_override, 'func', False)):
+                    continue
+                value = metadata.getVar(variable_name, True)
+                recipe_variables[variable_name] = '' if value is None else str(value)
+                flag_sources = [variable_name]
+                if winning_override and winning_override != variable_name:
+                    flag_sources.append(winning_override)
+                for flag_source in flag_sources:
+                    for flag in metadata.getVarFlags(flag_source) or {}:
+                        if flag == 'func':
+                            continue
+                        try:
+                            flag_value = metadata.getVarFlag(flag_source, flag, True)
+                            if flag_value is not None:
+                                recipe_variables['%s[%s]' % (variable_name, flag)] = str(flag_value)
+                        except Exception as exc:
+                            logger.debug("Omitting recipe variable flag %s[%s] from %s after %s",
+                                         variable_name, flag, filename, type(exc).__name__)
+            except Exception as exc:
+                logger.debug("Omitting recipe variable %s from %s after %s",
+                             variable_name, filename, type(exc).__name__)
+        self.recipe_variables = zlib.compress(
+            json.dumps(recipe_variables, separators=(',', ':')).encode('utf-8'))

     @classmethod
     def init_cacheData(cls, cachedata):
@@ -51,6 +148,7 @@  class HobRecipeInfo(RecipeInfoCommon):
         cachedata.bugtracker = {}
         cachedata.prevision = {}
         cachedata.files_info = {}
+        cachedata.recipe_variables = {}

     def add_cacheData(self, cachedata, fn):
         cachedata.summary[fn] = self.summary
@@ -61,3 +159,4 @@  class HobRecipeInfo(RecipeInfoCommon):
         cachedata.bugtracker[fn] = self.bugtracker
         cachedata.prevision[fn] = self.prevision
         cachedata.files_info[fn] = self.files_info
+        cachedata.recipe_variables[fn] = self.recipe_variables
diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py
index 59f808c96..e68829813 100644
--- a/lib/bb/cookerdata.py
+++ b/lib/bb/cookerdata.py
@@ -507,8 +507,9 @@  class CookerDataBuilder(object):

         return data

-    @staticmethod
-    def _parse_recipe(bb_data, bbfile, appends, mc, layername):
+    def _parse_recipe(self, bb_data, bbfile, appends, mc, layername):
+        if self.tracking:
+            bb_data.enableTracking()
         bb_data.setVar("__BBMULTICONFIG", mc)
         bb_data.setVar("FILE_LAYERNAME", layername)

diff --git a/lib/bb/tests/cache_extra.py b/lib/bb/tests/cache_extra.py
new file mode 100644
index 000000000..c695bb185
--- /dev/null
+++ b/lib/bb/tests/cache_extra.py
@@ -0,0 +1,235 @@ 
+#
+# BitBake Tests for extra cache data
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import unittest
+import json
+import os
+import tempfile
+import zlib
+
+import bb.data
+import bb.parse
+import bb.siggen
+from bb.cookerdata import CookerDataBuilder
+from bb.cache_extra import HobRecipeInfo
+
+
+class HobRecipeInfoTest(unittest.TestCase):
+
+    @staticmethod
+    def metadata():
+        metadata = bb.data.init()
+        metadata.enableTracking()
+        return metadata
+
+    def test_recipe_variables(self):
+        metadata = self.metadata()
+        metadata.setVar('TEXT', '${VALUE}', file='test.bb', line=1)
+        metadata.setVar('VALUE', 'expanded', file='test.bb', line=2)
+        metadata.setVar('EMPTY', '', file='test.bb', line=3)
+        metadata.setVar('NUMBER', 7, file='test.bbappend', line=1)
+        metadata.setVar('SPECIAL', 'café\nline\x00end',
+                        file='test.bb', line=4)
+        metadata.setVar('INHERITED', 'global', file='test.inc', line=1)
+        metadata.setVar('__INTERNAL', 'internal', file='test.bbclass', line=1)
+        metadata.setVar('FROM_CONFIG', 'config', file='conf/local.conf', line=1)
+        metadata.setVar('do_function', 'echo test', file='test.bb', line=5)
+        metadata.setVarFlag('do_function', 'func', True)
+
+        info = HobRecipeInfo('test.bb', metadata)
+        variables = json.loads(zlib.decompress(info.recipe_variables))
+
+        self.assertEqual(variables['TEXT'], 'expanded')
+        self.assertEqual(variables['EMPTY'], '')
+        self.assertEqual(variables['NUMBER'], '7')
+        self.assertEqual(variables['SPECIAL'], 'café\nline\x00end')
+        self.assertNotIn('INHERITED', variables)
+        self.assertNotIn('__INTERNAL', variables)
+        self.assertNotIn('FROM_CONFIG', variables)
+        self.assertNotIn('do_function', variables)
+
+    def test_parsed_recipe_variables_include_recipe_inc_but_not_class(self):
+        metadata = self.metadata()
+        metadata.setVar('__bbclasstype', 'recipe')
+        bb.parse.siggen = bb.siggen.init(metadata)
+        metadata.disableTracking()
+        builder = object.__new__(CookerDataBuilder)
+        builder.tracking = True
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            recipe = os.path.join(tempdir, 'test.bb')
+            include = os.path.join(tempdir, 'test.inc')
+            classes = os.path.join(tempdir, 'classes')
+            os.mkdir(classes)
+            metadata.setVar('BBPATH', tempdir)
+            with open(include, 'w') as handle:
+                handle.write('FROM_INC = "inc"\n')
+            with open(os.path.join(tempdir, 'class.inc'), 'w') as handle:
+                handle.write('FROM_CLASS_INC = "class-inc"\n')
+            with open(os.path.join(classes, 'testclass.bbclass'), 'w') as handle:
+                handle.write('require class.inc\nFROM_CLASS = "class"\n')
+            with open(recipe, 'w') as handle:
+                handle.write('require test.inc\ninherit testclass\nFROM_RECIPE = "recipe"\n')
+
+            parsed = builder._parse_recipe(metadata, recipe, [], '', '')['']
+            info = HobRecipeInfo(recipe, parsed)
+
+        variables = json.loads(zlib.decompress(info.recipe_variables))
+        self.assertEqual(variables['FROM_RECIPE'], 'recipe')
+        self.assertEqual(variables['FROM_INC'], 'inc')
+        self.assertNotIn('FROM_CLASS', variables)
+        self.assertNotIn('FROM_CLASS_INC', variables)
+
+    def test_recipe_variable_operations_are_included(self):
+        metadata = self.metadata()
+        metadata.setVar('TEXT', 'global', file='conf/bitbake.conf', line=1)
+        metadata.setVar('TEXT:append', ' recipe', file='test.bb', line=1)
+        metadata.setVar('TEXT:remove', 'global', file='test.bbappend', line=1)
+
+        info = HobRecipeInfo('test.bb', metadata)
+        variables = json.loads(zlib.decompress(info.recipe_variables))
+
+        self.assertEqual(variables['TEXT'], ' recipe')
+
+    def test_only_effective_overrides_are_included(self):
+        metadata = self.metadata()
+        metadata.setVar('OVERRIDES', 'machine', file='conf/bitbake.conf', line=1)
+        metadata.setVar('ACTIVE:machine', 'recipe', file='test.bb', line=1)
+        metadata.setVar('INACTIVE:other', 'recipe', file='test.bb', line=2)
+        metadata.setVar('GLOBAL', 'global', file='conf/bitbake.conf', line=2)
+        metadata.setVar('GLOBAL:append:other', ' recipe', file='test.bb', line=3)
+
+        info = HobRecipeInfo('test.bb', metadata)
+        variables = json.loads(zlib.decompress(info.recipe_variables))
+
+        self.assertEqual(variables['ACTIVE'], 'recipe')
+        self.assertNotIn('ACTIVE:machine', variables)
+        self.assertNotIn('INACTIVE', variables)
+        self.assertNotIn('INACTIVE:other', variables)
+        self.assertNotIn('GLOBAL', variables)
+
+    def test_weak_default_is_included(self):
+        metadata = self.metadata()
+        metadata.setVarFlag('PACKAGECONFIG', '_defaultval', 'feature',
+                            file='test.bb', line=1)
+
+        info = HobRecipeInfo('test.bb', metadata)
+        variables = json.loads(zlib.decompress(info.recipe_variables))
+
+        self.assertEqual(variables['PACKAGECONFIG'], 'feature')
+
+    def test_variable_flags_are_included(self):
+        metadata = self.metadata()
+        metadata.setVar('PACKAGECONFIG', 'feature', file='test.bb', line=1)
+        metadata.setVarFlag('PACKAGECONFIG', 'feature', '--enable-feature',
+                            file='test.inc', line=1)
+
+        info = HobRecipeInfo('test.bb', metadata)
+        variables = json.loads(zlib.decompress(info.recipe_variables))
+
+        self.assertEqual(variables['PACKAGECONFIG[feature]'], '--enable-feature')
+
+    def test_active_override_flags_are_included(self):
+        metadata = self.metadata()
+        metadata.setVar('OVERRIDES', 'machine', file='test.inc', line=1)
+        metadata.setVar('PACKAGECONFIG:machine', 'feature',
+                        file='test.bbappend', line=2)
+        metadata.setVarFlag('PACKAGECONFIG:machine', 'feature',
+                            '--enable-feature', file='test.bbappend', line=3)
+
+        info = HobRecipeInfo('test.bb', metadata)
+        variables = json.loads(zlib.decompress(info.recipe_variables))
+
+        self.assertEqual(variables['PACKAGECONFIG'], 'feature')
+        self.assertEqual(variables['PACKAGECONFIG[feature]'], '--enable-feature')
+
+    def test_active_override_flag_wins_over_base_flag(self):
+        metadata = self.metadata()
+        metadata.setVar('OVERRIDES', 'machine', file='conf/local.conf', line=1)
+        metadata.setVar('PACKAGECONFIG:machine', 'feature',
+                        file='test.bb', line=2)
+        metadata.setVarFlag('PACKAGECONFIG', 'feature', '--base',
+                            file='test.inc', line=3)
+        metadata.setVarFlag('PACKAGECONFIG:machine', 'feature', '--machine',
+                            file='test.bbappend', line=4)
+
+        info = HobRecipeInfo('test.bb', metadata)
+        variables = json.loads(zlib.decompress(info.recipe_variables))
+
+        self.assertEqual(variables['PACKAGECONFIG[feature]'], '--machine')
+
+    def test_inherited_only_values_are_excluded(self):
+        metadata = self.metadata()
+        metadata.setVar('FROM_INC', 'inc', file='test.inc', line=1)
+        metadata.setVar('FROM_CLASS', 'class', file='test.bbclass', line=2)
+        metadata.setVar('FROM_CONFIG', 'config', file='conf/local.conf', line=3)
+
+        info = HobRecipeInfo('test.bb', metadata)
+        variables = json.loads(zlib.decompress(info.recipe_variables))
+
+        self.assertNotIn('FROM_INC', variables)
+        self.assertNotIn('FROM_CLASS', variables)
+        self.assertNotIn('FROM_CONFIG', variables)
+
+    def test_combined_override_is_included_under_logical_name(self):
+        metadata = self.metadata()
+        metadata.setVar('OVERRIDES', 'foo:bar:local',
+                        file='conf/bitbake.conf', line=1)
+        metadata.setVar('COMBINED:local:foo:bar', 'recipe',
+                        file='test.bb', line=1)
+
+        info = HobRecipeInfo('test.bb', metadata)
+        variables = json.loads(zlib.decompress(info.recipe_variables))
+
+        self.assertEqual(variables['COMBINED'], 'recipe')
+        self.assertNotIn('OVERRIDES', variables)
+
+    def test_active_override_function_is_omitted(self):
+        metadata = self.metadata()
+        metadata.setVar('OVERRIDES', 'machine', file='conf/bitbake.conf', line=1)
+        metadata.setVar('do_work:machine', 'echo test', file='test.bb', line=1)
+        metadata.setVarFlag('do_work:machine', 'func', True,
+                            file='test.bb', line=1)
+
+        info = HobRecipeInfo('test.bb', metadata)
+        variables = json.loads(zlib.decompress(info.recipe_variables))
+
+        self.assertNotIn('do_work', variables)
+
+    def test_recipe_variables_survive_extra_cache_mapping(self):
+        metadata = self.metadata()
+        metadata.setVar('EMPTY', '', file='test.bb', line=1)
+        info = HobRecipeInfo('test.bb', metadata)
+        cachedata = type('CacheData', (), {})()
+
+        HobRecipeInfo.init_cacheData(cachedata)
+        info.add_cacheData(cachedata, 'test.bb')
+
+        self.assertIn('recipe_variables', HobRecipeInfo.cachefields)
+        variables = json.loads(zlib.decompress(
+            cachedata.recipe_variables['test.bb']))
+        self.assertEqual(variables['EMPTY'], '')
+
+    def test_recipe_variables_are_compressed_before_caching(self):
+        metadata = self.metadata()
+        metadata.setVar('LARGE', 'repeated-value-' * 1000,
+                        file='test.bb', line=1)
+
+        info = HobRecipeInfo('test.bb', metadata)
+
+        self.assertLess(len(info.recipe_variables), len(metadata.getVar('LARGE')))
+
+    def test_unexpandable_variable_is_omitted(self):
+        metadata = self.metadata()
+        metadata.setVar('BROKEN', '${BROKEN}', file='test.bb', line=1)
+
+        info = HobRecipeInfo('test.bb', metadata)
+
+        variables = json.loads(zlib.decompress(info.recipe_variables))
+        self.assertNotIn('BROKEN', variables)
+
+if __name__ == '__main__':
+    unittest.main()