From patchwork Fri Aug 14 15:35:54 2026 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Paolo Wattebled X-Patchwork-Id: 95280 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A965AC5CFC1 for ; Fri, 14 Aug 2026 15:36:23 +0000 (UTC) Received: from mail.savoirfairelinux.com (mail.savoirfairelinux.com [208.88.110.44]) by mx.groups.io with SMTP id smtpd.msgproc02-g2.15785.1786721773449096641 for ; Fri, 14 Aug 2026 08:36:13 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@savoirfairelinux.com header.s=DFC430D2-D198-11EC-948E-34200CB392D2 header.b=lJwik8tl; spf=pass (domain: savoirfairelinux.com, ip: 208.88.110.44, mailfrom: paolo.wattebled@savoirfairelinux.com) Received: from localhost (localhost [127.0.0.1]) by mail.savoirfairelinux.com (Postfix) with ESMTP id 458783D8185E for ; Fri, 14 Aug 2026 11:36:12 -0400 (EDT) Received: from mail.savoirfairelinux.com ([127.0.0.1]) by localhost (mail.savoirfairelinux.com [127.0.0.1]) (amavis, port 10032) with ESMTP id 1saMAWK8khzD; Fri, 14 Aug 2026 11:36:08 -0400 (EDT) Received: from localhost (localhost [127.0.0.1]) by mail.savoirfairelinux.com (Postfix) with ESMTP id 04CC43D818E9; Fri, 14 Aug 2026 11:36:08 -0400 (EDT) DKIM-Filter: OpenDKIM Filter v2.10.3 mail.savoirfairelinux.com 04CC43D818E9 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=savoirfairelinux.com; s=DFC430D2-D198-11EC-948E-34200CB392D2; t=1786721768; bh=fJVLWX8G1x8kU0shDbYZJQHJi2eKHblfJIgxyftUwWc=; h=From:To:Date:Message-ID:MIME-Version; b=lJwik8tlu2G+6xk1fAPYdNq5V8vt3RujDLB4XtzekkqJo3+qNfKHBYXSnFzrbvo7Q Tb6LjhxuKG9OACMrZNLIOdJbF9Mtf9avzacsH/fprc75ff/hJb4jlC+mzrODtYp2Bk q02HHRELR2duOkS3107AsgsQpnCcvSKw/pXeCjQubJorQ1VAT0//4MhgheqKKxsCPe 4nI23MDOUzOq5mz2hBpTEuCUu2Y1c8siHiAoWyLwpoQyyixv1y2a3uRSNujpKR9YTn PJearvhfDnfexkd9GZF+N7jjv9QHyj0Ve50FvU8/hAZzmxWPQ/DnyieP0Ht7CQUEkj HxUCDE3dnDtqg== X-Virus-Scanned: amavis at mail.savoirfairelinux.com Received: from mail.savoirfairelinux.com ([127.0.0.1]) by localhost (mail.savoirfairelinux.com [127.0.0.1]) (amavis, port 10026) with ESMTP id tU3ZQll3BR9I; Fri, 14 Aug 2026 11:36:07 -0400 (EDT) Received: from pwattebled.mtl.sfl (unknown [192.168.51.254]) by mail.savoirfairelinux.com (Postfix) with ESMTPSA id CED623D818A9; Fri, 14 Aug 2026 11:36:07 -0400 (EDT) From: Paolo Wattebled To: bitbake-devel@lists.openembedded.org Cc: Paolo Wattebled Subject: [PATCH v2 1/5] cache_extra: Collect recipe variable snapshots Date: Fri, 14 Aug 2026 11:35:54 -0400 Message-ID: <20260814153602.2107764-2-paolo.wattebled@savoirfairelinux.com> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260814153602.2107764-1-paolo.wattebled@savoirfairelinux.com> References: <20260814153602.2107764-1-paolo.wattebled@savoirfairelinux.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from 45-33-107-173.ip.linodeusercontent.com [45.33.107.173] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 14 Aug 2026 15:36:23 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/19944 Collect final values defined or modified by recipes, matching bbappends and recipe include files. Preserve active override and variable flag values while excluding inherited-only metadata and functions. Enable variable tracking while recipe metadata is parsed, add focused extraction tests and invalidate stale recipe information caches. AI-Generated: Uses GitHub Copilot and OpenCode with GPT-5.6 Sol Signed-off-by: Paolo Wattebled --- bin/bitbake-selftest | 1 + lib/bb/cache.py | 2 +- lib/bb/cache_extra.py | 101 +++++++++++++++- lib/bb/cookerdata.py | 5 +- lib/bb/tests/cache_extra.py | 235 ++++++++++++++++++++++++++++++++++++ 5 files changed, 340 insertions(+), 4 deletions(-) create mode 100644 lib/bb/tests/cache_extra.py -- 2.55.0 diff --git a/bin/bitbake-selftest b/bin/bitbake-selftest index fb7c57dd8..1b09815cd 100755 --- a/bin/bitbake-selftest +++ b/bin/bitbake-selftest @@ -21,6 +21,7 @@ except RuntimeError as exc: sys.exit(str(exc)) tests = ["bb.tests.codeparser", + "bb.tests.cache_extra", "bb.tests.color", "bb.tests.cooker", "bb.tests.cow", diff --git a/lib/bb/cache.py b/lib/bb/cache.py index 2361c5684..33fa61936 100644 --- a/lib/bb/cache.py +++ b/lib/bb/cache.py @@ -28,7 +28,7 @@ import shutil logger = logging.getLogger("BitBake.Cache") -__cache_version__ = "156" +__cache_version__ = "157" def getCacheFile(path, filename, mc, data_hash): mcspec = '' diff --git a/lib/bb/cache_extra.py b/lib/bb/cache_extra.py index bf4226d16..78ef6b638 100644 --- a/lib/bb/cache_extra.py +++ b/lib/bb/cache_extra.py @@ -13,7 +13,65 @@ # SPDX-License-Identifier: GPL-2.0-only # +import logging +import json +import zlib + from bb.cache import RecipeInfoCommon +logger = logging.getLogger("BitBake.CacheExtra") + + +def _winning_override(metadata, variable_name): + active = {} + metadata.need_overrides() + for override_variable, override in metadata.overridedata.get(variable_name, ()): + if (override in metadata.overridesset or + ':' in override and + set(override.split(':')).issubset(metadata.overridesset)): + active[override] = override_variable + + match = None + modified = True + while modified: + modified = False + for override in metadata.overrides: + for candidate in active.copy(): + if candidate.endswith(':' + override): + active[candidate.removesuffix(':' + override)] = active.pop(candidate) + modified = True + elif candidate == override: + match = active.pop(candidate) + return match + + +def _recipe_includes(metadata, filename): + includes = set() + + def collect(node): + for child in node.children: + if child.filename.endswith('.bbclass'): + continue + if child.filename.endswith('.inc'): + includes.add(child.filename) + collect(child) + + for child in metadata.inchistory.children: + if child.filename == filename: + collect(child) + return includes + + +def _recipe_value_event(metadata, event, recipe_includes): + event_file = event.get('file', '') + if (not event_file.endswith(('.bb', '.bbappend')) and + event_file not in recipe_includes): + return False + operation = event.get('op', '') + if '[' not in operation: + return True + override = operation.rsplit('[', 1)[1].removesuffix(']') + return set(override.split(':')).issubset(metadata.overridesset) + class HobRecipeInfo(RecipeInfoCommon): __slots__ = () @@ -27,7 +85,7 @@ class HobRecipeInfo(RecipeInfoCommon): # that this class will provide cachefields = ['summary', 'license', 'section', 'description', 'homepage', 'bugtracker', - 'prevision', 'files_info'] + 'prevision', 'files_info', 'recipe_variables'] def __init__(self, filename, metadata): @@ -39,6 +97,45 @@ class HobRecipeInfo(RecipeInfoCommon): self.bugtracker = self.getvar('BUGTRACKER', metadata) self.prevision = self.getvar('PR', metadata) self.files_info = self.getvar('FILES_INFO', metadata) + recipe_variables = {} + recipe_includes = _recipe_includes(metadata, filename) + for variable_name in metadata: + try: + if ':' in variable_name: + continue + winning_override = _winning_override(metadata, variable_name) + history = metadata.varhistory.variable(variable_name) + if winning_override: + history += metadata.varhistory.variable(winning_override) + if not any(_recipe_value_event(metadata, event, recipe_includes) + for event in history): + # Keep this snapshot limited to recipe metadata. + continue + if (metadata.getVarFlag(variable_name, 'func', False) or + winning_override and + metadata.getVarFlag(winning_override, 'func', False)): + continue + value = metadata.getVar(variable_name, True) + recipe_variables[variable_name] = '' if value is None else str(value) + flag_sources = [variable_name] + if winning_override and winning_override != variable_name: + flag_sources.append(winning_override) + for flag_source in flag_sources: + for flag in metadata.getVarFlags(flag_source) or {}: + if flag == 'func': + continue + try: + flag_value = metadata.getVarFlag(flag_source, flag, True) + if flag_value is not None: + recipe_variables['%s[%s]' % (variable_name, flag)] = str(flag_value) + except Exception as exc: + logger.debug("Omitting recipe variable flag %s[%s] from %s after %s", + variable_name, flag, filename, type(exc).__name__) + except Exception as exc: + logger.debug("Omitting recipe variable %s from %s after %s", + variable_name, filename, type(exc).__name__) + self.recipe_variables = zlib.compress( + json.dumps(recipe_variables, separators=(',', ':')).encode('utf-8')) @classmethod def init_cacheData(cls, cachedata): @@ -51,6 +148,7 @@ class HobRecipeInfo(RecipeInfoCommon): cachedata.bugtracker = {} cachedata.prevision = {} cachedata.files_info = {} + cachedata.recipe_variables = {} def add_cacheData(self, cachedata, fn): cachedata.summary[fn] = self.summary @@ -61,3 +159,4 @@ class HobRecipeInfo(RecipeInfoCommon): cachedata.bugtracker[fn] = self.bugtracker cachedata.prevision[fn] = self.prevision cachedata.files_info[fn] = self.files_info + cachedata.recipe_variables[fn] = self.recipe_variables diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py index 59f808c96..e68829813 100644 --- a/lib/bb/cookerdata.py +++ b/lib/bb/cookerdata.py @@ -507,8 +507,9 @@ class CookerDataBuilder(object): return data - @staticmethod - def _parse_recipe(bb_data, bbfile, appends, mc, layername): + def _parse_recipe(self, bb_data, bbfile, appends, mc, layername): + if self.tracking: + bb_data.enableTracking() bb_data.setVar("__BBMULTICONFIG", mc) bb_data.setVar("FILE_LAYERNAME", layername) diff --git a/lib/bb/tests/cache_extra.py b/lib/bb/tests/cache_extra.py new file mode 100644 index 000000000..c695bb185 --- /dev/null +++ b/lib/bb/tests/cache_extra.py @@ -0,0 +1,235 @@ +# +# BitBake Tests for extra cache data +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import unittest +import json +import os +import tempfile +import zlib + +import bb.data +import bb.parse +import bb.siggen +from bb.cookerdata import CookerDataBuilder +from bb.cache_extra import HobRecipeInfo + + +class HobRecipeInfoTest(unittest.TestCase): + + @staticmethod + def metadata(): + metadata = bb.data.init() + metadata.enableTracking() + return metadata + + def test_recipe_variables(self): + metadata = self.metadata() + metadata.setVar('TEXT', '${VALUE}', file='test.bb', line=1) + metadata.setVar('VALUE', 'expanded', file='test.bb', line=2) + metadata.setVar('EMPTY', '', file='test.bb', line=3) + metadata.setVar('NUMBER', 7, file='test.bbappend', line=1) + metadata.setVar('SPECIAL', 'café\nline\x00end', + file='test.bb', line=4) + metadata.setVar('INHERITED', 'global', file='test.inc', line=1) + metadata.setVar('__INTERNAL', 'internal', file='test.bbclass', line=1) + metadata.setVar('FROM_CONFIG', 'config', file='conf/local.conf', line=1) + metadata.setVar('do_function', 'echo test', file='test.bb', line=5) + metadata.setVarFlag('do_function', 'func', True) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['TEXT'], 'expanded') + self.assertEqual(variables['EMPTY'], '') + self.assertEqual(variables['NUMBER'], '7') + self.assertEqual(variables['SPECIAL'], 'café\nline\x00end') + self.assertNotIn('INHERITED', variables) + self.assertNotIn('__INTERNAL', variables) + self.assertNotIn('FROM_CONFIG', variables) + self.assertNotIn('do_function', variables) + + def test_parsed_recipe_variables_include_recipe_inc_but_not_class(self): + metadata = self.metadata() + metadata.setVar('__bbclasstype', 'recipe') + bb.parse.siggen = bb.siggen.init(metadata) + metadata.disableTracking() + builder = object.__new__(CookerDataBuilder) + builder.tracking = True + + with tempfile.TemporaryDirectory() as tempdir: + recipe = os.path.join(tempdir, 'test.bb') + include = os.path.join(tempdir, 'test.inc') + classes = os.path.join(tempdir, 'classes') + os.mkdir(classes) + metadata.setVar('BBPATH', tempdir) + with open(include, 'w') as handle: + handle.write('FROM_INC = "inc"\n') + with open(os.path.join(tempdir, 'class.inc'), 'w') as handle: + handle.write('FROM_CLASS_INC = "class-inc"\n') + with open(os.path.join(classes, 'testclass.bbclass'), 'w') as handle: + handle.write('require class.inc\nFROM_CLASS = "class"\n') + with open(recipe, 'w') as handle: + handle.write('require test.inc\ninherit testclass\nFROM_RECIPE = "recipe"\n') + + parsed = builder._parse_recipe(metadata, recipe, [], '', '')[''] + info = HobRecipeInfo(recipe, parsed) + + variables = json.loads(zlib.decompress(info.recipe_variables)) + self.assertEqual(variables['FROM_RECIPE'], 'recipe') + self.assertEqual(variables['FROM_INC'], 'inc') + self.assertNotIn('FROM_CLASS', variables) + self.assertNotIn('FROM_CLASS_INC', variables) + + def test_recipe_variable_operations_are_included(self): + metadata = self.metadata() + metadata.setVar('TEXT', 'global', file='conf/bitbake.conf', line=1) + metadata.setVar('TEXT:append', ' recipe', file='test.bb', line=1) + metadata.setVar('TEXT:remove', 'global', file='test.bbappend', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['TEXT'], ' recipe') + + def test_only_effective_overrides_are_included(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'machine', file='conf/bitbake.conf', line=1) + metadata.setVar('ACTIVE:machine', 'recipe', file='test.bb', line=1) + metadata.setVar('INACTIVE:other', 'recipe', file='test.bb', line=2) + metadata.setVar('GLOBAL', 'global', file='conf/bitbake.conf', line=2) + metadata.setVar('GLOBAL:append:other', ' recipe', file='test.bb', line=3) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['ACTIVE'], 'recipe') + self.assertNotIn('ACTIVE:machine', variables) + self.assertNotIn('INACTIVE', variables) + self.assertNotIn('INACTIVE:other', variables) + self.assertNotIn('GLOBAL', variables) + + def test_weak_default_is_included(self): + metadata = self.metadata() + metadata.setVarFlag('PACKAGECONFIG', '_defaultval', 'feature', + file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['PACKAGECONFIG'], 'feature') + + def test_variable_flags_are_included(self): + metadata = self.metadata() + metadata.setVar('PACKAGECONFIG', 'feature', file='test.bb', line=1) + metadata.setVarFlag('PACKAGECONFIG', 'feature', '--enable-feature', + file='test.inc', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['PACKAGECONFIG[feature]'], '--enable-feature') + + def test_active_override_flags_are_included(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'machine', file='test.inc', line=1) + metadata.setVar('PACKAGECONFIG:machine', 'feature', + file='test.bbappend', line=2) + metadata.setVarFlag('PACKAGECONFIG:machine', 'feature', + '--enable-feature', file='test.bbappend', line=3) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['PACKAGECONFIG'], 'feature') + self.assertEqual(variables['PACKAGECONFIG[feature]'], '--enable-feature') + + def test_active_override_flag_wins_over_base_flag(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'machine', file='conf/local.conf', line=1) + metadata.setVar('PACKAGECONFIG:machine', 'feature', + file='test.bb', line=2) + metadata.setVarFlag('PACKAGECONFIG', 'feature', '--base', + file='test.inc', line=3) + metadata.setVarFlag('PACKAGECONFIG:machine', 'feature', '--machine', + file='test.bbappend', line=4) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['PACKAGECONFIG[feature]'], '--machine') + + def test_inherited_only_values_are_excluded(self): + metadata = self.metadata() + metadata.setVar('FROM_INC', 'inc', file='test.inc', line=1) + metadata.setVar('FROM_CLASS', 'class', file='test.bbclass', line=2) + metadata.setVar('FROM_CONFIG', 'config', file='conf/local.conf', line=3) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertNotIn('FROM_INC', variables) + self.assertNotIn('FROM_CLASS', variables) + self.assertNotIn('FROM_CONFIG', variables) + + def test_combined_override_is_included_under_logical_name(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'foo:bar:local', + file='conf/bitbake.conf', line=1) + metadata.setVar('COMBINED:local:foo:bar', 'recipe', + file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertEqual(variables['COMBINED'], 'recipe') + self.assertNotIn('OVERRIDES', variables) + + def test_active_override_function_is_omitted(self): + metadata = self.metadata() + metadata.setVar('OVERRIDES', 'machine', file='conf/bitbake.conf', line=1) + metadata.setVar('do_work:machine', 'echo test', file='test.bb', line=1) + metadata.setVarFlag('do_work:machine', 'func', True, + file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + variables = json.loads(zlib.decompress(info.recipe_variables)) + + self.assertNotIn('do_work', variables) + + def test_recipe_variables_survive_extra_cache_mapping(self): + metadata = self.metadata() + metadata.setVar('EMPTY', '', file='test.bb', line=1) + info = HobRecipeInfo('test.bb', metadata) + cachedata = type('CacheData', (), {})() + + HobRecipeInfo.init_cacheData(cachedata) + info.add_cacheData(cachedata, 'test.bb') + + self.assertIn('recipe_variables', HobRecipeInfo.cachefields) + variables = json.loads(zlib.decompress( + cachedata.recipe_variables['test.bb'])) + self.assertEqual(variables['EMPTY'], '') + + def test_recipe_variables_are_compressed_before_caching(self): + metadata = self.metadata() + metadata.setVar('LARGE', 'repeated-value-' * 1000, + file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + + self.assertLess(len(info.recipe_variables), len(metadata.getVar('LARGE'))) + + def test_unexpandable_variable_is_omitted(self): + metadata = self.metadata() + metadata.setVar('BROKEN', '${BROKEN}', file='test.bb', line=1) + + info = HobRecipeInfo('test.bb', metadata) + + variables = json.loads(zlib.decompress(info.recipe_variables)) + self.assertNotIn('BROKEN', variables) + +if __name__ == '__main__': + unittest.main() From patchwork Fri Aug 14 15:35:55 2026 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paolo Wattebled X-Patchwork-Id: 95278 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5A869C5CFCF for ; Fri, 14 Aug 2026 15:36:13 +0000 (UTC) Received: from mail.savoirfairelinux.com (mail.savoirfairelinux.com [208.88.110.44]) by mx.groups.io with SMTP id smtpd.msgproc02-g2.15781.1786721770308435080 for ; Fri, 14 Aug 2026 08:36:11 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@savoirfairelinux.com header.s=DFC430D2-D198-11EC-948E-34200CB392D2 header.b=tZJxyPRT; spf=pass (domain: savoirfairelinux.com, ip: 208.88.110.44, mailfrom: paolo.wattebled@savoirfairelinux.com) Received: from localhost (localhost [127.0.0.1]) by mail.savoirfairelinux.com (Postfix) with ESMTP id 27E733D818BB for ; Fri, 14 Aug 2026 11:36:09 -0400 (EDT) Received: from mail.savoirfairelinux.com ([127.0.0.1]) by localhost (mail.savoirfairelinux.com [127.0.0.1]) (amavis, port 10032) with ESMTP id TqKjW2fOyT8N; Fri, 14 Aug 2026 11:36:08 -0400 (EDT) Received: from localhost (localhost [127.0.0.1]) by mail.savoirfairelinux.com (Postfix) with ESMTP id 0859F3D818F3; Fri, 14 Aug 2026 11:36:08 -0400 (EDT) DKIM-Filter: OpenDKIM Filter v2.10.3 mail.savoirfairelinux.com 0859F3D818F3 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=savoirfairelinux.com; s=DFC430D2-D198-11EC-948E-34200CB392D2; t=1786721768; bh=u/PuuYVfW9CdEuMtJt0hKZJZqy3DzY2bRprAgD99BiA=; h=From:To:Date:Message-ID:MIME-Version; b=tZJxyPRTDvMSdJU0p+HMer8N8vgf0znWMQc3Uj/dkoTtdNfT2617jKI5DdM8+lvdM 6S2lW0MIf+wUVGi/BlNl91A0e7qzkJic+WVucZHIRamilNV4hpf0ddHTk9UDdzRqO2 tzAWeciID+kdmn3WB/7qSfmMHILWBF/jCgjNuLOUoJpPIjmvQptj+gh6nCEZQLhVR+ sq2oc4f876+KulTIbSMtmUTDN4orRP8iVLS5R3+GY+iGV9beIrhJP085FA3XBDbINZ XwWYClGUmqnaGbn36hxK+O5tsYAK1PCGLOJNoHoI/OtTwE+9aw7tjXXnR4hp5pK0RW 2sRdSOEJFSsAQ== X-Virus-Scanned: amavis at mail.savoirfairelinux.com Received: from mail.savoirfairelinux.com ([127.0.0.1]) by localhost (mail.savoirfairelinux.com [127.0.0.1]) (amavis, port 10026) with ESMTP id MJBcZlGKC_OS; Fri, 14 Aug 2026 11:36:07 -0400 (EDT) Received: from pwattebled.mtl.sfl (unknown [192.168.51.254]) by mail.savoirfairelinux.com (Postfix) with ESMTPSA id DE6883D818B5; Fri, 14 Aug 2026 11:36:07 -0400 (EDT) From: Paolo Wattebled To: bitbake-devel@lists.openembedded.org Cc: Paolo Wattebled Subject: [PATCH v2 2/5] toaster: Add recipe variable snapshot model Date: Fri, 14 Aug 2026 11:35:55 -0400 Message-ID: <20260814153602.2107764-3-paolo.wattebled@savoirfairelinux.com> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260814153602.2107764-1-paolo.wattebled@savoirfairelinux.com> References: <20260814153602.2107764-1-paolo.wattebled@savoirfairelinux.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from 45-33-107-173.ip.linodeusercontent.com [45.33.107.173] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 14 Aug 2026 15:36:13 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/19940 Add one compressed variable snapshot per build and recipe. Store the entry count separately so recipe list queries do not need to decompress each snapshot. AI-Generated: Uses GitHub Copilot and OpenCode with GPT-5.6 Sol Signed-off-by: Paolo Wattebled --- .../orm/migrations/0022_recipevariable.py | 27 +++++++++++++++++++ lib/toaster/orm/models.py | 10 +++++++ 2 files changed, 37 insertions(+) create mode 100644 lib/toaster/orm/migrations/0022_recipevariable.py diff --git a/lib/toaster/orm/migrations/0022_recipevariable.py b/lib/toaster/orm/migrations/0022_recipevariable.py new file mode 100644 index 000000000..455a8ce23 --- /dev/null +++ b/lib/toaster/orm/migrations/0022_recipevariable.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.5 on 2026-08-11 00:00 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0021_eventlogsimports'), + ] + + operations = [ + migrations.CreateModel( + name='RecipeVariable', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('variables', models.BinaryField()), + ('variable_count', models.PositiveIntegerField(default=0)), + ('build', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='orm.build')), + ('recipe', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='orm.recipe')), + ], + options={ + 'unique_together': {('build', 'recipe')}, + }, + ), + ] diff --git a/lib/toaster/orm/models.py b/lib/toaster/orm/models.py index e2f488ed8..c240836cc 100644 --- a/lib/toaster/orm/models.py +++ b/lib/toaster/orm/models.py @@ -1346,6 +1346,16 @@ class Recipe(models.Model): unique_together = (("layer_version", "file_path", "pathflags"), ) +class RecipeVariable(models.Model): + build = models.ForeignKey(Build, on_delete=models.CASCADE) + recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE) + variables = models.BinaryField() + variable_count = models.PositiveIntegerField(default=0) + + class Meta: + unique_together = (("build", "recipe"), ) + + class Recipe_DependencyManager(models.Manager): use_for_related_fields = True From patchwork Fri Aug 14 15:35:56 2026 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paolo Wattebled X-Patchwork-Id: 95279 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 8CB9CC5CFC1 for ; Fri, 14 Aug 2026 15:36:13 +0000 (UTC) Received: from mail.savoirfairelinux.com (mail.savoirfairelinux.com [208.88.110.44]) by mx.groups.io with SMTP id smtpd.msgproc01-g2.15592.1786721771881735523 for ; Fri, 14 Aug 2026 08:36:12 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@savoirfairelinux.com header.s=DFC430D2-D198-11EC-948E-34200CB392D2 header.b=c9WOacvR; spf=pass (domain: savoirfairelinux.com, ip: 208.88.110.44, mailfrom: paolo.wattebled@savoirfairelinux.com) Received: from localhost (localhost [127.0.0.1]) by mail.savoirfairelinux.com (Postfix) with ESMTP id B50B23D818B5 for ; Fri, 14 Aug 2026 11:36:10 -0400 (EDT) Received: from mail.savoirfairelinux.com ([127.0.0.1]) by localhost (mail.savoirfairelinux.com [127.0.0.1]) (amavis, port 10032) with ESMTP id e36tCeD_58ua; Fri, 14 Aug 2026 11:36:08 -0400 (EDT) Received: from localhost (localhost [127.0.0.1]) by mail.savoirfairelinux.com (Postfix) with ESMTP id 1CC153D8189E; Fri, 14 Aug 2026 11:36:08 -0400 (EDT) DKIM-Filter: OpenDKIM Filter v2.10.3 mail.savoirfairelinux.com 1CC153D8189E DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=savoirfairelinux.com; s=DFC430D2-D198-11EC-948E-34200CB392D2; t=1786721768; bh=XOBMhM+S2ZKrr+mNE+gJGdSSS+j34S6CWB/WROb2uQA=; h=From:To:Date:Message-ID:MIME-Version; b=c9WOacvRBKDLD3fxdW6kgJN3c0AbMnpCplRKOizi9DCwL2iVyXs9hgftXOGj+MH3d pSRwg4UeRLo+WF20WMQZGSt8Bq6TwRP9bx9gnS//IEC2DW0BxtYPzHVsdgNIwYc6em opOo+KUvq1loCFi+KZHNqZcwVJWLVSIXmZXQgnzMpSXkioG82tlUtOPC1HeNO7G4R1 HbWPVrhEJsu5YefIT+TWF763QtIO11l560oUcIOd0IMlxlRlRjlFm4jp+Bjw67KbPd zHifYS715hEETC+qAPWOFNeiw1T7fNZcr+ZYWWezeqIb2kAPhPhpVEpN6jSBGIW90f CMrf2SpaLbAJw== X-Virus-Scanned: amavis at mail.savoirfairelinux.com Received: from mail.savoirfairelinux.com ([127.0.0.1]) by localhost (mail.savoirfairelinux.com [127.0.0.1]) (amavis, port 10026) with ESMTP id OhoucXlKwotJ; Fri, 14 Aug 2026 11:36:07 -0400 (EDT) Received: from pwattebled.mtl.sfl (unknown [192.168.51.254]) by mail.savoirfairelinux.com (Postfix) with ESMTPSA id E3F663D818BB; Fri, 14 Aug 2026 11:36:07 -0400 (EDT) From: Paolo Wattebled To: bitbake-devel@lists.openembedded.org Cc: Paolo Wattebled Subject: [PATCH v2 3/5] toaster: Store recipe variable snapshots Date: Fri, 14 Aug 2026 11:35:56 -0400 Message-ID: <20260814153602.2107764-4-paolo.wattebled@savoirfairelinux.com> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260814153602.2107764-1-paolo.wattebled@savoirfairelinux.com> References: <20260814153602.2107764-1-paolo.wattebled@savoirfairelinux.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from 45-33-107-173.ip.linodeusercontent.com [45.33.107.173] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 14 Aug 2026 15:36:13 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/19942 Persist compressed recipe variable snapshots from dependency graph data and replace each build's snapshot set atomically. Keep existing data when older dependency payloads omit the field, remove stale rows on refresh and cover rollback behavior with database tests. AI-Generated: Uses GitHub Copilot and OpenCode with GPT-5.6 Sol Signed-off-by: Paolo Wattebled --- lib/bb/ui/buildinfohelper.py | 21 ++++++- lib/toaster/tests/db/test_db.py | 101 ++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/lib/bb/ui/buildinfohelper.py b/lib/bb/ui/buildinfohelper.py index 4ee45d67a..679f016bd 100644 --- a/lib/bb/ui/buildinfohelper.py +++ b/lib/bb/ui/buildinfohelper.py @@ -8,8 +8,10 @@ import sys import bb +import json import re import os +import zlib import django from django.utils import timezone @@ -31,7 +33,7 @@ from orm.models import Target_Image_File, TargetKernelFile, TargetSDKFile from orm.models import Variable, VariableHistory from orm.models import Package, Package_File, Target_Installed_Package, Target_File from orm.models import Task_Dependency, Package_Dependency -from orm.models import Recipe_Dependency, Provides +from orm.models import Recipe_Dependency, RecipeVariable, Provides from orm.models import Project, CustomImagePackage from orm.models import signal_runbuilds @@ -1459,6 +1461,23 @@ class BuildInfoHelper(object): t.save() self.internal_state['recipes'][pn] = recipe + if any('recipe_variables' in recipe_data + for recipe_data in event._depgraph['pn'].values()): + recipe_variables = [] + for pn, recipe_data in event._depgraph['pn'].items(): + variables = recipe_data.get('recipe_variables') + if variables is None: + continue + recipe_variables.append(RecipeVariable( + build=self.internal_state['build'], + recipe=self.internal_state['recipes'][pn], + variables=variables, + variable_count=len(json.loads(zlib.decompress(variables))))) + with transaction.atomic(): + RecipeVariable.objects.filter( + build=self.internal_state['build']).delete() + RecipeVariable.objects.bulk_create(recipe_variables) + # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED assume_provided = self.server.runCommand(["getVariable", "ASSUME_PROVIDED"])[0].split() diff --git a/lib/toaster/tests/db/test_db.py b/lib/toaster/tests/db/test_db.py index 072ab9436..7c48b1de3 100644 --- a/lib/toaster/tests/db/test_db.py +++ b/lib/toaster/tests/db/test_db.py @@ -24,6 +24,10 @@ import sys import pytest +from types import SimpleNamespace +from unittest.mock import Mock, patch +import json +import zlib try: from StringIO import StringIO @@ -34,6 +38,11 @@ from contextlib import contextmanager from django.core import management from django.test import TestCase +from django.utils import timezone + +from bb.ui.buildinfohelper import BuildInfoHelper, ORMWrapper +from orm.models import Build, Layer, Layer_Version, Project, Recipe +from orm.models import RecipeVariable @contextmanager @@ -56,3 +65,95 @@ class MigrationTest(TestCase): with capture(makemigrations) as output: self.assertEqual(output, "No changes detected\n") + + +class RecipeVariableTest(TestCase): + + def setUp(self): + now = timezone.now() + project = Project.objects.get_or_create_default_project() + self.build = Build.objects.create( + project=project, machine='', distro='', distro_version='', + started_on=now, completed_on=now, cooker_log_path='', + bitbake_version='', progress_item='') + layer = Layer.objects.create(name='test', layer_index_url='') + self.layer_version = Layer_Version.objects.create( + build=self.build, layer=layer, branch='', commit='', + local_path='/layer') + + def _store(self, recipe_data): + if 'recipe_variables' in recipe_data: + recipe_data = dict(recipe_data) + recipe_data['recipe_variables'] = zlib.compress(json.dumps( + recipe_data['recipe_variables'], separators=(',', ':') + ).encode('utf-8')) + helper = BuildInfoHelper.__new__(BuildInfoHelper) + helper.internal_state = {'build': self.build, 'targets': []} + helper.orm_wrapper = ORMWrapper() + helper.server = Mock() + helper.server.runCommand.return_value = ['', None] + helper._get_layer_version_for_path = Mock(return_value=self.layer_version) + event = SimpleNamespace(_depgraph={ + 'layer-priorities': [], + 'pn': {'test': dict({'filename': '/layer/test.bb'}, **recipe_data)}, + 'depends': {'test': []}, + 'tdepends': {}, + }) + + helper.store_dependency_information(event) + + return helper.internal_state['recipes']['test'] + + def test_dependency_information_stores_recipe_variables(self): + recipe = self._store({'recipe_variables': {'EMPTY': '', 'FOO': 'bar'}}) + + snapshot = RecipeVariable.objects.get(build=self.build, recipe=recipe) + self.assertEqual( + json.loads(zlib.decompress(snapshot.variables)), + {'EMPTY': '', 'FOO': 'bar'}) + self.assertEqual(snapshot.variable_count, 2) + + def test_old_payload_does_not_delete_recipe_variables(self): + recipe = self._store({'recipe_variables': {'FOO': 'bar'}}) + self._store({}) + + snapshot = RecipeVariable.objects.get(build=self.build, recipe=recipe) + self.assertEqual( + json.loads(zlib.decompress(snapshot.variables)), {'FOO': 'bar'}) + + def test_dependency_information_replaces_recipe_variables(self): + recipe = self._store({'recipe_variables': {'FOO': 'old'}}) + self._store({'recipe_variables': {'BAR': 'new'}}) + + snapshot = RecipeVariable.objects.get(build=self.build, recipe=recipe) + self.assertEqual( + json.loads(zlib.decompress(snapshot.variables)), {'BAR': 'new'}) + self.assertEqual(snapshot.variable_count, 1) + + def test_dependency_information_removes_stale_recipe_snapshot(self): + stale_recipe = Recipe.objects.create( + name='stale', version='', layer_version=self.layer_version, + file_path='stale.bb') + RecipeVariable.objects.create( + build=self.build, recipe=stale_recipe, + variables=zlib.compress(b'{"STALE":"value"}')) + + current_recipe = self._store({'recipe_variables': {'FOO': 'bar'}}) + + self.assertFalse(RecipeVariable.objects.filter( + build=self.build, recipe=stale_recipe).exists()) + self.assertTrue(RecipeVariable.objects.filter( + build=self.build, recipe=current_recipe).exists()) + + def test_dependency_information_rolls_back_failed_bulk_create(self): + recipe = self._store({'recipe_variables': {'FOO': 'old'}}) + with patch.object(RecipeVariable.objects, 'bulk_create', + side_effect=RuntimeError('injected failure')): + with self.assertRaisesRegex(RuntimeError, 'injected failure'): + self._store({'recipe_variables': {'FOO': 'new'}}) + + snapshots = RecipeVariable.objects.filter(build=self.build) + self.assertEqual(snapshots.count(), 1) + snapshot = snapshots.get(recipe=recipe) + self.assertEqual( + json.loads(zlib.decompress(snapshot.variables)), {'FOO': 'old'}) From patchwork Fri Aug 14 15:35:57 2026 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paolo Wattebled X-Patchwork-Id: 95277 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 69ED5C5CFDB for ; Fri, 14 Aug 2026 15:36:13 +0000 (UTC) Received: from mail.savoirfairelinux.com (mail.savoirfairelinux.com [208.88.110.44]) by mx.groups.io with SMTP id smtpd.msgproc02-g2.15782.1786721771470651839 for ; Fri, 14 Aug 2026 08:36:11 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@savoirfairelinux.com header.s=DFC430D2-D198-11EC-948E-34200CB392D2 header.b=O2fu98MH; spf=pass (domain: savoirfairelinux.com, ip: 208.88.110.44, mailfrom: paolo.wattebled@savoirfairelinux.com) Received: from localhost (localhost [127.0.0.1]) by mail.savoirfairelinux.com (Postfix) with ESMTP id 889513D8185E for ; Fri, 14 Aug 2026 11:36:10 -0400 (EDT) Received: from mail.savoirfairelinux.com ([127.0.0.1]) by localhost (mail.savoirfairelinux.com [127.0.0.1]) (amavis, port 10032) with ESMTP id 9xljp5a8teHL; Fri, 14 Aug 2026 11:36:09 -0400 (EDT) Received: from localhost (localhost [127.0.0.1]) by mail.savoirfairelinux.com (Postfix) with ESMTP id 4AC213D818B5; Fri, 14 Aug 2026 11:36:08 -0400 (EDT) DKIM-Filter: OpenDKIM Filter v2.10.3 mail.savoirfairelinux.com 4AC213D818B5 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=savoirfairelinux.com; s=DFC430D2-D198-11EC-948E-34200CB392D2; t=1786721768; bh=tsoPw+pvdQM1mJ8+hKpGH3S1RnNdrd6GL2U2MESIiIc=; h=From:To:Date:Message-ID:MIME-Version; b=O2fu98MH1OM6KMxuHQlMZfwFsN6BQgssNiREOxxsjJkHsDkbqcndZvz+ALbJbMK1b kXnekCqETIvU7gC1VqTraCD5POi/PqF0y2rqsngGgBDIBCJH2AawWZuoevK71nH+ZU uoGI9+eapnKkpeKcLxNk70M/wn8MpqlAh76EyeEXFq9D5XB1ODt3Y6JzVW9AgkbwIm rRY3TVAQjvwpNPhXWlUuX0xYUNhU9SEPLRvnep77e0gJUnDDWVzkgFQlU+S1UsY/va 2nfrTPFc5Re79XpJauMSeqXIWaxTANe0OdcKOTxIZ/z4Zdh59N94f5jR9YNvv6fa+6 YGBRBCebxyEVA== X-Virus-Scanned: amavis at mail.savoirfairelinux.com Received: from mail.savoirfairelinux.com ([127.0.0.1]) by localhost (mail.savoirfairelinux.com [127.0.0.1]) (amavis, port 10026) with ESMTP id X4xlCnYJ3zE3; Fri, 14 Aug 2026 11:36:08 -0400 (EDT) Received: from pwattebled.mtl.sfl (unknown [192.168.51.254]) by mail.savoirfairelinux.com (Postfix) with ESMTPSA id EFFDD3D818D3; Fri, 14 Aug 2026 11:36:07 -0400 (EDT) From: Paolo Wattebled To: bitbake-devel@lists.openembedded.org Cc: Paolo Wattebled Subject: [PATCH v2 4/5] toaster: Generalize detail table controls Date: Fri, 14 Aug 2026 11:35:57 -0400 Message-ID: <20260814153602.2107764-5-paolo.wattebled@savoirfairelinux.com> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260814153602.2107764-1-paolo.wattebled@savoirfairelinux.com> References: <20260814153602.2107764-1-paolo.wattebled@savoirfairelinux.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from 45-33-107-173.ip.linodeusercontent.com [45.33.107.173] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 14 Aug 2026 15:36:13 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/19941 Allow detail tables to provide their validated page size and default ordering while preserving existing defaults for current users. AI-Generated: Uses GitHub Copilot and OpenCode with GPT-5.6 Sol Signed-off-by: Paolo Wattebled --- .../toastergui/templates/detail_pagination_bottom.html | 4 ++-- lib/toaster/toastergui/templates/detail_search_header.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/toaster/toastergui/templates/detail_pagination_bottom.html b/lib/toaster/toastergui/templates/detail_pagination_bottom.html index 15adfbc91..ddcba133e 100644 --- a/lib/toaster/toastergui/templates/detail_pagination_bottom.html +++ b/lib/toaster/toastergui/templates/detail_pagination_bottom.html @@ -41,8 +41,8 @@ ', + }), + self._recipe_variables(other_build, self.recipe1, { + 'OTHER_BUILD': 'hidden', + }), + self._recipe_variables(build, other_recipe, { + 'OTHER_RECIPE': 'hidden', + }), + ]) + url = reverse('recipe', args=(build.pk, self.recipe1.pk, '5')) + + response = self.client.get(url, { + 'count': 100, + 'page': 1, + 'orderby': 'variable_name:+', + 'search': 'SRC_URI', + }) + + self.assertEqual(response.status_code, 200) + self.assertTemplateUsed(response, 'recipe.html') + self.assertContains(response, 'SRC_URI') + self.assertContains(response, 'git://example.invalid/src') + self.assertNotContains(response, 'OTHER_BUILD') + self.assertNotContains(response, 'OTHER_RECIPE') + + response = self.client.get(url, { + 'count': 100, + 'page': 1, + 'orderby': 'variable_name:+', + }) + self.assertContains(response, 'Variables (2)') + self.assertContains(response, '<script>alert(1)</script>') + self.assertNotContains(response, '') + + def test_recipe_variables_tab_redirects_paginates_and_handles_empty(self): + build = Build.objects.get(pk=1) + url = reverse('recipe', args=(build.pk, self.recipe1.pk, '5')) + + response = self.client.get(url) + self.assertEqual(response.status_code, 302) + self.assertIn('count=100', response.url) + self.assertIn('orderby=variable_name%3A%2B', response.url) + + RecipeVariable.objects.bulk_create([ + self._recipe_variables(build, self.recipe1, { + 'VAR_%02d' % index: str(index) for index in range(12) + }) + ]) + response = self.client.get(url, { + 'count': 10, + 'page': 2, + 'orderby': 'variable_name:+', + }) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.context['objects'].paginator.count, 12) + self.assertEqual(response.context['objects'].number, 2) + self.assertContains(response, 'VAR_10') + self.assertContains(response, 'VAR_11') + self.assertNotContains(response, 'VAR_00') + + RecipeVariable.objects.filter(build=build, recipe=self.recipe1).delete() + response = self.client.get(url, { + 'count': 100, + 'page': 1, + 'orderby': 'variable_name:+', + }) + self.assertContains(response, 'No resolved recipe datastore variables are available.') + + def test_recipe_variables_tab_bounds_query_parameters(self): + build = Build.objects.get(pk=1) + RecipeVariable.objects.bulk_create([ + self._recipe_variables(build, self.recipe1, {'SRC_URI': 'value'}) + ]) + url = reverse('recipe', args=(build.pk, self.recipe1.pk, '5')) + + response = self.client.get(url, { + 'count': 'alert(document.domain)', + 'page': 1, + 'orderby': 'variable_value:+', + 'filter': 'variable_value__regex:(a+)+$', + }) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.context['objects'].paginator.per_page, 100) + self.assertNotContains(response, 'alert(document.domain)') + self.assertEqual( + [variable['variable_name'] + for variable in response.context['objects'].object_list], + ['SRC_URI']) + + response = self.client.get(url, { + 'count': 100, + 'page': 1, + 'orderby': 'variable_name:+', + 'search': ('SRC_URI ' * 20) + ('x' * 300), + }) + self.assertLessEqual(len(response.context['search_term']), 256) + self.assertLessEqual(len(response.context['search_term'].split()), 16) + + def test_built_recipes_table_shows_variable_count(self): + build = Build.objects.get(pk=1) + self._recipe_variables(build, self.recipe1, { + 'FOO': 'one', + 'BAR': 'two', + }).save() + url = reverse('recipes', args=(build.pk,)) + + response = self.client.get(url, { + 'format': 'json', + 'limit': 25, + 'page': 1, + }) + data = json.loads(response.content) + row = next(row for row in data['rows'] + if self.recipe1.name in row['name']) + + self.assertIn('Variables', [column['title'] for column in data['columns']]) + self.assertIn('>2', row['variable_count']) + + def test_recipe_packages_tab_shows_variable_count(self): + build = Build.objects.get(pk=1) + self._recipe_variables(build, self.recipe1, { + 'FOO': 'one', + 'BAR': 'two', + }).save() + + response = self.client.get(reverse( + 'recipe_packages', args=(build.pk, self.recipe1.pk)), { + 'count': 10, + 'page': 1, + 'orderby': 'name:+', + }) + + self.assertEqual(response.status_code, 200) + self.assertContains(response, 'Variables (2)') + def test_typeaheads(self): """Test typeahead ReST API""" layers_url = reverse('xhr_layerstypeahead', args=(self.project.id,)) diff --git a/lib/toaster/toastergui/buildtables.py b/lib/toaster/toastergui/buildtables.py index 327059d00..52c3dd779 100644 --- a/lib/toaster/toastergui/buildtables.py +++ b/lib/toaster/toastergui/buildtables.py @@ -239,7 +239,10 @@ class BuiltRecipesTable(BuildTablesMixin): def setup_queryset(self, *args, **kwargs): build = Build.objects.get(pk=kwargs['build_id']) self.static_context_extra['build'] = build - self.queryset = build.get_recipes() + self.queryset = build.get_recipes().annotate( + variable_count=Sum( + 'recipevariable__variable_count', + filter=Q(recipevariable__build=build), default=0)) self.queryset = self.queryset.order_by(self.default_orderby) def setup_columns(self, *args, **kwargs): @@ -323,6 +326,12 @@ class BuiltRecipesTable(BuildTablesMixin): hideable=False, field_name="version") + self.add_column( + title="Variables", + field_name="variable_count", + static_data_name="variable_count", + static_data_template='{{data.variable_count|default:0}}') + self.add_column(title="Dependencies", static_data_name="dependencies", static_data_template=depends_on_tmpl) diff --git a/lib/toaster/toastergui/templates/recipe.html b/lib/toaster/toastergui/templates/recipe.html index 4b5301b54..be99ebc54 100644 --- a/lib/toaster/toastergui/templates/recipe.html +++ b/lib/toaster/toastergui/templates/recipe.html @@ -52,6 +52,13 @@ Reverse build dependencies ({{object.r_dependencies_depends.all.count}}) +
  • + + + Variables ({{recipe_variable_count}}) + +
    • @@ -277,6 +284,31 @@ {% endif %}
    +
    + {% if not objects and not request.GET.search %} +
    + No resolved recipe datastore variables are available. +
    + {% else %} + {% with "variables" as search_what %} + {% include "detail_search_header.html" %} + {% endwith %} + {% if objects %} + + {% include "detail_sorted_header.html" %} + + {% for variable in objects %} + + + + + {% endfor %} + +
    {{variable.variable_name}}{{variable.variable_value}}
    + {% include "detail_pagination_bottom.html" %} + {% endif %} + {% endif %} +
    diff --git a/lib/toaster/toastergui/templates/recipe_packages.html b/lib/toaster/toastergui/templates/recipe_packages.html index 37a586f38..abd64ca7e 100644 --- a/lib/toaster/toastergui/templates/recipe_packages.html +++ b/lib/toaster/toastergui/templates/recipe_packages.html @@ -51,6 +51,13 @@ Reverse build dependencies ({{recipe.r_dependencies_depends.all.count}}) +
  • + + + Variables ({{variable_count}}) + +
    • {#
    #} diff --git a/lib/toaster/toastergui/views.py b/lib/toaster/toastergui/views.py index 061e6436c..26cf5ca26 100644 --- a/lib/toaster/toastergui/views.py +++ b/lib/toaster/toastergui/views.py @@ -10,6 +10,7 @@ import ast import re import subprocess import sys +import zlib import bb.cooker from bb.ui import toasterui @@ -20,6 +21,7 @@ from django.db import IntegrityError from django.shortcuts import render, redirect, get_object_or_404, HttpResponseRedirect from django.utils.http import urlencode from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe +from orm.models import RecipeVariable from orm.models import LogMessage, Variable, Package_Dependency, Package from orm.models import Task_Dependency, Package_File from orm.models import Target_Installed_Package, Target_File @@ -619,10 +621,13 @@ def recipe(request, build_id, recipe_id, active_tab="1"): layer = Layer.objects.get(pk=layer_version.layer_id) tasks_list = Task.objects.filter(recipe_id = recipe_id, build_id = build_id).exclude(order__isnull=True).exclude(task_name__endswith='_setscene').exclude(outcome=Task.OUTCOME_NA) package_count = Package.objects.filter(recipe_id = recipe_id).filter(build_id = build_id).filter(size__gte=0).count() + recipe_variable_count = RecipeVariable.objects.filter( + build_id=build_id, recipe_id=recipe_id).values_list( + 'variable_count', flat=True).first() or 0 - if active_tab != '1' and active_tab != '3' and active_tab != '4' : + if active_tab not in ('1', '3', '4', '5'): active_tab = '1' - tab_states = {'1': '', '3': '', '4': ''} + tab_states = {'1': '', '3': '', '4': '', '5': ''} tab_states[active_tab] = 'active' context = { @@ -632,8 +637,63 @@ def recipe(request, build_id, recipe_id, active_tab="1"): 'layer' : layer, 'tasks' : tasks_list, 'package_count' : package_count, + 'recipe_variable_count' : recipe_variable_count, 'tab_states' : tab_states, } + + if active_tab == '5': + (requested_pagesize, requested_orderby) = _get_parameters_values( + request, 100, 'variable_name:+') + pagesize = requested_pagesize \ + if str(requested_pagesize) in ('10', '25', '50', '100', '150') \ + else 100 + orderby = requested_orderby \ + if requested_orderby in ('variable_name:+', 'variable_name:-') \ + else 'variable_name:+' + mandatory_parameters = { + 'count': pagesize, + 'page': 1, + 'orderby': orderby, + } + if _verify_parameters(request.GET, mandatory_parameters): + return _redirect_parameters( + 'recipe', request.GET, mandatory_parameters, + build_id=build_id, recipe_id=recipe_id, active_tab='5') + + search_term = request.GET.get('search', '')[:256] + search_term = ' '.join(search_term.split()[:16]) + snapshot = RecipeVariable.objects.filter( + build_id=build_id, recipe_id=recipe_id).first() + variables = [] + if snapshot: + values = json.loads(zlib.decompress(snapshot.variables).decode('utf-8')) + variables = [ + {'variable_name': name, 'variable_value': value} + for name, value in values.items() + if not search_term or search_term.lower() in name.lower() + or search_term.lower() in value.lower() + ] + variables.sort( + key=lambda variable: variable['variable_name'], + reverse=orderby.endswith(':-')) + context['variable_count'] = len(variables) + context['objects'] = _build_page_range( + Paginator(variables, pagesize), request.GET.get('page', 1)) + context['object_count'] = context['variable_count'] + context['validated_pagesize'] = pagesize + context['search_term'] = search_term + context['search_orderby'] = 'variable_name:+' + context['tablecols'] = [ + { + 'name': 'Variable', + 'orderfield': _get_toggle_order(request, 'variable_name'), + 'ordericon': _get_toggle_order_icon(request, 'variable_name'), + 'orderkey': 'variable_name', + }, + {'name': 'Value'}, + ] + _set_parameters_values(pagesize, orderby, request) + return toaster_render(request, template, context) def recipe_packages(request, build_id, recipe_id): @@ -651,6 +711,9 @@ def recipe_packages(request, build_id, recipe_id): recipe_object = Recipe.objects.get(pk=recipe_id) queryset = Package.objects.filter(recipe_id = recipe_id).filter(build_id = build_id).filter(size__gte=0) package_count = queryset.count() + variable_count = RecipeVariable.objects.filter( + build_id=build_id, recipe_id=recipe_id).values_list( + 'variable_count', flat=True).first() or 0 queryset = _get_queryset(Package, queryset, filter_string, search_term, ordering_string, 'name') packages = _build_page_range(Paginator(queryset, pagesize),request.GET.get('page', 1)) @@ -660,6 +723,7 @@ def recipe_packages(request, build_id, recipe_id): 'recipe' : recipe_object, 'objects' : packages, 'object_count' : package_count, + 'variable_count' : variable_count, 'tablecols':[ { 'name':'Package',