diff --git a/lib/bb/ui/buildinfohelper.py b/lib/bb/ui/buildinfohelper.py index 4ee45d67a..679f016bd 100644 --- a/lib/bb/ui/buildinfohelper.py +++ b/lib/bb/ui/buildinfohelper.py @@ -8,8 +8,10 @@ import sys import bb +import json import re import os +import zlib import django from django.utils import timezone @@ -31,7 +33,7 @@ from orm.models import Target_Image_File, TargetKernelFile, TargetSDKFile from orm.models import Variable, VariableHistory from orm.models import Package, Package_File, Target_Installed_Package, Target_File from orm.models import Task_Dependency, Package_Dependency -from orm.models import Recipe_Dependency, Provides +from orm.models import Recipe_Dependency, RecipeVariable, Provides from orm.models import Project, CustomImagePackage from orm.models import signal_runbuilds @@ -1459,6 +1461,23 @@ class BuildInfoHelper(object): t.save() self.internal_state['recipes'][pn] = recipe + if any('recipe_variables' in recipe_data + for recipe_data in event._depgraph['pn'].values()): + recipe_variables = [] + for pn, recipe_data in event._depgraph['pn'].items(): + variables = recipe_data.get('recipe_variables') + if variables is None: + continue + recipe_variables.append(RecipeVariable( + build=self.internal_state['build'], + recipe=self.internal_state['recipes'][pn], + variables=variables, + variable_count=len(json.loads(zlib.decompress(variables))))) + with transaction.atomic(): + RecipeVariable.objects.filter( + build=self.internal_state['build']).delete() + RecipeVariable.objects.bulk_create(recipe_variables) + # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED assume_provided = self.server.runCommand(["getVariable", "ASSUME_PROVIDED"])[0].split() diff --git a/lib/toaster/tests/db/test_db.py b/lib/toaster/tests/db/test_db.py index 072ab9436..7c48b1de3 100644 --- a/lib/toaster/tests/db/test_db.py +++ b/lib/toaster/tests/db/test_db.py @@ -24,6 +24,10 @@ import sys import pytest +from types import SimpleNamespace +from unittest.mock import Mock, patch +import json +import zlib try: from StringIO import StringIO @@ -34,6 +38,11 @@ from contextlib import contextmanager from django.core import management from django.test import TestCase +from django.utils import timezone + +from bb.ui.buildinfohelper import BuildInfoHelper, ORMWrapper +from orm.models import Build, Layer, Layer_Version, Project, Recipe +from orm.models import RecipeVariable @contextmanager @@ -56,3 +65,95 @@ class MigrationTest(TestCase): with capture(makemigrations) as output: self.assertEqual(output, "No changes detected\n") + + +class RecipeVariableTest(TestCase): + + def setUp(self): + now = timezone.now() + project = Project.objects.get_or_create_default_project() + self.build = Build.objects.create( + project=project, machine='', distro='', distro_version='', + started_on=now, completed_on=now, cooker_log_path='', + bitbake_version='', progress_item='') + layer = Layer.objects.create(name='test', layer_index_url='') + self.layer_version = Layer_Version.objects.create( + build=self.build, layer=layer, branch='', commit='', + local_path='/layer') + + def _store(self, recipe_data): + if 'recipe_variables' in recipe_data: + recipe_data = dict(recipe_data) + recipe_data['recipe_variables'] = zlib.compress(json.dumps( + recipe_data['recipe_variables'], separators=(',', ':') + ).encode('utf-8')) + helper = BuildInfoHelper.__new__(BuildInfoHelper) + helper.internal_state = {'build': self.build, 'targets': []} + helper.orm_wrapper = ORMWrapper() + helper.server = Mock() + helper.server.runCommand.return_value = ['', None] + helper._get_layer_version_for_path = Mock(return_value=self.layer_version) + event = SimpleNamespace(_depgraph={ + 'layer-priorities': [], + 'pn': {'test': dict({'filename': '/layer/test.bb'}, **recipe_data)}, + 'depends': {'test': []}, + 'tdepends': {}, + }) + + helper.store_dependency_information(event) + + return helper.internal_state['recipes']['test'] + + def test_dependency_information_stores_recipe_variables(self): + recipe = self._store({'recipe_variables': {'EMPTY': '', 'FOO': 'bar'}}) + + snapshot = RecipeVariable.objects.get(build=self.build, recipe=recipe) + self.assertEqual( + json.loads(zlib.decompress(snapshot.variables)), + {'EMPTY': '', 'FOO': 'bar'}) + self.assertEqual(snapshot.variable_count, 2) + + def test_old_payload_does_not_delete_recipe_variables(self): + recipe = self._store({'recipe_variables': {'FOO': 'bar'}}) + self._store({}) + + snapshot = RecipeVariable.objects.get(build=self.build, recipe=recipe) + self.assertEqual( + json.loads(zlib.decompress(snapshot.variables)), {'FOO': 'bar'}) + + def test_dependency_information_replaces_recipe_variables(self): + recipe = self._store({'recipe_variables': {'FOO': 'old'}}) + self._store({'recipe_variables': {'BAR': 'new'}}) + + snapshot = RecipeVariable.objects.get(build=self.build, recipe=recipe) + self.assertEqual( + json.loads(zlib.decompress(snapshot.variables)), {'BAR': 'new'}) + self.assertEqual(snapshot.variable_count, 1) + + def test_dependency_information_removes_stale_recipe_snapshot(self): + stale_recipe = Recipe.objects.create( + name='stale', version='', layer_version=self.layer_version, + file_path='stale.bb') + RecipeVariable.objects.create( + build=self.build, recipe=stale_recipe, + variables=zlib.compress(b'{"STALE":"value"}')) + + current_recipe = self._store({'recipe_variables': {'FOO': 'bar'}}) + + self.assertFalse(RecipeVariable.objects.filter( + build=self.build, recipe=stale_recipe).exists()) + self.assertTrue(RecipeVariable.objects.filter( + build=self.build, recipe=current_recipe).exists()) + + def test_dependency_information_rolls_back_failed_bulk_create(self): + recipe = self._store({'recipe_variables': {'FOO': 'old'}}) + with patch.object(RecipeVariable.objects, 'bulk_create', + side_effect=RuntimeError('injected failure')): + with self.assertRaisesRegex(RuntimeError, 'injected failure'): + self._store({'recipe_variables': {'FOO': 'new'}}) + + snapshots = RecipeVariable.objects.filter(build=self.build) + self.assertEqual(snapshots.count(), 1) + snapshot = snapshots.get(recipe=recipe) + self.assertEqual( + json.loads(zlib.decompress(snapshot.variables)), {'FOO': 'old'})