[2/4] cookerdata: Separate out data_hash and hook to tinfoil

Within tinfoil, the user can write to the configuration data but it won't
cause the data_hash checksum to be re-written, meaning cached parsing
data can be reused when it would now be incorrect.

Abstract out the data_hash code and add it to the invalidateCaches
command, called by tinfoil.modified_files() meaning that tinfoil can
instruct bitbake to update the caches and re-parse if necessary.

Also move the data_hash entirely into databuilder and drop the copy
in cooker as obsolete and not needed.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/cooker.py     | 10 +++++++---
 lib/bb/cookerdata.py | 13 +++++++++----
 2 files changed, 16 insertions(+), 7 deletions(-)

> -----Original Message-----
> From: bitbake-devel@lists.openembedded.org <bitbake-devel@lists.openembedded.org> On Behalf Of Richard Purdie
> Sent: den 12 augusti 2024 17:54
> To: bitbake-devel@lists.openembedded.org
> Subject: [bitbake-devel] [PATCH 2/4] cookerdata: Separate out data_hash and hook to tinfoil
> 
> Within tinfoil, the user can write to the configuration data but it won't
> cause the data_hash checksum to be re-written, meaning cached parsing
> data can be reused when it would now be incorrect.
> 
> Abstract out the data_hash code and add it to the invalidateCaches
> command, called by tinfoil.modified_files() meaning that tinfoil can
> instruct bitbake to update the caches and re-parse if necessary.
> 
> Also move the data_hash entirely into databuilder and drop the copy
> in cooker as obsolete and not needed.
> 
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  lib/bb/cooker.py     | 10 +++++++---
>  lib/bb/cookerdata.py | 13 +++++++++----
>  2 files changed, 16 insertions(+), 7 deletions(-)
> 
> diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
> index 6754f986bf..31fee2bead 100644
> --- a/lib/bb/cooker.py
> +++ b/lib/bb/cooker.py
> @@ -281,7 +281,6 @@ class BBCooker:
>          self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False)
>          self.databuilder.parseBaseConfiguration()
>          self.data = self.databuilder.data
> -        self.data_hash = self.databuilder.data_hash
>          self.extraconfigdata = {}
> 
>          eventlog = self.data.getVar("BB_DEFAULT_EVENTLOG")
> @@ -370,6 +369,11 @@ class BBCooker:
>          if not clean:
>              bb.parse.BBHandler.cached_statements = {}
> 
> +        # If writes were made to any of the data stores, we need to recalcuate the data

Typo: recalcuate -> recalculate

> +        # store cache
> +        if hasattr(self, "databuilder"):
> +            self.databuilder.calc_datastore_hashes()
> +
>      def parseConfiguration(self):
>          self.updateCacheSync()
> 
> @@ -1338,7 +1342,7 @@ class BBCooker:
>          self.buildSetVars()
>          self.reset_mtime_caches()
> 
> -        bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
> +        bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.databuilder.data_hash, self.caches_array)
> 
>          layername = self.collections[mc].calc_bbfile_priority(fn)[2]
>          infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn), layername)
> @@ -2112,7 +2116,7 @@ class CookerParser(object):
>          self.mcfilelist = mcfilelist
>          self.cooker = cooker
>          self.cfgdata = cooker.data
> -        self.cfghash = cooker.data_hash
> +        self.cfghash = cooker.databuilder.data_hash
>          self.cfgbuilder = cooker.databuilder
> 
>          # Accounting statistics
> diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py
> index 0649e40995..3ad5cf3dd0 100644
> --- a/lib/bb/cookerdata.py
> +++ b/lib/bb/cookerdata.py
> @@ -254,9 +254,16 @@ class CookerDataBuilder(object):
>          self.data = self.basedata
>          self.mcdata = {}
> 
> +    def calc_datastore_hashes(self):
> +        data_hash = hashlib.sha256()
> +        data_hash.update(self.data.get_hash().encode('utf-8'))
> +        multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
> +        for config in multiconfig:
> +            data_hash.update(self.mcdata[config].get_hash().encode('utf-8'))
> +        self.data_hash = data_hash.hexdigest()
> +
>      def parseBaseConfiguration(self, worker=False):
>          mcdata = {}
> -        data_hash = hashlib.sha256()
>          try:
>              self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
> 
> @@ -279,7 +286,6 @@ class CookerDataBuilder(object):
>                  bb.event.fire(bb.event.ConfigParsed(), self.data)
> 
>              bb.parse.init_parser(self.data)
> -            data_hash.update(self.data.get_hash().encode('utf-8'))
>              mcdata[''] = self.data
> 
>              multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
> @@ -289,11 +295,9 @@ class CookerDataBuilder(object):
>                  parsed_mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
>                  bb.event.fire(bb.event.ConfigParsed(), parsed_mcdata)
>                  mcdata[config] = parsed_mcdata
> -                data_hash.update(parsed_mcdata.get_hash().encode('utf-8'))
>              if multiconfig:
>                  bb.event.fire(bb.event.MultiConfigParsed(mcdata), self.data)
> 
> -            self.data_hash = data_hash.hexdigest()
>          except bb.data_smart.ExpansionError as e:
>              logger.error(str(e))
>              raise bb.BBHandledException()
> @@ -328,6 +332,7 @@ class CookerDataBuilder(object):
>          for mc in mcdata:
>              self.mcdata[mc] = bb.data.createCopy(mcdata[mc])
>          self.data = self.mcdata['']
> +        self.calc_datastore_hashes()
> 
>      def reset(self):
>          # We may not have run parseBaseConfiguration() yet

//Peter