@@ -23,7 +23,6 @@ from collections import defaultdict
from collections.abc import Mapping
import bb.utils
from bb import PrefixLoggerAdapter
-import re
import shutil
logger = logging.getLogger("BitBake.Cache")
@@ -404,7 +403,6 @@ class Cache(object):
self.data_fn = None
self.cacheclean = True
self.data_hash = data_hash
- self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
if self.cachedir in [None, '']:
bb.fatal("Please ensure CACHE is set to the cache directory for BitBake to use")
@@ -614,7 +612,7 @@ class Cache(object):
if not fl:
continue
# Have to be careful about spaces and colons in filenames
- flist = self.filelist_regex.split(fl)
+ flist = bb.utils.split_file_checksums(fl)
for f in flist:
if not f:
continue
@@ -11,13 +11,10 @@ import os
import stat
import bb.utils
import logging
-import re
from bb.cache import MultiProcessCache
logger = logging.getLogger("BitBake.Cache")
-filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
-
# mtime cache (non-persistent)
# based upon the assumption that files do not change during bitbake run
class FileMtimeCache(object):
@@ -112,7 +109,7 @@ class FileChecksumCache(MultiProcessCache):
return dirchecksums
checksums = []
- for pth in filelist_regex.split(filelist):
+ for pth in bb.utils.split_file_checksums(filelist):
if not pth:
continue
pth = pth.strip()
@@ -703,3 +703,29 @@ class FilemodeTests(unittest.TestCase):
with self.assertRaises(ValueError):
bb.utils.to_filemode("999")
+
+class SplitFileChecksums(unittest.TestCase):
+ def check(self, filelist, expected):
+ regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
+ self.assertEqual(regex.split(filelist), expected)
+ self.assertEqual(bb.utils.split_file_checksums(filelist), expected)
+
+ def test_simple(self):
+ self.check("/a:True /b:False", ["/a:True", "/b:False"])
+ self.check("/a:True", ["/a:True"])
+ self.check("", [""])
+ self.check(" ", [" "])
+
+ def test_whitespace(self):
+ self.check("/a:True\t/b:False\n/c:True /d:False", ["/a:True", "/b:False", "/c:True", "/d:False"])
+ self.check("/a:True\xa0/b:False\u3000/c:True", ["/a:True", "/b:False", "/c:True"])
+ self.check(" /a:True /b:False ", [" /a:True", "/b:False", ""])
+
+ def test_spaces_in_paths(self):
+ self.check("/a b/c:True /d e:False", ["/a b/c:True", "/d e:False"])
+ self.check("/a:True /b c:False /d", ["/a:True", "/b c:False", "/d"])
+
+ def test_suffix_lookalikes(self):
+ self.check("/a:Truex /b:False", ["/a:Truex /b:False"])
+ self.check(":True :False", [":True", ":False"])
+ self.check("/a:True:False /b:False", ["/a:True:False", "/b:False"])
@@ -2359,3 +2359,32 @@ def is_path_on_nfs(path):
import bb.process
fstype = bb.process.run(['stat', '-f', '-c', '%T', path])[0].strip()
return fstype == "nfs"
+
+_file_checksums_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
+
+def split_file_checksums(filelist):
+ """
+ Split a file-checksums value into its ``path:True`` and ``path:False``
+ entries at the whitespace following ``:True`` or ``:False``, since the
+ paths can contain whitespace.
+
+ Arguments:
+
+ - ``filelist``: the file-checksums string.
+
+ Returns the list of entries. When every whitespace-separated word ends
+ with ``:True`` or ``:False`` (the usual case), this is the same as
+ splitting on whitespace, which is much faster for long values than the
+ regular expression.
+ """
+ words = filelist.split()
+ if words and all(word.endswith((":True", ":False")) for word in words):
+ # Match the regular expression split exactly: leading whitespace stays
+ # with the first entry and trailing whitespace gives an empty entry
+ leading = len(filelist) - len(filelist.lstrip())
+ if leading:
+ words[0] = filelist[:leading] + words[0]
+ if filelist[-1].isspace():
+ words.append("")
+ return words
+ return _file_checksums_regex.split(filelist)
cacheValidUpdate() splits the file-checksums values of every recipe on every warm parse, with a regex matching the whitespace after ":True" or ":False" since paths can contain spaces. The values have an entry for every FILESPATH directory searched for each local file (229 KB for busybox with 18 layers), so the regex split is slow. Add bb.utils.split_file_checksums(), which uses str.split() when every word ends with ":True" or ":False", as happens when no path contains whitespace, and the regex otherwise. The result is identical to the regex split, as str.split() and \s agree on what is whitespace. A warm 'bitbake -p' with 18 layers goes from 8.24s to 6.92s, and with OE-core from 2.11s to 1.98s (median of 10 runs). Task signatures and 'bitbake -e' output are unchanged. AI-Generated: Uses Claude Code (Claude Opus 5.5) Signed-off-by: Ricardo Salveti <ricardo.salveti@oss.qualcomm.com> --- lib/bb/cache.py | 4 +--- lib/bb/checksum.py | 5 +---- lib/bb/tests/utils.py | 26 ++++++++++++++++++++++++++ lib/bb/utils.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 7 deletions(-)