diff mbox series

[v2] cache: Cache missing files in the mtime cache

Message ID 20260930083437.2242917-1-ecordonnier@snap.com
State Accepted, archived
Commit de0e096816e3d69309b42f9162846956952094b0
Headers show
Series [v2] cache: Cache missing files in the mtime cache | expand

Commit Message

Etienne Cordonnier Sept. 30, 2026, 8:34 a.m. UTC 
From: Etienne Cordonnier <ecordonnier@snap.com>

cacheValidUpdate() looks up file_depends entries with
bb.parse.cached_mtime_noerror() and file_checksums entries with
os.path.exists(). Neither caches a missing file, so the same non-existent
paths are stat()ed again on every lookup.

In a warm parse of 5,111 targets there are 585,773 file_checksums lookups
for 48,907 unique paths, and 300,648 file_depends lookups for only 545
missing files.

Record failed lookups in the mtime cache (as 0) and use it for the
file_checksums existence check as well. A warm 'bitbake -p' goes from 6.56s
to 5.82s (median of 12 runs).

AI-Generated: Uses Claude Code (Claude Sonnet 5.5)
Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
---
 lib/bb/cache.py          | 3 ++-
 lib/bb/parse/__init__.py | 8 ++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)
diff mbox series

Patch

diff --git a/lib/bb/cache.py b/lib/bb/cache.py
index 24bde62a9..d05d341f4 100644
--- a/lib/bb/cache.py
+++ b/lib/bb/cache.py
@@ -619,7 +619,8 @@  class Cache(object):
                     if not f:
                         continue
                     f, exist = f.rsplit(":", 1)
-                    if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
+                    file_exists = bb.parse.cached_mtime_noerror(f) != 0
+                    if (exist == "True" and not file_exists) or (exist == "False" and file_exists):
                         self.logger.debug2("%s's file checksum list file %s changed",
                                              fn, f)
                         self.remove(fn)
diff --git a/lib/bb/parse/__init__.py b/lib/bb/parse/__init__.py
index d428d8a4b..815ff1279 100644
--- a/lib/bb/parse/__init__.py
+++ b/lib/bb/parse/__init__.py
@@ -48,18 +48,22 @@  class SkipPackage(SkipRecipe):
 
 __mtime_cache = {}
 def cached_mtime(f):
-    if f not in __mtime_cache:
+    # A cached 0 is a failure recorded by cached_mtime_noerror(), stat again
+    # to raise the real error (or pick up the file if it has appeared since)
+    if __mtime_cache.get(f, 0) == 0:
         res = os.stat(f)
         __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino)
     return __mtime_cache[f]
 
 def cached_mtime_noerror(f):
+    # Failures are cached as 0 so that repeated lookups of a missing file
+    # don't hit the filesystem again
     if f not in __mtime_cache:
         try:
             res = os.stat(f)
             __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino)
         except OSError:
-            return 0
+            __mtime_cache[f] = 0
     return __mtime_cache[f]
 
 def check_mtime(f, mtime):