| Message ID | 20250718135437.247096-1-richard.purdie@linuxfoundation.org |
|---|---|
| State | Accepted, archived |
| Commit | ee5fce67ce35b025c68aa61e2e758903269ee346 |
| Headers | show |
| Series | [1/2] utils: Optimise signal/sigmask performance | expand |
LGTM Reviewed-by: Joshua Watt <JPEWhacker@gmail.com> On Fri, Jul 18, 2025 at 7:54 AM Richard Purdie via lists.openembedded.org <richard.purdie=linuxfoundation.org@lists.openembedded.org> wrote: > > Running "time bitbake -pP idle" with a valid cache shows around 800,000 > calls to enum creation from python's signal.py. We don't care about this > overhead and it adversely affects cache load time quite badly. > > Try and use _signal directly, falling back to signal, which avoids > this overhead we don't need and makes cache loading much faster. > > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> > --- > lib/bb/utils.py | 17 +++++++++++++---- > 1 file changed, 13 insertions(+), 4 deletions(-) > > diff --git a/lib/bb/utils.py b/lib/bb/utils.py > index f688f7dd687..016036dbce4 100644 > --- a/lib/bb/utils.py > +++ b/lib/bb/utils.py > @@ -2226,6 +2226,15 @@ def path_is_descendant(descendant, ancestor): > > return False > > +# Recomputing the sets in signal.py is expensive (bitbake -pP idle) > +# so try and use _signal directly to avoid it > +valid_signals = signal.valid_signals() > +try: > + import _signal > + sigmask = _signal.pthread_sigmask > +except ImportError: > + sigmask = signal.pthread_sigmask > + > # If we don't have a timeout of some kind and a process/thread exits badly (for example > # OOM killed) and held a lock, we'd just hang in the lock futex forever. It is better > # we exit at some point than hang. 5 minutes with no progress means we're probably deadlocked. > @@ -2235,7 +2244,7 @@ def path_is_descendant(descendant, ancestor): > @contextmanager > def lock_timeout(lock): > try: > - s = signal.pthread_sigmask(signal.SIG_BLOCK, signal.valid_signals()) > + s = sigmask(signal.SIG_BLOCK, valid_signals) > held = lock.acquire(timeout=5*60) > if not held: > bb.server.process.serverlog("Couldn't get the lock for 5 mins, timed out, exiting.\n%s" % traceback.format_stack()) > @@ -2243,17 +2252,17 @@ def lock_timeout(lock): > yield held > finally: > lock.release() > - signal.pthread_sigmask(signal.SIG_SETMASK, s) > + sigmask(signal.SIG_SETMASK, s) > > # A version of lock_timeout without the check that the lock was locked and a shorter timeout > @contextmanager > def lock_timeout_nocheck(lock): > l = False > try: > - s = signal.pthread_sigmask(signal.SIG_BLOCK, signal.valid_signals()) > + s = sigmask(signal.SIG_BLOCK, valid_signals) > l = lock.acquire(timeout=10) > yield l > finally: > if l: > lock.release() > - signal.pthread_sigmask(signal.SIG_SETMASK, s) > + sigmask(signal.SIG_SETMASK, s) > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#17794): https://lists.openembedded.org/g/bitbake-devel/message/17794 > Mute This Topic: https://lists.openembedded.org/mt/114221231/3616693 > Group Owner: bitbake-devel+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [JPEWhacker@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- >
diff --git a/lib/bb/utils.py b/lib/bb/utils.py index f688f7dd687..016036dbce4 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -2226,6 +2226,15 @@ def path_is_descendant(descendant, ancestor): return False +# Recomputing the sets in signal.py is expensive (bitbake -pP idle) +# so try and use _signal directly to avoid it +valid_signals = signal.valid_signals() +try: + import _signal + sigmask = _signal.pthread_sigmask +except ImportError: + sigmask = signal.pthread_sigmask + # If we don't have a timeout of some kind and a process/thread exits badly (for example # OOM killed) and held a lock, we'd just hang in the lock futex forever. It is better # we exit at some point than hang. 5 minutes with no progress means we're probably deadlocked. @@ -2235,7 +2244,7 @@ def path_is_descendant(descendant, ancestor): @contextmanager def lock_timeout(lock): try: - s = signal.pthread_sigmask(signal.SIG_BLOCK, signal.valid_signals()) + s = sigmask(signal.SIG_BLOCK, valid_signals) held = lock.acquire(timeout=5*60) if not held: bb.server.process.serverlog("Couldn't get the lock for 5 mins, timed out, exiting.\n%s" % traceback.format_stack()) @@ -2243,17 +2252,17 @@ def lock_timeout(lock): yield held finally: lock.release() - signal.pthread_sigmask(signal.SIG_SETMASK, s) + sigmask(signal.SIG_SETMASK, s) # A version of lock_timeout without the check that the lock was locked and a shorter timeout @contextmanager def lock_timeout_nocheck(lock): l = False try: - s = signal.pthread_sigmask(signal.SIG_BLOCK, signal.valid_signals()) + s = sigmask(signal.SIG_BLOCK, valid_signals) l = lock.acquire(timeout=10) yield l finally: if l: lock.release() - signal.pthread_sigmask(signal.SIG_SETMASK, s) + sigmask(signal.SIG_SETMASK, s)
Running "time bitbake -pP idle" with a valid cache shows around 800,000 calls to enum creation from python's signal.py. We don't care about this overhead and it adversely affects cache load time quite badly. Try and use _signal directly, falling back to signal, which avoids this overhead we don't need and makes cache loading much faster. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> --- lib/bb/utils.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-)