diff mbox series

[bitbake-devel] runqueue.py: make sure we use bb multiprocessing

Message ID 20260303165657.1803606-1-tgamblin@baylibre.com
State New
Headers show
Series [bitbake-devel] runqueue.py: make sure we use bb multiprocessing | expand

Commit Message

Trevor Gamblin March 3, 2026, 4:56 p.m. UTC 
Otherwise, we see errors like:

|ERROR: An uncaught exception occurred in runqueue
|Traceback (most recent call last):
|  File "/srv/pokybuild/yocto-worker/check-layer/build/layers/bitbake/lib/bb/runqueue.py", line 1663, in execute_runqueue
|    return self._execute_runqueue()
|           ~~~~~~~~~~~~~~~~~~~~~~^^
|  File "/srv/pokybuild/yocto-worker/check-layer/build/layers/bitbake/lib/bb/runqueue.py", line 1607, in _execute_runqueue
|    retval = self.dump_signatures(dumpsigs)
|  File "/srv/pokybuild/yocto-worker/check-layer/build/layers/bitbake/lib/bb/runqueue.py", line 1718, in dump_signatures
|    p.start()
|    ~~~~~~~^^
|  File "/usr/lib64/python3.14/multiprocessing/process.py", line 121, in start
|    self._popen = self._Popen(self)
|                  ~~~~~~~~~~~^^^^^^
|  File "/usr/lib64/python3.14/multiprocessing/context.py", line 224, in _Popen
|    return _default_context.get_context().Process._Popen(process_obj)
|           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
|  File "/usr/lib64/python3.14/multiprocessing/context.py", line 300, in _Popen
|    return Popen(process_obj)
|  File "/usr/lib64/python3.14/multiprocessing/popen_forkserver.py", line 35, in __init__
|    super().__init__(process_obj)
|    ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
|  File "/usr/lib64/python3.14/multiprocessing/popen_fork.py", line 20, in __init__
|    self._launch(process_obj)
|    ~~~~~~~~~~~~^^^^^^^^^^^^^
|  File "/usr/lib64/python3.14/multiprocessing/popen_forkserver.py", line 47, in _launch
|    reduction.dump(process_obj, buf)
|    ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|  File "/usr/lib64/python3.14/multiprocessing/reduction.py", line 60, in dump
|    ForkingPickler(file, protocol).dump(obj)
|    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^
|_pickle.PicklingError: Can't pickle local object <function CoreRecipeInfo.init_cacheData.<locals>.<lambda> at 0x7f7925667d70>
|when serializing tuple item 0
|when serializing collections.defaultdict reconstructor arguments
|when serializing collections.defaultdict object
|when serializing dict item 'rundeps'
|when serializing bb.cache.CacheData state
|when serializing bb.cache.CacheData object
|when serializing dict item ''
|when serializing dict item 'recipecaches'
|when serializing bb.cooker.BBCooker state
|when serializing bb.cooker.BBCooker object
|when serializing dict item 'cooker'
|when serializing bb.runqueue.RunQueue state
|when serializing bb.runqueue.RunQueue object
|when serializing tuple item 0
|when serializing method reconstructor arguments
|when serializing method object
|when serializing dict item '_target'
|when serializing multiprocessing.context.Process state
|when serializing multiprocessing.context.Process object

Fixes: [YOCTO #16184]

Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com>
---
 lib/bb/runqueue.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

Comments

Peter Kjellerstedt March 3, 2026, 9:49 p.m. UTC | #1 
> -----Original Message-----
> From: bitbake-devel@lists.openembedded.org <bitbake-devel@lists.openembedded.org> On Behalf Of Trevor Gamblin
> Sent: den 3 mars 2026 17:57
> To: bitbake-devel@lists.openembedded.org
> Subject: [bitbake-devel][PATCH] runqueue.py: make sure we use bb multiprocessing
> 
> Otherwise, we see errors like:
> 
> |ERROR: An uncaught exception occurred in runqueue
> |Traceback (most recent call last):
> |  File "/srv/pokybuild/yocto-worker/check-layer/build/layers/bitbake/lib/bb/runqueue.py", line 1663, in execute_runqueue
> |    return self._execute_runqueue()
> |           ~~~~~~~~~~~~~~~~~~~~~~^^
> |  File "/srv/pokybuild/yocto-worker/check-layer/build/layers/bitbake/lib/bb/runqueue.py", line 1607, in _execute_runqueue
> |    retval = self.dump_signatures(dumpsigs)
> |  File "/srv/pokybuild/yocto-worker/check-layer/build/layers/bitbake/lib/bb/runqueue.py", line 1718, in dump_signatures
> |    p.start()
> |    ~~~~~~~^^
> |  File "/usr/lib64/python3.14/multiprocessing/process.py", line 121, in start
> |    self._popen = self._Popen(self)
> |                  ~~~~~~~~~~~^^^^^^
> |  File "/usr/lib64/python3.14/multiprocessing/context.py", line 224, in _Popen
> |    return _default_context.get_context().Process._Popen(process_obj)
> |           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
> |  File "/usr/lib64/python3.14/multiprocessing/context.py", line 300, in _Popen
> |    return Popen(process_obj)
> |  File "/usr/lib64/python3.14/multiprocessing/popen_forkserver.py", line 35, in __init__
> |    super().__init__(process_obj)
> |    ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
> |  File "/usr/lib64/python3.14/multiprocessing/popen_fork.py", line 20, in __init__
> |    self._launch(process_obj)
> |    ~~~~~~~~~~~~^^^^^^^^^^^^^
> |  File "/usr/lib64/python3.14/multiprocessing/popen_forkserver.py", line 47, in _launch
> |    reduction.dump(process_obj, buf)
> |    ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
> |  File "/usr/lib64/python3.14/multiprocessing/reduction.py", line 60, in dump
> |    ForkingPickler(file, protocol).dump(obj)
> |    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^
> |_pickle.PicklingError: Can't pickle local object <function CoreRecipeInfo.init_cacheData.<locals>.<lambda> at 0x7f7925667d70>
> |when serializing tuple item 0
> |when serializing collections.defaultdict reconstructor arguments
> |when serializing collections.defaultdict object
> |when serializing dict item 'rundeps'
> |when serializing bb.cache.CacheData state
> |when serializing bb.cache.CacheData object
> |when serializing dict item ''
> |when serializing dict item 'recipecaches'
> |when serializing bb.cooker.BBCooker state
> |when serializing bb.cooker.BBCooker object
> |when serializing dict item 'cooker'
> |when serializing bb.runqueue.RunQueue state
> |when serializing bb.runqueue.RunQueue object
> |when serializing tuple item 0
> |when serializing method reconstructor arguments
> |when serializing method object
> |when serializing dict item '_target'
> |when serializing multiprocessing.context.Process state
> |when serializing multiprocessing.context.Process object
> 
> Fixes: [YOCTO #16184]
> 
> Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com>
> ---
>  lib/bb/runqueue.py | 4 +++-
>  1 file changed, 3 insertions(+), 1 deletion(-)
> 
> diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
> index a880a0d54..27da5efaa 100644
> --- a/lib/bb/runqueue.py
> +++ b/lib/bb/runqueue.py
> @@ -21,13 +21,15 @@ import re
>  import bb
>  from bb import msg, event
>  from bb import monitordisk
> +from bb import multiprocessing

Is this really needed? You use bb.multiprocessing below, 
not multiprocessing.

>  import subprocess
>  import pickle
> -from multiprocessing import Process
>  import shlex
>  import pprint
>  import time
> 
> +Process = bb.multiprocessing.Process
> +
>  bblogger = logging.getLogger("BitBake")
>  logger = logging.getLogger("BitBake.RunQueue")
>  hashequiv_logger = logging.getLogger("BitBake.RunQueue.HashEquiv")
> --
> 2.53.0

//Peter
Trevor Gamblin March 4, 2026, 1:27 p.m. UTC | #2 
On 2026-03-03 16:49, Peter Kjellerstedt wrote:
>> -----Original Message-----
>> From: bitbake-devel@lists.openembedded.org <bitbake-devel@lists.openembedded.org> On Behalf Of Trevor Gamblin
>> Sent: den 3 mars 2026 17:57
>> To: bitbake-devel@lists.openembedded.org
>> Subject: [bitbake-devel][PATCH] runqueue.py: make sure we use bb multiprocessing
>>
>> Otherwise, we see errors like:
>>
>> |ERROR: An uncaught exception occurred in runqueue
>> |Traceback (most recent call last):
>> |  File "/srv/pokybuild/yocto-worker/check-layer/build/layers/bitbake/lib/bb/runqueue.py", line 1663, in execute_runqueue
>> |    return self._execute_runqueue()
>> |           ~~~~~~~~~~~~~~~~~~~~~~^^
>> |  File "/srv/pokybuild/yocto-worker/check-layer/build/layers/bitbake/lib/bb/runqueue.py", line 1607, in _execute_runqueue
>> |    retval = self.dump_signatures(dumpsigs)
>> |  File "/srv/pokybuild/yocto-worker/check-layer/build/layers/bitbake/lib/bb/runqueue.py", line 1718, in dump_signatures
>> |    p.start()
>> |    ~~~~~~~^^
>> |  File "/usr/lib64/python3.14/multiprocessing/process.py", line 121, in start
>> |    self._popen = self._Popen(self)
>> |                  ~~~~~~~~~~~^^^^^^
>> |  File "/usr/lib64/python3.14/multiprocessing/context.py", line 224, in _Popen
>> |    return _default_context.get_context().Process._Popen(process_obj)
>> |           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
>> |  File "/usr/lib64/python3.14/multiprocessing/context.py", line 300, in _Popen
>> |    return Popen(process_obj)
>> |  File "/usr/lib64/python3.14/multiprocessing/popen_forkserver.py", line 35, in __init__
>> |    super().__init__(process_obj)
>> |    ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
>> |  File "/usr/lib64/python3.14/multiprocessing/popen_fork.py", line 20, in __init__
>> |    self._launch(process_obj)
>> |    ~~~~~~~~~~~~^^^^^^^^^^^^^
>> |  File "/usr/lib64/python3.14/multiprocessing/popen_forkserver.py", line 47, in _launch
>> |    reduction.dump(process_obj, buf)
>> |    ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
>> |  File "/usr/lib64/python3.14/multiprocessing/reduction.py", line 60, in dump
>> |    ForkingPickler(file, protocol).dump(obj)
>> |    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^
>> |_pickle.PicklingError: Can't pickle local object <function CoreRecipeInfo.init_cacheData.<locals>.<lambda> at 0x7f7925667d70>
>> |when serializing tuple item 0
>> |when serializing collections.defaultdict reconstructor arguments
>> |when serializing collections.defaultdict object
>> |when serializing dict item 'rundeps'
>> |when serializing bb.cache.CacheData state
>> |when serializing bb.cache.CacheData object
>> |when serializing dict item ''
>> |when serializing dict item 'recipecaches'
>> |when serializing bb.cooker.BBCooker state
>> |when serializing bb.cooker.BBCooker object
>> |when serializing dict item 'cooker'
>> |when serializing bb.runqueue.RunQueue state
>> |when serializing bb.runqueue.RunQueue object
>> |when serializing tuple item 0
>> |when serializing method reconstructor arguments
>> |when serializing method object
>> |when serializing dict item '_target'
>> |when serializing multiprocessing.context.Process state
>> |when serializing multiprocessing.context.Process object
>>
>> Fixes: [YOCTO #16184]
>>
>> Signed-off-by: Trevor Gamblin <tgamblin@baylibre.com>
>> ---
>>   lib/bb/runqueue.py | 4 +++-
>>   1 file changed, 3 insertions(+), 1 deletion(-)
>>
>> diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
>> index a880a0d54..27da5efaa 100644
>> --- a/lib/bb/runqueue.py
>> +++ b/lib/bb/runqueue.py
>> @@ -21,13 +21,15 @@ import re
>>   import bb
>>   from bb import msg, event
>>   from bb import monitordisk
>> +from bb import multiprocessing
> Is this really needed? You use bb.multiprocessing below,
> not multiprocessing.
No, it isn't - looks like I committed extra. I'll send a v2.
>
>>   import subprocess
>>   import pickle
>> -from multiprocessing import Process
>>   import shlex
>>   import pprint
>>   import time
>>
>> +Process = bb.multiprocessing.Process
>> +
>>   bblogger = logging.getLogger("BitBake")
>>   logger = logging.getLogger("BitBake.RunQueue")
>>   hashequiv_logger = logging.getLogger("BitBake.RunQueue.HashEquiv")
>> --
>> 2.53.0
> //Peter
>
diff mbox series

Patch

diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index a880a0d54..27da5efaa 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -21,13 +21,15 @@  import re
 import bb
 from bb import msg, event
 from bb import monitordisk
+from bb import multiprocessing
 import subprocess
 import pickle
-from multiprocessing import Process
 import shlex
 import pprint
 import time
 
+Process = bb.multiprocessing.Process
+
 bblogger = logging.getLogger("BitBake")
 logger = logging.getLogger("BitBake.RunQueue")
 hashequiv_logger = logging.getLogger("BitBake.RunQueue.HashEquiv")