[bitbake-devel,v3] avoid some unnecessary copies

From: Etienne Cordonnier <ecordonnier@snap.com>

declaring queue=b"" creates an object of types bytes().
bytes() is an immutable object, and therefore doing "self.queue = self.queue + r"
creates a new object containing "self.queue" concatenated with "r".

On my test setup, we are passing 180MB of data of "workerdata" to the bitbake-worker,
so those copies significantly slow down the initialization of the bitbake-worker.

Rather use bytearray() which a mutable type, and use extend() to avoid copies.
In my test setup, byterray.extend() is 10.000 times faster than copying the queue,
for a queue size of 180MB.

Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
---
 bin/bitbake-worker | 14 +++++++-------
 lib/bb/runqueue.py |  4 ++--
 2 files changed, 9 insertions(+), 9 deletions(-)

When sending iterations, please add a note explaining what the changes are.

That said, solid improvement, well spotted!

Ross

> On 20 Sep 2023, at 08:41, Etienne Cordonnier via lists.openembedded.org <ecordonnier=snap.com@lists.openembedded.org> wrote:
> 
> From: Etienne Cordonnier <ecordonnier@snap.com>
> 
> declaring queue=b"" creates an object of types bytes().
> bytes() is an immutable object, and therefore doing "self.queue = self.queue + r"
> creates a new object containing "self.queue" concatenated with "r".
> 
> On my test setup, we are passing 180MB of data of "workerdata" to the bitbake-worker,
> so those copies significantly slow down the initialization of the bitbake-worker.
> 
> Rather use bytearray() which a mutable type, and use extend() to avoid copies.
> In my test setup, byterray.extend() is 10.000 times faster than copying the queue,
> for a queue size of 180MB.
> 
> Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
> ---
> bin/bitbake-worker | 14 +++++++-------
> lib/bb/runqueue.py |  4 ++--
> 2 files changed, 9 insertions(+), 9 deletions(-)
> 
> diff --git a/bin/bitbake-worker b/bin/bitbake-worker
> index 451e6926..609e276f 100755
> --- a/bin/bitbake-worker
> +++ b/bin/bitbake-worker
> @@ -91,19 +91,19 @@ def worker_fire_prepickled(event):
> worker_thread_exit = False
> 
> def worker_flush(worker_queue):
> -    worker_queue_int = b""
> +    worker_queue_int = bytearray()
>     global worker_pipe, worker_thread_exit
> 
>     while True:
>         try:
> -            worker_queue_int = worker_queue_int + worker_queue.get(True, 1)
> +            worker_queue_int.extend(worker_queue.get(True, 1))
>         except queue.Empty:
>             pass
>         while (worker_queue_int or not worker_queue.empty()):
>             try:
>                 (_, ready, _) = select.select([], [worker_pipe], [], 1)
>                 if not worker_queue.empty():
> -                    worker_queue_int = worker_queue_int + worker_queue.get()
> +                    worker_queue_int.extend(worker_queue.get())
>                 written = os.write(worker_pipe, worker_queue_int)
>                 worker_queue_int = worker_queue_int[written:]
>             except (IOError, OSError) as e:
> @@ -346,12 +346,12 @@ class runQueueWorkerPipe():
>         if pipeout:
>             pipeout.close()
>         bb.utils.nonblockingfd(self.input)
> -        self.queue = b""
> +        self.queue = bytearray()
> 
>     def read(self):
>         start = len(self.queue)
>         try:
> -            self.queue = self.queue + (self.input.read(102400) or b"")
> +            self.queue.extend(self.input.read(102400) or b"")
>         except (OSError, IOError) as e:
>             if e.errno != errno.EAGAIN:
>                 raise
> @@ -379,7 +379,7 @@ class BitbakeWorker(object):
>     def __init__(self, din):
>         self.input = din
>         bb.utils.nonblockingfd(self.input)
> -        self.queue = b""
> +        self.queue = bytearray()
>         self.cookercfg = None
>         self.databuilder = None
>         self.data = None
> @@ -413,7 +413,7 @@ class BitbakeWorker(object):
>                     if len(r) == 0:
>                         # EOF on pipe, server must have terminated
>                         self.sigterm_exception(signal.SIGTERM, None)
> -                    self.queue = self.queue + r
> +                    self.queue.extend(r)
>                 except (OSError, IOError):
>                     pass
>             if len(self.queue):
> diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
> index c88d7129..c40a3be2 100644
> --- a/lib/bb/runqueue.py
> +++ b/lib/bb/runqueue.py
> @@ -3159,7 +3159,7 @@ class runQueuePipe():
>         if pipeout:
>             pipeout.close()
>         bb.utils.nonblockingfd(self.input)
> -        self.queue = b""
> +        self.queue = bytearray()
>         self.d = d
>         self.rq = rq
>         self.rqexec = rqexec
> @@ -3178,7 +3178,7 @@ class runQueuePipe():
> 
>         start = len(self.queue)
>         try:
> -            self.queue = self.queue + (self.input.read(102400) or b"")
> +            self.queue.extend(self.input.read(102400) or b"")
>         except (OSError, IOError) as e:
>             if e.errno != errno.EAGAIN:
>                 raise
> -- 
> 2.36.1.vfs.0.0
> 
> 
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#15088): https://lists.openembedded.org/g/bitbake-devel/message/15088
> Mute This Topic: https://lists.openembedded.org/mt/101474114/6875888
> Group Owner: bitbake-devel+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [ross.burton@arm.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>