diff mbox series

[2/2] bitbake-worker/cooker: Increase default pipe size

Message ID 20241120104311.2365179-2-richard.purdie@linuxfoundation.org
State Accepted, archived
Commit 69c14e46600ba5ae9703f67704ab2548875ae6d7
Headers show
Series [1/2] bitbake-worker: Improve bytearray truncation performance | expand

Commit Message

Richard Purdie Nov. 20, 2024, 10:43 a.m. UTC
The default pipe size is 64kb on builds, which can be inefficient
for larger log files from workers. Increase the pipe size to 512kb
since build systems have decent amounts of memory and this is a more
efficient way of batching the data.

Tweak the default read sizes to match the pipe size for efficiency.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 bin/bitbake-worker | 4 +++-
 lib/bb/runqueue.py | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)
diff mbox series

Patch

diff --git a/bin/bitbake-worker b/bin/bitbake-worker
index a8a9660c5f..af4fb0ed1b 100755
--- a/bin/bitbake-worker
+++ b/bin/bitbake-worker
@@ -21,6 +21,7 @@  import traceback
 import queue
 import shlex
 import subprocess
+import fcntl
 from multiprocessing import Lock
 from threading import Thread
 
@@ -56,6 +57,7 @@  logger = logging.getLogger("BitBake")
 
 worker_pipe = sys.stdout.fileno()
 bb.utils.nonblockingfd(worker_pipe)
+fcntl.fcntl(worker_pipe, fcntl.F_SETPIPE_SZ, 512 * 1024)
 # Need to guard against multiprocessing being used in child processes
 # and multiple processes trying to write to the parent at the same time
 worker_pipe_lock = None
@@ -357,7 +359,7 @@  class runQueueWorkerPipe():
     def read(self):
         start = len(self.queue)
         try:
-            self.queue.extend(self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(512*1024) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 1b5b58f352..61608ac603 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -3318,7 +3318,7 @@  class runQueuePipe():
 
         start = len(self.queue)
         try:
-            self.queue.extend(self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(512 * 1024) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise