diff mbox series

[v3] bitbake-worker/cooker: Increase default pipe size

Message ID 20241121121249.2574880-1-richard.purdie@linuxfoundation.org
State Accepted, archived
Commit 69c14e46600ba5ae9703f67704ab2548875ae6d7
Headers show
Series [v3] bitbake-worker/cooker: Increase default pipe size | expand

Commit Message

Richard Purdie Nov. 21, 2024, 12:12 p.m. UTC 
The default pipe size is 64kb on builds, which can be inefficient
for larger log files from workers. Increase the pipe size to 512kb
since build systems have decent amounts of memory and this is a more
efficient way of batching the data.

Tweak the default read sizes to match the pipe size for efficiency.

Since the contstant is only present in python 3.10 onwards, add
some compatibility code.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 bin/bitbake-worker | 14 ++++++++++++--
 lib/bb/runqueue.py |  2 +-
 2 files changed, 13 insertions(+), 3 deletions(-)
diff mbox series

Patch

diff --git a/bin/bitbake-worker b/bin/bitbake-worker
index a8a9660c5f..88217182fb 100755
--- a/bin/bitbake-worker
+++ b/bin/bitbake-worker
@@ -21,9 +21,14 @@  import traceback
 import queue
 import shlex
 import subprocess
+import fcntl
 from multiprocessing import Lock
 from threading import Thread
 
+# Remove when we have a minimum of python 3.10
+if not hasattr(fcntl, 'F_SETPIPE_SZ'):
+    fcntl.F_SETPIPE_SZ = 1031
+
 bb.utils.check_system_locale()
 
 # Users shouldn't be running this code directly
@@ -44,7 +49,6 @@  if sys.argv[1].startswith("decafbadbad"):
 # updates to log files for use with tail
 try:
     if sys.stdout.name == '<stdout>':
-        import fcntl
         fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
         fl |= os.O_SYNC 
         fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
@@ -56,6 +60,12 @@  logger = logging.getLogger("BitBake")
 
 worker_pipe = sys.stdout.fileno()
 bb.utils.nonblockingfd(worker_pipe)
+# Try to make the pipe buffers larger as it is much more efficient. If we can't
+# e.g. out of buffer space (/proc/sys/fs/pipe-user-pages-soft) then just pass over.
+try:
+    fcntl.fcntl(worker_pipe, fcntl.F_SETPIPE_SZ, 512 * 1024)
+except:
+    pass
 # Need to guard against multiprocessing being used in child processes
 # and multiple processes trying to write to the parent at the same time
 worker_pipe_lock = None
@@ -357,7 +367,7 @@  class runQueueWorkerPipe():
     def read(self):
         start = len(self.queue)
         try:
-            self.queue.extend(self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(512*1024) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 1b5b58f352..61608ac603 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -3318,7 +3318,7 @@  class runQueuePipe():
 
         start = len(self.queue)
         try:
-            self.queue.extend(self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(512 * 1024) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise