buildstats.py: Add tracking of network I/O per interface

From: Olga Denisova <denisova.olga.k@yandex.ru>

This patch extends SystemStats to collect and store data from /proc/net/dev.
It extracts per-interface received and transmitted bytes, calculates deltas
between samples, and stores them for further analysis.

Useful for identifying network bottlenecks during long-running builds.

Signed-off-by: denisova-ok <denisova.olga.k@yandex.ru>
---
 lib/oe/buildstats.py | 43 +++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 41 insertions(+), 2 deletions(-)

Can you include sample output for this into the commit message,
especially if it showcases a network bottleneck?

Alex

On Tue, 15 Apr 2025 at 18:20, denisova.olga.k via
lists.openembedded.org
<denisova.olga.k=yandex.ru@lists.openembedded.org> wrote:
>
> From: Olga Denisova <denisova.olga.k@yandex.ru>
>
> This patch extends SystemStats to collect and store data from /proc/net/dev.
> It extracts per-interface received and transmitted bytes, calculates deltas
> between samples, and stores them for further analysis.
>
> Useful for identifying network bottlenecks during long-running builds.
>
> Signed-off-by: denisova-ok <denisova.olga.k@yandex.ru>
> ---
>  lib/oe/buildstats.py | 43 +++++++++++++++++++++++++++++++++++++++++--
>  1 file changed, 41 insertions(+), 2 deletions(-)
>
> diff --git a/lib/oe/buildstats.py b/lib/oe/buildstats.py
> index 1ffe679801..6f5dad953c 100644
> --- a/lib/oe/buildstats.py
> +++ b/lib/oe/buildstats.py
> @@ -10,6 +10,7 @@
>  import time
>  import re
>  import bb.event
> +from collections import deque
>
>  class SystemStats:
>      def __init__(self, d):
> @@ -18,7 +19,8 @@ class SystemStats:
>          bb.utils.mkdirhier(bsdir)
>          file_handlers =  [('diskstats', self._reduce_diskstats),
>                              ('meminfo', self._reduce_meminfo),
> -                            ('stat', self._reduce_stat)]
> +                            ('stat', self._reduce_stat),
> +                            ('net/dev', self._reduce_net)]
>
>          # Some hosts like openSUSE have readable /proc/pressure files
>          # but throw errors when these files are opened. Catch these error
> @@ -47,7 +49,10 @@ class SystemStats:
>                  # not strictly necessary, but using it makes the class
>                  # more robust should two processes ever write
>                  # concurrently.
> -                destfile = os.path.join(bsdir, '%sproc_%s.log' % ('reduced_' if handler else '', filename))
> +                if filename == 'net/dev':
> +                    destfile = os.path.join(bsdir, 'reduced_proc_net.log')
> +                else:
> +                    destfile = os.path.join(bsdir, '%sproc_%s.log' % ('reduced_' if handler else '', filename))
>                  self.proc_files.append((filename, open(destfile, 'ab'), handler))
>          self.monitor_disk = open(os.path.join(bsdir, 'monitor_disk.log'), 'ab')
>          # Last time that we sampled /proc data resp. recorded disk monitoring data.
> @@ -72,6 +77,7 @@ class SystemStats:
>          self.stat_ltimes = None
>          # Last time we sampled /proc/pressure. All resources stored in a single dict with the key as filename
>          self.last_pressure = {"pressure/cpu": None, "pressure/io": None, "pressure/memory": None}
> +        self.net_stats = {}
>
>      def close(self):
>          self.monitor_disk.close()
> @@ -93,6 +99,39 @@ class SystemStats:
>                      b' '.join([values[x] for x in
>                                 (b'MemTotal', b'MemFree', b'Buffers', b'Cached', b'SwapTotal', b'SwapFree')]) + b'\n')
>
> +    def _reduce_net(self, time, data, filename):
> +        data = data.split(b'\n')
> +        for line in data[2:]:
> +            if b":" not in line:
> +                continue
> +            try:
> +                parts = line.split()
> +                iface = (parts[0].strip(b':')).decode('ascii')
> +                receive_bytes = int(parts[1])
> +                transmit_bytes = int(parts[9])
> +            except Exception:
> +                continue
> +
> +            if iface not in self.net_stats:
> +                self.net_stats[iface] = deque(maxlen=2)
> +                self.net_stats[iface].append((receive_bytes, transmit_bytes, 0, 0))
> +            prev = self.net_stats[iface][-1] if self.net_stats[iface] else (0, 0, 0, 0)
> +            receive_diff = receive_bytes - prev[0]
> +            transmit_diff = transmit_bytes - prev[1]
> +            self.net_stats[iface].append((
> +                receive_bytes,
> +                transmit_bytes,
> +                receive_diff,
> +                transmit_diff
> +            ))
> +
> +        result_str = "\n".join(
> +            f"{iface}: {net_data[-1][0]} {net_data[-1][1]} {net_data[-1][2]} {net_data[-1][3]}"
> +            for iface, net_data in self.net_stats.items()
> +        ) + "\n"
> +
> +        return time, result_str.encode('ascii')
> +
>      def _diskstats_is_relevant_line(self, linetokens):
>          if len(linetokens) != 14:
>              return False
> --
> 2.34.1
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#214955): https://lists.openembedded.org/g/openembedded-core/message/214955
> Mute This Topic: https://lists.openembedded.org/mt/112279424/1686489
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alex.kanavin@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>