pybootchartgui: visualize /proc/net/dev network stats in graphs

From: Olga Denisova <denisova.olga.k@yandex.ru>

This patch adds support for parsing and visualizing network interface statistics from /proc/net/dev in pybootchartgui. It introduces a new NetSample class to hold per-interface metrics, including received/transmitted bytes and their deltas over time.

The data is drawn using line and box charts in draw.py and helps to monitor
network usage during the boot process for each interface individually.

Signed-off-by: denisova-ok <denisova.olga.k@yandex.ru>
---
 pybootchartgui/pybootchartgui/draw.py    | 48 ++++++++++++++++++++++++
 pybootchartgui/pybootchartgui/parsing.py | 18 +++++++++
 pybootchartgui/pybootchartgui/samples.py | 10 +++++
 3 files changed, 76 insertions(+)

Also here it would be good to see a demo, perhaps you can publish an
image somewhere and link to it?

Alex

On Tue, 15 Apr 2025 at 20:37, denisova.olga.k via
lists.openembedded.org
<denisova.olga.k=yandex.ru@lists.openembedded.org> wrote:
>
> From: Olga Denisova <denisova.olga.k@yandex.ru>
>
> This patch adds support for parsing and visualizing network interface statistics from /proc/net/dev in pybootchartgui. It introduces a new NetSample class to hold per-interface metrics, including received/transmitted bytes and their deltas over time.
>
> The data is drawn using line and box charts in draw.py and helps to monitor
> network usage during the boot process for each interface individually.
>
> Signed-off-by: denisova-ok <denisova.olga.k@yandex.ru>
> ---
>  pybootchartgui/pybootchartgui/draw.py    | 48 ++++++++++++++++++++++++
>  pybootchartgui/pybootchartgui/parsing.py | 18 +++++++++
>  pybootchartgui/pybootchartgui/samples.py | 10 +++++
>  3 files changed, 76 insertions(+)
>
> diff --git a/pybootchartgui/pybootchartgui/draw.py b/pybootchartgui/pybootchartgui/draw.py
> index c6e67833ab..16739a0fa1 100644
> --- a/pybootchartgui/pybootchartgui/draw.py
> +++ b/pybootchartgui/pybootchartgui/draw.py
> @@ -69,6 +69,11 @@ CPU_COLOR = (0.40, 0.55, 0.70, 1.0)
>  IO_COLOR = (0.76, 0.48, 0.48, 0.5)
>  # Disk throughput color.
>  DISK_TPUT_COLOR = (0.20, 0.71, 0.20, 1.0)
> +
> +BYTES_RECEIVED_COLOR = (0.0, 0.0, 1.0, 1.0)
> +BYTES_TRANSMITTED_COLOR = (1.0, 0.0, 0.0, 1.0)
> +BYTES_RECEIVE_DIFF_COLOR = (0.0, 0.0, 1.0, 0.3)
> +BYTES_TRANSMIT_DIFF_COLOR = (1.0, 0.0, 0.0, 0.3)
>  # CPU load chart color.
>  FILE_OPEN_COLOR = (0.20, 0.71, 0.71, 1.0)
>  # Mem cached color
> @@ -437,6 +442,49 @@ def render_charts(ctx, options, clip, trace, curr_y, w, h, sec_w):
>
>          curr_y = curr_y + 30 + bar_h
>
> +    if trace.net_stats:
> +        for iface, samples in trace.net_stats.items():
> +            max_received_sample = max(samples, key=lambda s: s.received_bytes)
> +            max_transmitted_sample = max(samples, key=lambda s: s.transmitted_bytes)
> +            max_receive_diff_sample = max(samples, key=lambda s: s.receive_diff)
> +            max_transmit_diff_sample = max(samples, key=lambda s: s.transmit_diff)
> +
> +            draw_text(ctx, "Iface: %s" % (iface), TEXT_COLOR, off_x, curr_y+20)
> +            draw_legend_line(ctx, "Bytes received (max %d)" % (max_received_sample.received_bytes),
> +                             BYTES_RECEIVED_COLOR, off_x+150, curr_y+20, leg_s)
> +            draw_legend_line(ctx, "Bytes transmitted (max %d)" % (max_transmitted_sample.transmitted_bytes),
> +                             BYTES_TRANSMITTED_COLOR, off_x+400, curr_y+20, leg_s)
> +            draw_legend_box(ctx, "Bytes receive diff (max %d)" % (max_receive_diff_sample.receive_diff),
> +                             BYTES_RECEIVE_DIFF_COLOR, off_x+650, curr_y+20, leg_s)
> +            draw_legend_box(ctx, "Bytes transmit diff (max %d)" % (max_transmit_diff_sample.transmit_diff),
> +                             BYTES_TRANSMIT_DIFF_COLOR, off_x+900, curr_y+20, leg_s)
> +
> +
> +            chart_rect = (off_x, curr_y + 30, w, bar_h)
> +            if clip_visible(clip, chart_rect):
> +                draw_box_ticks(ctx, chart_rect, sec_w)
> +                draw_annotations(ctx, proc_tree, trace.times, chart_rect)
> +
> +            if clip_visible (clip, chart_rect):
> +                draw_chart (ctx, BYTES_RECEIVED_COLOR, False, chart_rect, \
> +                        [(sample.time, sample.received_bytes) for sample in samples], \
> +                        proc_tree, None)
> +
> +                draw_chart (ctx, BYTES_TRANSMITTED_COLOR, False, chart_rect, \
> +                        [(sample.time, sample.transmitted_bytes) for sample in samples], \
> +                        proc_tree, None)
> +
> +            if clip_visible (clip, chart_rect):
> +                draw_chart (ctx, BYTES_RECEIVE_DIFF_COLOR, True, chart_rect, \
> +                        [(sample.time, sample.receive_diff) for sample in samples], \
> +                        proc_tree, None)
> +
> +                draw_chart (ctx, BYTES_TRANSMIT_DIFF_COLOR, True, chart_rect, \
> +                        [(sample.time, sample.transmit_diff) for sample in samples], \
> +                        proc_tree, None)
> +
> +            curr_y = curr_y + 30 + bar_h
> +
>      # render CPU pressure chart
>      if trace.cpu_pressure:
>          max_sample_avg = max (trace.cpu_pressure, key = lambda s: s.avg10)
> diff --git a/pybootchartgui/pybootchartgui/parsing.py b/pybootchartgui/pybootchartgui/parsing.py
> index 144a16c723..72a54c6ba5 100644
> --- a/pybootchartgui/pybootchartgui/parsing.py
> +++ b/pybootchartgui/pybootchartgui/parsing.py
> @@ -48,6 +48,7 @@ class Trace:
>          self.filename = None
>          self.parent_map = None
>          self.mem_stats = []
> +        self.net_stats = []
>          self.monitor_disk = None
>          self.cpu_pressure = []
>          self.io_pressure = []
> @@ -557,6 +558,21 @@ def _parse_monitor_disk_log(file):
>
>      return disk_stats
>
> +
> +def _parse_reduced_net_log(file):
> +    net_stats = {}
> +    for time, lines in _parse_timed_blocks(file):
> +
> +        for line in lines:
> +            parts = line.split()
> +            iface = parts[0][:-1]
> +            if iface not in net_stats:
> +                net_stats[iface] = [NetSample(time, iface, int(parts[1]), int(parts[2]), int(parts[3]), int(parts[4]))]
> +            else:
> +                net_stats[iface].append(NetSample(time, iface, int(parts[1]), int(parts[2]), int(parts[3]), int(parts[4])))
> +    return net_stats
> +
> +
>  def _parse_pressure_logs(file, filename):
>      """
>      Parse file for "some" pressure with 'avg10', 'avg60' 'avg300' and delta total values
> @@ -767,6 +783,8 @@ def _do_parse(writer, state, filename, file):
>          state.cmdline = _parse_cmdline_log(writer, file)
>      elif name == "monitor_disk.log":
>          state.monitor_disk = _parse_monitor_disk_log(file)
> +    elif name == "reduced_proc_net.log":
> +        state.net_stats = _parse_reduced_net_log(file)
>      #pressure logs are in a subdirectory
>      elif name == "cpu.log":
>          state.cpu_pressure = _parse_pressure_logs(file, name)
> diff --git a/pybootchartgui/pybootchartgui/samples.py b/pybootchartgui/pybootchartgui/samples.py
> index a70d8a5a28..7c92d2ce6a 100644
> --- a/pybootchartgui/pybootchartgui/samples.py
> +++ b/pybootchartgui/pybootchartgui/samples.py
> @@ -37,6 +37,16 @@ class CPUSample:
>          return str(self.time) + "\t" + str(self.user) + "\t" + \
>                 str(self.sys) + "\t" + str(self.io) + "\t" + str (self.swap)
>
> +
> +class NetSample:
> +    def __init__(self, time, iface, received_bytes, transmitted_bytes, receive_diff, transmit_diff):
> +        self.time = time
> +        self.iface = iface
> +        self.received_bytes = received_bytes
> +        self.transmitted_bytes = transmitted_bytes
> +        self.receive_diff = receive_diff
> +        self.transmit_diff = transmit_diff
> +
>  class CPUPressureSample:
>      def __init__(self, time, avg10, avg60, avg300, deltaTotal):
>          self.time = time
> --
> 2.34.1
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#214962): https://lists.openembedded.org/g/openembedded-core/message/214962
> Mute This Topic: https://lists.openembedded.org/mt/112282219/1686489
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alex.kanavin@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>

On 2025-04-22 10:50 a.m., Alexander Kanavin via lists.openembedded.org 
wrote:
> Also here it would be good to see a demo, perhaps you can publish an
> image somewhere and link to it?


Here are 4 screenshots showing the new network traffic:
https://photos.app.goo.gl/AeqMBiRQzRWbMadU9

c-i-sato-full-fetch-full-time.png
c-i-sato-full-fetch.png
c-i-sato-prefetch-full-time.png
c-i-sato-prefetch.png

one set (prefetch) after running bitbake --runall fetch then bitbake 
core-image-sato
and the other from a clean build (full-fetch) without any downloads.

The builder is my 20 core laptop with an SSD and wired 100/30 (dl/ul) 
Mbps home network.
There were a few apps running in the background so there's some low 
bandwidth network chatter.


The pybootchartgui works but it would be nice to be able to:

1.  select which stats are displayed using a command line or interactively,

2. separate the display of various stats over time from the dependency 
waterfall
    so that you could zoom into a small area of the waterfall and see 
the corresponding area of the stats.

I wonder if there are any web devs who would perhaps use OSM 
(https://openmaptiles.org/) to do 2. ...

../Randy




From: /home/rmacleod/wrs/lts25/pybootchartgui


>
> Alex
>
> On Tue, 15 Apr 2025 at 20:37, denisova.olga.k via
> lists.openembedded.org
> <denisova.olga.k=yandex.ru@lists.openembedded.org> wrote:
>> From: Olga Denisova<denisova.olga.k@yandex.ru>
>>
>> This patch adds support for parsing and visualizing network interface statistics from /proc/net/dev in pybootchartgui. It introduces a new NetSample class to hold per-interface metrics, including received/transmitted bytes and their deltas over time.
>>
>> The data is drawn using line and box charts in draw.py and helps to monitor
>> network usage during the boot process for each interface individually.
>>
>> Signed-off-by: denisova-ok<denisova.olga.k@yandex.ru>
>> ---
>>   pybootchartgui/pybootchartgui/draw.py    | 48 ++++++++++++++++++++++++
>>   pybootchartgui/pybootchartgui/parsing.py | 18 +++++++++
>>   pybootchartgui/pybootchartgui/samples.py | 10 +++++
>>   3 files changed, 76 insertions(+)
>>
>> diff --git a/pybootchartgui/pybootchartgui/draw.py b/pybootchartgui/pybootchartgui/draw.py
>> index c6e67833ab..16739a0fa1 100644
>> --- a/pybootchartgui/pybootchartgui/draw.py
>> +++ b/pybootchartgui/pybootchartgui/draw.py
>> @@ -69,6 +69,11 @@ CPU_COLOR = (0.40, 0.55, 0.70, 1.0)
>>   IO_COLOR = (0.76, 0.48, 0.48, 0.5)
>>   # Disk throughput color.
>>   DISK_TPUT_COLOR = (0.20, 0.71, 0.20, 1.0)
>> +
>> +BYTES_RECEIVED_COLOR = (0.0, 0.0, 1.0, 1.0)
>> +BYTES_TRANSMITTED_COLOR = (1.0, 0.0, 0.0, 1.0)
>> +BYTES_RECEIVE_DIFF_COLOR = (0.0, 0.0, 1.0, 0.3)
>> +BYTES_TRANSMIT_DIFF_COLOR = (1.0, 0.0, 0.0, 0.3)
>>   # CPU load chart color.
>>   FILE_OPEN_COLOR = (0.20, 0.71, 0.71, 1.0)
>>   # Mem cached color
>> @@ -437,6 +442,49 @@ def render_charts(ctx, options, clip, trace, curr_y, w, h, sec_w):
>>
>>           curr_y = curr_y + 30 + bar_h
>>
>> +    if trace.net_stats:
>> +        for iface, samples in trace.net_stats.items():
>> +            max_received_sample = max(samples, key=lambda s: s.received_bytes)
>> +            max_transmitted_sample = max(samples, key=lambda s: s.transmitted_bytes)
>> +            max_receive_diff_sample = max(samples, key=lambda s: s.receive_diff)
>> +            max_transmit_diff_sample = max(samples, key=lambda s: s.transmit_diff)
>> +
>> +            draw_text(ctx, "Iface: %s" % (iface), TEXT_COLOR, off_x, curr_y+20)
>> +            draw_legend_line(ctx, "Bytes received (max %d)" % (max_received_sample.received_bytes),
>> +                             BYTES_RECEIVED_COLOR, off_x+150, curr_y+20, leg_s)
>> +            draw_legend_line(ctx, "Bytes transmitted (max %d)" % (max_transmitted_sample.transmitted_bytes),
>> +                             BYTES_TRANSMITTED_COLOR, off_x+400, curr_y+20, leg_s)
>> +            draw_legend_box(ctx, "Bytes receive diff (max %d)" % (max_receive_diff_sample.receive_diff),
>> +                             BYTES_RECEIVE_DIFF_COLOR, off_x+650, curr_y+20, leg_s)
>> +            draw_legend_box(ctx, "Bytes transmit diff (max %d)" % (max_transmit_diff_sample.transmit_diff),
>> +                             BYTES_TRANSMIT_DIFF_COLOR, off_x+900, curr_y+20, leg_s)
>> +
>> +
>> +            chart_rect = (off_x, curr_y + 30, w, bar_h)
>> +            if clip_visible(clip, chart_rect):
>> +                draw_box_ticks(ctx, chart_rect, sec_w)
>> +                draw_annotations(ctx, proc_tree, trace.times, chart_rect)
>> +
>> +            if clip_visible (clip, chart_rect):
>> +                draw_chart (ctx, BYTES_RECEIVED_COLOR, False, chart_rect, \
>> +                        [(sample.time, sample.received_bytes) for sample in samples], \
>> +                        proc_tree, None)
>> +
>> +                draw_chart (ctx, BYTES_TRANSMITTED_COLOR, False, chart_rect, \
>> +                        [(sample.time, sample.transmitted_bytes) for sample in samples], \
>> +                        proc_tree, None)
>> +
>> +            if clip_visible (clip, chart_rect):
>> +                draw_chart (ctx, BYTES_RECEIVE_DIFF_COLOR, True, chart_rect, \
>> +                        [(sample.time, sample.receive_diff) for sample in samples], \
>> +                        proc_tree, None)
>> +
>> +                draw_chart (ctx, BYTES_TRANSMIT_DIFF_COLOR, True, chart_rect, \
>> +                        [(sample.time, sample.transmit_diff) for sample in samples], \
>> +                        proc_tree, None)
>> +
>> +            curr_y = curr_y + 30 + bar_h
>> +
>>       # render CPU pressure chart
>>       if trace.cpu_pressure:
>>           max_sample_avg = max (trace.cpu_pressure, key = lambda s: s.avg10)
>> diff --git a/pybootchartgui/pybootchartgui/parsing.py b/pybootchartgui/pybootchartgui/parsing.py
>> index 144a16c723..72a54c6ba5 100644
>> --- a/pybootchartgui/pybootchartgui/parsing.py
>> +++ b/pybootchartgui/pybootchartgui/parsing.py
>> @@ -48,6 +48,7 @@ class Trace:
>>           self.filename = None
>>           self.parent_map = None
>>           self.mem_stats = []
>> +        self.net_stats = []
>>           self.monitor_disk = None
>>           self.cpu_pressure = []
>>           self.io_pressure = []
>> @@ -557,6 +558,21 @@ def _parse_monitor_disk_log(file):
>>
>>       return disk_stats
>>
>> +
>> +def _parse_reduced_net_log(file):
>> +    net_stats = {}
>> +    for time, lines in _parse_timed_blocks(file):
>> +
>> +        for line in lines:
>> +            parts = line.split()
>> +            iface = parts[0][:-1]
>> +            if iface not in net_stats:
>> +                net_stats[iface] = [NetSample(time, iface, int(parts[1]), int(parts[2]), int(parts[3]), int(parts[4]))]
>> +            else:
>> +                net_stats[iface].append(NetSample(time, iface, int(parts[1]), int(parts[2]), int(parts[3]), int(parts[4])))
>> +    return net_stats
>> +
>> +
>>   def _parse_pressure_logs(file, filename):
>>       """
>>       Parse file for "some" pressure with 'avg10', 'avg60' 'avg300' and delta total values
>> @@ -767,6 +783,8 @@ def _do_parse(writer, state, filename, file):
>>           state.cmdline = _parse_cmdline_log(writer, file)
>>       elif name == "monitor_disk.log":
>>           state.monitor_disk = _parse_monitor_disk_log(file)
>> +    elif name == "reduced_proc_net.log":
>> +        state.net_stats = _parse_reduced_net_log(file)
>>       #pressure logs are in a subdirectory
>>       elif name == "cpu.log":
>>           state.cpu_pressure = _parse_pressure_logs(file, name)
>> diff --git a/pybootchartgui/pybootchartgui/samples.py b/pybootchartgui/pybootchartgui/samples.py
>> index a70d8a5a28..7c92d2ce6a 100644
>> --- a/pybootchartgui/pybootchartgui/samples.py
>> +++ b/pybootchartgui/pybootchartgui/samples.py
>> @@ -37,6 +37,16 @@ class CPUSample:
>>           return str(self.time) + "\t" + str(self.user) + "\t" + \
>>                  str(self.sys) + "\t" + str(self.io) + "\t" + str (self.swap)
>>
>> +
>> +class NetSample:
>> +    def __init__(self, time, iface, received_bytes, transmitted_bytes, receive_diff, transmit_diff):
>> +        self.time = time
>> +        self.iface = iface
>> +        self.received_bytes = received_bytes
>> +        self.transmitted_bytes = transmitted_bytes
>> +        self.receive_diff = receive_diff
>> +        self.transmit_diff = transmit_diff
>> +
>>   class CPUPressureSample:
>>       def __init__(self, time, avg10, avg60, avg300, deltaTotal):
>>           self.time = time
>> --
>> 2.34.1
>>
>>
>>
>>
>>
>> -=-=-=-=-=-=-=-=-=-=-=-
>> Links: You receive all messages sent to this group.
>> View/Reply Online (#215228):https://lists.openembedded.org/g/openembedded-core/message/215228
>> Mute This Topic:https://lists.openembedded.org/mt/112282219/3616765
>> Group Owner:openembedded-core+owner@lists.openembedded.org
>> Unsubscribe:https://lists.openembedded.org/g/openembedded-core/unsub [randy.macleod@windriver.com]
>> -=-=-=-=-=-=-=-=-=-=-=-
>>