@@ -24,11 +24,13 @@ class Task:
task: str
start: datetime.datetime
duration: datetime.timedelta
+ memory: int
class Sorting(enum.Enum):
start = 1
duration = 2
+ memory = 3
# argparse integration
def __str__(self) -> str:
@@ -64,6 +66,7 @@ def dump_buildstats(args, bs: buildstats.BuildStats):
task,
datetime.datetime.fromtimestamp(stats["start_time"]),
datetime.timedelta(seconds=int(stats.walltime)),
+ max(stats["rusage"]["ru_maxrss"], stats["child_rusage"]["ru_maxrss"])
)
tasks.append(t)
@@ -71,10 +74,12 @@ def dump_buildstats(args, bs: buildstats.BuildStats):
minimum = datetime.timedelta(seconds=args.shortest)
highlight = datetime.timedelta(seconds=args.highlight)
+ memlimit = args.memlimit
+ print("start duration maxrss recipe:task")
for t in tasks:
- if t.duration >= minimum:
- line = f"{t.duration} {t.recipe}:{t.task}"
+ if t.duration >= minimum and t.memory >= memlimit:
+ line = f"{t.start} {t.duration} {t.memory} {t.recipe}:{t.task}"
if args.highlight and t.duration >= highlight:
print(f"\033[1m{line}\033[0m")
else:
@@ -113,6 +118,14 @@ def main(argv=None) -> int:
metavar="SECS",
help="Highlight tasks longer than SECS seconds (0 disabled)",
)
+ parser.add_argument(
+ "--memlimit",
+ "-m",
+ type=int,
+ default=0,
+ metavar="LIMIT",
+ help="Hide tasks with lower than LIMIT kilobytes (0 disabled)",
+ )
args = parser.parse_args(argv)
I was observing processes being killed by the OOM killer when certain recipes and tasks were being executed. Many of the physics or mathematical libraries consume a lot of memory in the do_compile phase. Running a build with buildstats enabled showed that the maxrss for these tasks was very high. The highest maxrss value I have observed is 7.3GB. I wanted to summarize the results, but noticed that buildstats-summary only accounts for the duration of the tasks. I wrote a quick enhancement to extend it to also print and filter based on maxrss. An example of its use and the modified output is shown below: ``` buildstats-summary tmp-glibc/buildstats/20250901144058/ --memlimit 2000000 --sort=memory start duration maxrss recipe:task 2025-09-01 09:27:54.740000 0:01:22 2027764 create3-republisher-1.0.0:do_compile 2025-09-01 09:21:13.290000 0:01:16 2033980 ublox-gps-2.3.0:do_compile 2025-09-01 09:23:04.960000 0:01:04 2037628 pcl-ros-2.6.2:do_compile 2025-09-01 08:07:39.810000 0:21:03 2061548 clang-native:do_compile 2025-09-01 08:45:10.120000 0:07:03 2351164 lanelet2-core-1.2.1:do_compile 2025-09-01 08:56:17.390000 0:02:02 2414996 dartsim:do_compile 2025-09-01 09:15:27.110000 0:02:00 2445608 mavros-2.10.1:do_compile 2025-09-01 08:27:01.290000 0:03:16 2450224 rmf-traffic-3.3.3:do_compile 2025-09-01 09:26:44.610000 0:00:04 2468600 openvdb-vendor-2.5.5:do_package_qa 2025-09-01 09:15:26.830000 0:02:18 2535556 irobot-create-nodes-3.0.4:do_compile 2025-09-01 09:26:48.090000 0:02:05 2542520 mavros-extras-2.10.1:do_compile 2025-09-01 09:27:21.880000 0:02:38 2552664 sbg-driver-3.2.0:do_compile 2025-09-01 09:28:58.090000 0:01:12 2586168 fuse-viz-1.1.3:do_compile 2025-09-01 09:26:07.880000 0:00:26 2627588 openvdb-vendor-2.5.5:do_package 2025-09-01 08:59:03.790000 0:04:26 2636372 libflann:do_compile 2025-09-01 09:07:55.820000 0:00:04 2657452 openvdb:do_package_qa 2025-09-01 08:25:42.940000 0:02:41 2674024 ceres-solver:do_compile 2025-09-01 08:27:56.430000 0:06:14 2804552 cargo-native:do_compile 2025-09-01 09:15:24.760000 0:03:05 2823900 robot-localization-3.8.2:do_compile 2025-09-01 07:58:15.970000 0:09:31 2990388 linux-raspberrypi:do_fetch 2025-09-01 09:28:31.750000 0:01:55 3033436 fuse-models-1.1.3:do_compile 2025-09-01 09:15:02.480000 0:04:02 3064352 gz-sim8:do_compile 2025-09-01 08:44:39.500000 0:09:58 3176792 fcl:do_compile 2025-09-01 08:46:12.870000 0:09:08 3213876 gtsam-4.2.0:do_compile 2025-09-01 09:07:19.880000 0:00:30 3261900 openvdb:do_package 2025-09-01 09:22:39.530000 0:02:00 3455096 microstrain-inertial-driver-4.7.0:do_compile 2025-09-01 08:36:26.350000 0:08:22 3495032 cargo:do_compile 2025-09-01 08:59:07.570000 0:00:55 3532888 gz-physics7:do_compile 2025-09-01 08:44:16.130000 0:17:53 3716804 opencv:do_compile 2025-09-01 08:58:28.070000 0:02:46 3862464 lanelet2-python-1.2.1:do_compile 2025-09-01 09:11:32.360000 0:09:22 3985780 rmf-traffic-ros2-2.7.2:do_compile 2025-09-01 09:16:20.870000 0:03:37 4400996 ublox-dgnss-node-0.5.7:do_compile 2025-09-01 09:12:30.060000 0:05:34 4592184 ros-gz-bridge-1.0.16:do_compile 2025-09-01 09:11:05.580000 0:08:14 4599324 pcl:do_compile 2025-09-01 08:28:38.090000 0:04:39 6326400 sophus-1.22.9102:do_compile 2025-09-01 08:46:44.050000 0:18:08 6964496 openvdb:do_compile 2025-09-01 09:24:36.750000 0:05:58 7093536 performance-test-2.3.0:do_compile 2025-09-01 09:14:37.610000 0:11:23 7352348 openvdb-vendor-2.5.5:do_compile ``` I would appreciate any feedback folks have to revise the work: Notably: 1. Do any existing scripts depend on preserving the old output? I added the start time and maxrss as new columns as well as a header to describe what each column is. Should any changes to the output be disabled by default and only take effect if you pass in a new option? 2. There are additional variables in the buildstat files that are still not exposed. Should we create a generic way to expose these values or wait for a use case to arise? Signed-off-by: Rob Woolley <rob.woolley@windriver.com>