| Message ID | 20260325084021.3915804-2-daniel.turull@ericsson.com |
|---|---|
| State | Changes Requested |
| Headers | show |
| Series | [1/2] python3-kirk: Add version 4.0.0 | expand |
On Wed, 25 Mar 2026 at 12:07, Daniel Turull via lists.openembedded.org <daniel.turull=ericsson.com@lists.openembedded.org> wrote: > - parser = LtpParser() > - results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group)) > + # Copy kirk JSON report from target > + dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group) > + remote_src = "/opt/ltp/results/%s.json" % stress_group > + (status, output) = self.target.copyFrom(remote_src, dst, True) > + if status: > + msg = 'File could not be copied. Output: %s' % output > + self.target.logger.warning(msg) > + > + parser = LtpKirkParser() > + results, sections = parser.parse(dst) Why is an extra copying step added? Alex
On Wed, Mar 25, 2026 at 7:07 AM Daniel Turull via lists.openembedded.org <daniel.turull=ericsson.com@lists.openembedded.org> wrote: > From: Daniel Turull <daniel.turull@ericsson.com> > > runltp has been removed from ltp and kirk is the official tool to > invoke linux ltp tests. For something like this to be considered for merging, I'd have expected a lot more information. Is the test coverage the same as before ? What are the results of the tests versus the previous harness ? Is it truly a drop in replacement for the previous runner ? .. etc Switching the executable is the easy part, ensuring that it actually works a replacement is the hard part. Bruce > > > See: > > https://github.com/linux-test-project/ltp/commit/6efd3605dc005c3ed135b463f182174e24bdce1b > > Signed-off-by: Daniel Turull <daniel.turull@ericsson.com> > Assisted-by: Claude, Anthropic > --- > meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++------- > meta/lib/oeqa/runtime/cases/ltp_stress.py | 16 +++++++++---- > meta/lib/oeqa/utils/logparser.py | 29 +++++++++++++++++++++++ > meta/recipes-extended/ltp/ltp_20260130.bb | 1 + > 4 files changed, 50 insertions(+), 12 deletions(-) > > diff --git a/meta/lib/oeqa/runtime/cases/ltp.py > b/meta/lib/oeqa/runtime/cases/ltp.py > index 0ffdbe23e4..11c4814090 100644 > --- a/meta/lib/oeqa/runtime/cases/ltp.py > +++ b/meta/lib/oeqa/runtime/cases/ltp.py > @@ -12,7 +12,7 @@ import pprint > from oeqa.runtime.case import OERuntimeTestCase > from oeqa.core.decorator.depends import OETestDepends > from oeqa.runtime.decorator.package import OEHasPackage > -from oeqa.utils.logparser import LtpParser > +from oeqa.utils.logparser import LtpKirkParser > > class LtpTestBase(OERuntimeTestCase): > > @@ -66,9 +66,9 @@ class LtpTest(LtpTestBase): > > def runltp(self, ltp_group): > # LTP appends to log files, so ensure we start with a clean > log > - self.target.deleteFiles("/opt/ltp/results/", ltp_group) > + self.target.deleteFiles("/opt/ltp/results/", "%s.json" % > ltp_group) > > - cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l > /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) > + cmd = 'kirk --run-suite %s --json-report > /opt/ltp/results/%s.json -n' % (ltp_group, ltp_group) > > starttime = time.time() > (status, output) = self.target.run(cmd, timeout=1200) > @@ -87,14 +87,14 @@ class LtpTest(LtpTestBase): > self.extras['ltpresult.rawlogs']['log'] = > self.extras['ltpresult.rawlogs']['log'] + output > > # Copy the machine-readable test results locally so we can > parse it > - dst = os.path.join(self.ltptest_log_dir, ltp_group) > - remote_src = "/opt/ltp/results/%s" % ltp_group > + dst = os.path.join(self.ltptest_log_dir, "%s.json" % > ltp_group) > + remote_src = "/opt/ltp/results/%s.json" % ltp_group > (status, output) = self.target.copyFrom(remote_src, dst, True) > if status: > msg = 'File could not be copied. Output: %s' % output > self.target.logger.warning(msg) > > - parser = LtpParser() > + parser = LtpKirkParser() > results, sections = parser.parse(dst) > > sections['duration'] = int(endtime-starttime) > @@ -113,9 +113,9 @@ class LtpTest(LtpTestBase): > > # LTP runtime tests > @OETestDepends(['ssh.SSHTest.test_ssh']) > - @OEHasPackage(["ltp"]) > + @OEHasPackage(["ltp", "python3-kirk"]) > def test_ltp_help(self): > - (status, output) = self.target.run('/opt/ltp/runltp --help') > + (status, output) = self.target.run('kirk --help') > msg = 'Failed to get ltp help. Output: %s' % output > self.assertEqual(status, 0, msg=msg) > > diff --git a/meta/lib/oeqa/runtime/cases/ltp_stress.py > b/meta/lib/oeqa/runtime/cases/ltp_stress.py > index ce6f4bf59d..cf84ec1182 100644 > --- a/meta/lib/oeqa/runtime/cases/ltp_stress.py > +++ b/meta/lib/oeqa/runtime/cases/ltp_stress.py > @@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCase > from oeqa.core.decorator.depends import OETestDepends > from oeqa.runtime.decorator.package import OEHasPackage > from oeqa.core.decorator.data import skipIfQemu > -from oeqa.utils.logparser import LtpParser > +from oeqa.utils.logparser import LtpKirkParser > > class LtpStressBase(OERuntimeTestCase): > > @@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase): > class LtpStressTest(LtpStressBase): > > def runltp(self, stress_group): > - cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee > /opt/ltp/results/%s' % (stress_group, stress_group) > + cmd = 'kirk --run-suite %s --json-report > /opt/ltp/results/%s.json -n' % (stress_group, stress_group) > starttime = time.time() > (status, output) = self.target.run(cmd) > endtime = time.time() > @@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase): > > self.extras['ltpstressresult.rawlogs']['log'] = > self.extras['ltpstressresult.rawlogs']['log'] + output > > - parser = LtpParser() > - results, sections = > parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group)) > + # Copy kirk JSON report from target > + dst = os.path.join(self.ltptest_log_dir, "%s.json" % > stress_group) > + remote_src = "/opt/ltp/results/%s.json" % stress_group > + (status, output) = self.target.copyFrom(remote_src, dst, True) > + if status: > + msg = 'File could not be copied. Output: %s' % output > + self.target.logger.warning(msg) > + > + parser = LtpKirkParser() > + results, sections = parser.parse(dst) > > runtime = int(endtime-starttime) > sections['duration'] = runtime > diff --git a/meta/lib/oeqa/utils/logparser.py > b/meta/lib/oeqa/utils/logparser.py > index c479864162..a907421fab 100644 > --- a/meta/lib/oeqa/utils/logparser.py > +++ b/meta/lib/oeqa/utils/logparser.py > @@ -5,6 +5,7 @@ > # > > import enum > +import json > import os > import re > > @@ -158,6 +159,34 @@ class LtpParser: > return results, section > > > +class LtpKirkParser: > + """Parse kirk JSON report into the same format as LtpParser.""" > + > + STATUS_MAP = { > + "pass": "PASSED", > + "fail": "FAILED", > + "brok": "FAILED", > + "conf": "SKIPPED", > + "warn": "PASSED", > + } > + > + def parse(self, jsonfile): > + with open(jsonfile, errors="replace") as f: > + report = json.load(f) > + > + results = {} > + section = {"duration": 0, "log": ""} > + > + for entry in report.get("results", []): > + results[entry["test_fqn"]] = > self.STATUS_MAP.get(entry.get("status", ""), "FAILED") > + test = entry.get("test", {}) > + section["log"] += test.get("log", "") > + > + section["duration"] = int(report.get("stats", {}).get("runtime", > 0)) > + > + return results, section > + > + > # ltp Compliance log parsing > class LtpComplianceParser(object): > def __init__(self): > diff --git a/meta/recipes-extended/ltp/ltp_20260130.bb > b/meta/recipes-extended/ltp/ltp_20260130.bb > index dcd1e81398..1ff20a7898 100644 > --- a/meta/recipes-extended/ltp/ltp_20260130.bb > +++ b/meta/recipes-extended/ltp/ltp_20260130.bb > @@ -104,6 +104,7 @@ RDEPENDS:${PN} = "\ > net-tools \ > perl \ > python3-core \ > + python3-kirk \ > procps \ > quota \ > unzip \ > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#233871): > https://lists.openembedded.org/g/openembedded-core/message/233871 > Mute This Topic: https://lists.openembedded.org/mt/118498904/1050810 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ > bruce.ashfield@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
On Wed, 25 Mar 2026 at 13:58, Bruce Ashfield via lists.openembedded.org <bruce.ashfield=gmail.com@lists.openembedded.org> wrote: > For something like this to be considered for merging, I'd have expected > a lot more information. > > Is the test coverage the same as before ? > What are the results of the tests versus the previous harness ? > Is it truly a drop in replacement for the previous runner ? > .. etc > > Switching the executable is the easy part, ensuring that it actually works > a replacement is the hard part. >> Assisted-by: Claude, Anthropic I have to add that this line ^^^ doesn't inspire confidence in the patch either. On the contrary, it raises suspicions that the code was written quickly after the issue was raised yesterday in the tech meeting, and wasn't actually tested well or at all. The commit message *must* reassure that it was. Alex
> -----Original Message----- > From: Alexander Kanavin <alex.kanavin@gmail.com> > Sent: Wednesday, 25 March 2026 13:39 > To: Daniel Turull <daniel.turull@ericsson.com> > Cc: openembedded-core@lists.openembedded.org; pratik.farkase@est.tech; > richard.purdie@linuxfoundation.org > Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk > > [You don't often get email from alex.kanavin@gmail.com. Learn why this is > important at https://aka.ms/LearnAboutSenderIdentification ] > > On Wed, 25 Mar 2026 at 12:07, Daniel Turull via lists.openembedded.org > <daniel.turull=ericsson.com@lists.openembedded.org> wrote: > > - parser = LtpParser() > > - results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % > stress_group)) > > + # Copy kirk JSON report from target > > + dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group) > > + remote_src = "/opt/ltp/results/%s.json" % stress_group > > + (status, output) = self.target.copyFrom(remote_src, dst, True) > > + if status: > > + msg = 'File could not be copied. Output: %s' % output > > + self.target.logger.warning(msg) > > + > > + parser = LtpKirkParser() > > + results, sections = parser.parse(dst) > > Why is an extra copying step added? Kirk creates a json file with the results that it is more machine readable friendly and needs to be copied over from the target. The old code was having the output results in the log. > > Alex
Hi, Kirk is official way to run ltp since runltp has been dropped. Richard mentioned yesterday in the Weekly Project Engineering Sync if no one was fixing the runltp issue, he will drop ltp from oe-core. The test coverage is the same since the logic there is not touched but the output is different. Now it is a json file with the results, which is machine readable without any extra parsing. Do we need a conversion from the new to the old to keep compatibility? I’m attaching the old test output with the new one for the math test suite. New: math-raw.log and math.json Old: math.old math-raw.log.old. You can see at the end of the old output: ------------------------------------------- INFO: runltp script is deprecated, try kirk https://github.com/linux-test-project/kirk ------------------------------------------- Best regards, Daniel From: Bruce Ashfield <bruce.ashfield@gmail.com> Sent: Wednesday, 25 March 2026 13:59 To: Daniel Turull <daniel.turull@ericsson.com> Cc: openembedded-core@lists.openembedded.org; pratik.farkase@est.tech; richard.purdie@linuxfoundation.org Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk On Wed, Mar 25, 2026 at 7:07 AM Daniel Turull via lists.openembedded.org<http://lists.openembedded.org/> <daniel.turull=ericsson.com@lists.openembedded.org<mailto:ericsson.com@lists.openembedded.org>> wrote: From: Daniel Turull <daniel.turull@ericsson.com<mailto:daniel.turull@ericsson.com>> runltp has been removed from ltp and kirk is the official tool to invoke linux ltp tests. For something like this to be considered for merging, I'd have expected a lot more information. Is the test coverage the same as before ? What are the results of the tests versus the previous harness ? Is it truly a drop in replacement for the previous runner ? .. etc Switching the executable is the easy part, ensuring that it actually works a replacement is the hard part. Bruce See: https://github.com/linux-test-project/ltp/commit/6efd3605dc005c3ed135b463f182174e24bdce1b Signed-off-by: Daniel Turull <daniel.turull@ericsson.com<mailto:daniel.turull@ericsson.com>> Assisted-by: Claude, Anthropic --- meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++------- meta/lib/oeqa/runtime/cases/ltp_stress.py | 16 +++++++++---- meta/lib/oeqa/utils/logparser.py | 29 +++++++++++++++++++++++ meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> | 1 + 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/meta/lib/oeqa/runtime/cases/ltp.py b/meta/lib/oeqa/runtime/cases/ltp.py index 0ffdbe23e4..11c4814090 100644 --- a/meta/lib/oeqa/runtime/cases/ltp.py +++ b/meta/lib/oeqa/runtime/cases/ltp.py @@ -12,7 +12,7 @@ import pprint from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpTestBase(OERuntimeTestCase): @@ -66,9 +66,9 @@ class LtpTest(LtpTestBase): def runltp(self, ltp_group): # LTP appends to log files, so ensure we start with a clean log - self.target.deleteFiles("/opt/ltp/results/", ltp_group) + self.target.deleteFiles("/opt/ltp/results/", "%s.json" % ltp_group) - cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (ltp_group, ltp_group) starttime = time.time() (status, output) = self.target.run(cmd, timeout=1200) @@ -87,14 +87,14 @@ class LtpTest(LtpTestBase): self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output # Copy the machine-readable test results locally so we can parse it - dst = os.path.join(self.ltptest_log_dir, ltp_group) - remote_src = "/opt/ltp/results/%s" % ltp_group + dst = os.path.join(self.ltptest_log_dir, "%s.json" % ltp_group) + remote_src = "/opt/ltp/results/%s.json" % ltp_group (status, output) = self.target.copyFrom(remote_src, dst, True) if status: msg = 'File could not be copied. Output: %s' % output self.target.logger.warning(msg) - parser = LtpParser() + parser = LtpKirkParser() results, sections = parser.parse(dst) sections['duration'] = int(endtime-starttime) @@ -113,9 +113,9 @@ class LtpTest(LtpTestBase): # LTP runtime tests @OETestDepends(['ssh.SSHTest.test_ssh']) - @OEHasPackage(["ltp"]) + @OEHasPackage(["ltp", "python3-kirk"]) def test_ltp_help(self): - (status, output) = self.target.run('/opt/ltp/runltp --help') + (status, output) = self.target.run('kirk --help') msg = 'Failed to get ltp help. Output: %s' % output self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/ltp_stress.py b/meta/lib/oeqa/runtime/cases/ltp_stress.py index ce6f4bf59d..cf84ec1182 100644 --- a/meta/lib/oeqa/runtime/cases/ltp_stress.py +++ b/meta/lib/oeqa/runtime/cases/ltp_stress.py @@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage from oeqa.core.decorator.data import skipIfQemu -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpStressBase(OERuntimeTestCase): @@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase): class LtpStressTest(LtpStressBase): def runltp(self, stress_group): - cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group, stress_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (stress_group, stress_group) starttime = time.time() (status, output) = self.target.run(cmd) endtime = time.time() @@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase): self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output - parser = LtpParser() - results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group)) + # Copy kirk JSON report from target + dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group) + remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst) runtime = int(endtime-starttime) sections['duration'] = runtime diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py index c479864162..a907421fab 100644 --- a/meta/lib/oeqa/utils/logparser.py +++ b/meta/lib/oeqa/utils/logparser.py @@ -5,6 +5,7 @@ # import enum +import json import os import re @@ -158,6 +159,34 @@ class LtpParser: return results, section +class LtpKirkParser: + """Parse kirk JSON report into the same format as LtpParser.""" + + STATUS_MAP = { + "pass": "PASSED", + "fail": "FAILED", + "brok": "FAILED", + "conf": "SKIPPED", + "warn": "PASSED", + } + + def parse(self, jsonfile): + with open(jsonfile, errors="replace") as f: + report = json.load(f) + + results = {} + section = {"duration": 0, "log": ""} + + for entry in report.get("results", []): + results[entry["test_fqn"]] = self.STATUS_MAP.get(entry.get("status", ""), "FAILED") + test = entry.get("test", {}) + section["log"] += test.get("log", "") + + section["duration"] = int(report.get("stats", {}).get("runtime", 0)) + + return results, section + + # ltp Compliance log parsing class LtpComplianceParser(object): def __init__(self): diff --git a/meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> b/meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> index dcd1e81398..1ff20a7898 100644 --- a/meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> +++ b/meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> @@ -104,6 +104,7 @@ RDEPENDS:${PN} = "\ net-tools \ perl \ python3-core \ + python3-kirk \ procps \ quota \ unzip \ -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#233871): https://lists.openembedded.org/g/openembedded-core/message/233871 Mute This Topic: https://lists.openembedded.org/mt/118498904/1050810 Group Owner: openembedded-core+owner@lists.openembedded.org<mailto:openembedded-core%2Bowner@lists.openembedded.org> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [bruce.ashfield@gmail.com<mailto:bruce.ashfield@gmail.com>] -=-=-=-=-=-=-=-=-=-=-=- -- - Thou shalt not follow the NULL pointer, for chaos and madness await thee at its end - "Use the force Harry" - Gandalf, Star Trek II
Hi, I added for transparency the "assisted by". I have tested the patch with the ltp math test set to be able to do verify it's output. It generates two files: a raw file with the log and a json file. The default output from kirk is a json file with the results, instead of dumping them into a stdout. I'm attaching the output with the patches: ls tmp/work/qemux86_64-oe-linux/core-image-minimal/1.0/testimage/ltp_log/ math.json math-raw.log I'll modify the commit message to reflect that. Daniel > -----Original Message----- > From: openembedded-core@lists.openembedded.org <openembedded- > core@lists.openembedded.org> On Behalf Of Alexander Kanavin via > lists.openembedded.org > Sent: Wednesday, 25 March 2026 14:02 > To: bruce.ashfield@gmail.com > Cc: Daniel Turull <daniel.turull@ericsson.com>; openembedded- > core@lists.openembedded.org; pratik.farkase@est.tech; > richard.purdie@linuxfoundation.org > Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk > > On Wed, 25 Mar 2026 at 13:58, Bruce Ashfield via lists.openembedded.org > <bruce.ashfield=gmail.com@lists.openembedded.org> wrote: > > For something like this to be considered for merging, I'd have > > expected a lot more information. > > > > Is the test coverage the same as before ? > > What are the results of the tests versus the previous harness ? > > Is it truly a drop in replacement for the previous runner ? > > .. etc > > > > Switching the executable is the easy part, ensuring that it actually > > works a replacement is the hard part. > > >> Assisted-by: Claude, Anthropic > > I have to add that this line ^^^ doesn't inspire confidence in the patch either. On > the contrary, it raises suspicions that the code was written quickly after the issue > was raised yesterday in the tech meeting, and wasn't actually tested well or at > all. The commit message > *must* reassure that it was. > > Alex
On Wed, Mar 25, 2026 at 9:44 AM Daniel Turull <daniel.turull@ericsson.com> wrote: > Hi, > > > > Kirk is official way to run ltp since runltp has been dropped. Richard > mentioned yesterday in the Weekly Project Engineering Sync if no one was > fixing the runltp issue, he will drop ltp from oe-core. > > > Yes, we know this, it's been mentioned for months. We are fundamentally too late for the LTS release in my opinion, since Richard would likely be left fixing issues with LTP failures that could gate the release. It isn't even the results that are the issue. How will this perform on a fully loaded machine (i.e the auto builder), will we get inconsistent results ? Intermittent failures ? Different results based on the architecture ? > The test coverage is the same since the logic there is not touched but the > output is different. Now it is a json file with the results, which is > machine readable without any extra parsing. Do we need a conversion from > the new to the old to keep compatibility? > Only if it impacts the reset of the QA or Autobuilder testing. If everything in the test stack continues to work with the new format, I'd say it is ok as-is. > > > I’m attaching the old test output with the new one for the math test suite. > > New: math-raw.log and math.json > > Old: math.old math-raw.log.old. > > > > You can see at the end of the old output: > This all needs to be in the commit itself. As well as a results summary from all the supported primary architectures. Bruce > > > ------------------------------------------- > > INFO: runltp script is deprecated, try kirk > > https://github.com/linux-test-project/kirk > > ------------------------------------------- > > > > Best regards, > > Daniel > > > > > > *From:* Bruce Ashfield <bruce.ashfield@gmail.com> > *Sent:* Wednesday, 25 March 2026 13:59 > *To:* Daniel Turull <daniel.turull@ericsson.com> > *Cc:* openembedded-core@lists.openembedded.org; pratik.farkase@est.tech; > richard.purdie@linuxfoundation.org > *Subject:* Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk > > > > > > > > On Wed, Mar 25, 2026 at 7:07 AM Daniel Turull via lists.openembedded.org > <daniel.turull=ericsson.com@lists.openembedded.org> wrote: > > From: Daniel Turull <daniel.turull@ericsson.com> > > runltp has been removed from ltp and kirk is the official tool to > invoke linux ltp tests. > > > > For something like this to be considered for merging, I'd have expected > > a lot more information. > > > > Is the test coverage the same as before ? > > What are the results of the tests versus the previous harness ? > > Is it truly a drop in replacement for the previous runner ? > > .. etc > > > > Switching the executable is the easy part, ensuring that it actually works > > a replacement is the hard part. > > > > Bruce > > > > > > > > See: > > https://github.com/linux-test-project/ltp/commit/6efd3605dc005c3ed135b463f182174e24bdce1b > > Signed-off-by: Daniel Turull <daniel.turull@ericsson.com> > Assisted-by: Claude, Anthropic > --- > meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++------- > meta/lib/oeqa/runtime/cases/ltp_stress.py | 16 +++++++++---- > meta/lib/oeqa/utils/logparser.py | 29 +++++++++++++++++++++++ > meta/recipes-extended/ltp/ltp_20260130.bb | 1 + > 4 files changed, 50 insertions(+), 12 deletions(-) > > diff --git a/meta/lib/oeqa/runtime/cases/ltp.py > b/meta/lib/oeqa/runtime/cases/ltp.py > index 0ffdbe23e4..11c4814090 100644 > --- a/meta/lib/oeqa/runtime/cases/ltp.py > +++ b/meta/lib/oeqa/runtime/cases/ltp.py > @@ -12,7 +12,7 @@ import pprint > from oeqa.runtime.case import OERuntimeTestCase > from oeqa.core.decorator.depends import OETestDepends > from oeqa.runtime.decorator.package import OEHasPackage > -from oeqa.utils.logparser import LtpParser > +from oeqa.utils.logparser import LtpKirkParser > > class LtpTestBase(OERuntimeTestCase): > > @@ -66,9 +66,9 @@ class LtpTest(LtpTestBase): > > def runltp(self, ltp_group): > # LTP appends to log files, so ensure we start with a clean > log > - self.target.deleteFiles("/opt/ltp/results/", ltp_group) > + self.target.deleteFiles("/opt/ltp/results/", "%s.json" % > ltp_group) > > - cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l > /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) > + cmd = 'kirk --run-suite %s --json-report > /opt/ltp/results/%s.json -n' % (ltp_group, ltp_group) > > starttime = time.time() > (status, output) = self.target.run(cmd, timeout=1200) > @@ -87,14 +87,14 @@ class LtpTest(LtpTestBase): > self.extras['ltpresult.rawlogs']['log'] = > self.extras['ltpresult.rawlogs']['log'] + output > > # Copy the machine-readable test results locally so we can > parse it > - dst = os.path.join(self.ltptest_log_dir, ltp_group) > - remote_src = "/opt/ltp/results/%s" % ltp_group > + dst = os.path.join(self.ltptest_log_dir, "%s.json" % > ltp_group) > + remote_src = "/opt/ltp/results/%s.json" % ltp_group > (status, output) = self.target.copyFrom(remote_src, dst, True) > if status: > msg = 'File could not be copied. Output: %s' % output > self.target.logger.warning(msg) > > - parser = LtpParser() > + parser = LtpKirkParser() > results, sections = parser.parse(dst) > > sections['duration'] = int(endtime-starttime) > @@ -113,9 +113,9 @@ class LtpTest(LtpTestBase): > > # LTP runtime tests > @OETestDepends(['ssh.SSHTest.test_ssh']) > - @OEHasPackage(["ltp"]) > + @OEHasPackage(["ltp", "python3-kirk"]) > def test_ltp_help(self): > - (status, output) = self.target.run('/opt/ltp/runltp --help') > + (status, output) = self.target.run('kirk --help') > msg = 'Failed to get ltp help. Output: %s' % output > self.assertEqual(status, 0, msg=msg) > > diff --git a/meta/lib/oeqa/runtime/cases/ltp_stress.py > b/meta/lib/oeqa/runtime/cases/ltp_stress.py > index ce6f4bf59d..cf84ec1182 100644 > --- a/meta/lib/oeqa/runtime/cases/ltp_stress.py > +++ b/meta/lib/oeqa/runtime/cases/ltp_stress.py > @@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCase > from oeqa.core.decorator.depends import OETestDepends > from oeqa.runtime.decorator.package import OEHasPackage > from oeqa.core.decorator.data import skipIfQemu > -from oeqa.utils.logparser import LtpParser > +from oeqa.utils.logparser import LtpKirkParser > > class LtpStressBase(OERuntimeTestCase): > > @@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase): > class LtpStressTest(LtpStressBase): > > def runltp(self, stress_group): > - cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee > /opt/ltp/results/%s' % (stress_group, stress_group) > + cmd = 'kirk --run-suite %s --json-report > /opt/ltp/results/%s.json -n' % (stress_group, stress_group) > starttime = time.time() > (status, output) = self.target.run(cmd) > endtime = time.time() > @@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase): > > self.extras['ltpstressresult.rawlogs']['log'] = > self.extras['ltpstressresult.rawlogs']['log'] + output > > - parser = LtpParser() > - results, sections = > parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group)) > + # Copy kirk JSON report from target > + dst = os.path.join(self.ltptest_log_dir, "%s.json" % > stress_group) > + remote_src = "/opt/ltp/results/%s.json" % stress_group > + (status, output) = self.target.copyFrom(remote_src, dst, True) > + if status: > + msg = 'File could not be copied. Output: %s' % output > + self.target.logger.warning(msg) > + > + parser = LtpKirkParser() > + results, sections = parser.parse(dst) > > runtime = int(endtime-starttime) > sections['duration'] = runtime > diff --git a/meta/lib/oeqa/utils/logparser.py > b/meta/lib/oeqa/utils/logparser.py > index c479864162..a907421fab 100644 > --- a/meta/lib/oeqa/utils/logparser.py > +++ b/meta/lib/oeqa/utils/logparser.py > @@ -5,6 +5,7 @@ > # > > import enum > +import json > import os > import re > > @@ -158,6 +159,34 @@ class LtpParser: > return results, section > > > +class LtpKirkParser: > + """Parse kirk JSON report into the same format as LtpParser.""" > + > + STATUS_MAP = { > + "pass": "PASSED", > + "fail": "FAILED", > + "brok": "FAILED", > + "conf": "SKIPPED", > + "warn": "PASSED", > + } > + > + def parse(self, jsonfile): > + with open(jsonfile, errors="replace") as f: > + report = json.load(f) > + > + results = {} > + section = {"duration": 0, "log": ""} > + > + for entry in report.get("results", []): > + results[entry["test_fqn"]] = > self.STATUS_MAP.get(entry.get("status", ""), "FAILED") > + test = entry.get("test", {}) > + section["log"] += test.get("log", "") > + > + section["duration"] = int(report.get("stats", {}).get("runtime", > 0)) > + > + return results, section > + > + > # ltp Compliance log parsing > class LtpComplianceParser(object): > def __init__(self): > diff --git a/meta/recipes-extended/ltp/ltp_20260130.bb > b/meta/recipes-extended/ltp/ltp_20260130.bb > index dcd1e81398..1ff20a7898 100644 > --- a/meta/recipes-extended/ltp/ltp_20260130.bb > +++ b/meta/recipes-extended/ltp/ltp_20260130.bb > @@ -104,6 +104,7 @@ RDEPENDS:${PN} = "\ > net-tools \ > perl \ > python3-core \ > + python3-kirk \ > procps \ > quota \ > unzip \ > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#233871): > https://lists.openembedded.org/g/openembedded-core/message/233871 > Mute This Topic: https://lists.openembedded.org/mt/118498904/1050810 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ > bruce.ashfield@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > > > > > -- > > - Thou shalt not follow the NULL pointer, for chaos and madness await thee > at its end > - "Use the force Harry" - Gandalf, Star Trek II >
Ok. I’ll run it with the supported architectures and update the commit message with the information. Daniel From: openembedded-core@lists.openembedded.org <openembedded-core@lists.openembedded.org> On Behalf Of Bruce Ashfield via lists.openembedded.org Sent: Wednesday, 25 March 2026 15:22 To: Daniel Turull <daniel.turull@ericsson.com> Cc: openembedded-core@lists.openembedded.org; pratik.farkase@est.tech; richard.purdie@linuxfoundation.org Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk On Wed, Mar 25, 2026 at 9:44 AM Daniel Turull <daniel.turull@ericsson.com<mailto:daniel.turull@ericsson.com>> wrote: Hi, Kirk is official way to run ltp since runltp has been dropped. Richard mentioned yesterday in the Weekly Project Engineering Sync if no one was fixing the runltp issue, he will drop ltp from oe-core. Yes, we know this, it's been mentioned for months. We are fundamentally too late for the LTS release in my opinion, since Richard would likely be left fixing issues with LTP failures that could gate the release. It isn't even the results that are the issue. How will this perform on a fully loaded machine (i.e the auto builder), will we get inconsistent results ? Intermittent failures ? Different results based on the architecture ? The test coverage is the same since the logic there is not touched but the output is different. Now it is a json file with the results, which is machine readable without any extra parsing. Do we need a conversion from the new to the old to keep compatibility? Only if it impacts the reset of the QA or Autobuilder testing. If everything in the test stack continues to work with the new format, I'd say it is ok as-is. I’m attaching the old test output with the new one for the math test suite. New: math-raw.log and math.json Old: math.old math-raw.log.old. You can see at the end of the old output: This all needs to be in the commit itself. As well as a results summary from all the supported primary architectures. Bruce ------------------------------------------- INFO: runltp script is deprecated, try kirk https://github.com/linux-test-project/kirk ------------------------------------------- Best regards, Daniel From: Bruce Ashfield <bruce.ashfield@gmail.com<mailto:bruce.ashfield@gmail.com>> Sent: Wednesday, 25 March 2026 13:59 To: Daniel Turull <daniel.turull@ericsson.com<mailto:daniel.turull@ericsson.com>> Cc: openembedded-core@lists.openembedded.org<mailto:openembedded-core@lists.openembedded.org>; pratik.farkase@est.tech<mailto:pratik.farkase@est.tech>; richard.purdie@linuxfoundation.org<mailto:richard.purdie@linuxfoundation.org> Subject: Re: [OE-core] [PATCH 2/2] oeqa: replace runltp with kirk On Wed, Mar 25, 2026 at 7:07 AM Daniel Turull via lists.openembedded.org<http://lists.openembedded.org/> <daniel.turull=ericsson.com@lists.openembedded.org<mailto:ericsson.com@lists.openembedded.org>> wrote: From: Daniel Turull <daniel.turull@ericsson.com<mailto:daniel.turull@ericsson.com>> runltp has been removed from ltp and kirk is the official tool to invoke linux ltp tests. For something like this to be considered for merging, I'd have expected a lot more information. Is the test coverage the same as before ? What are the results of the tests versus the previous harness ? Is it truly a drop in replacement for the previous runner ? .. etc Switching the executable is the easy part, ensuring that it actually works a replacement is the hard part. Bruce See: https://github.com/linux-test-project/ltp/commit/6efd3605dc005c3ed135b463f182174e24bdce1b Signed-off-by: Daniel Turull <daniel.turull@ericsson.com<mailto:daniel.turull@ericsson.com>> Assisted-by: Claude, Anthropic --- meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++------- meta/lib/oeqa/runtime/cases/ltp_stress.py | 16 +++++++++---- meta/lib/oeqa/utils/logparser.py | 29 +++++++++++++++++++++++ meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> | 1 + 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/meta/lib/oeqa/runtime/cases/ltp.py b/meta/lib/oeqa/runtime/cases/ltp.py index 0ffdbe23e4..11c4814090 100644 --- a/meta/lib/oeqa/runtime/cases/ltp.py +++ b/meta/lib/oeqa/runtime/cases/ltp.py @@ -12,7 +12,7 @@ import pprint from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpTestBase(OERuntimeTestCase): @@ -66,9 +66,9 @@ class LtpTest(LtpTestBase): def runltp(self, ltp_group): # LTP appends to log files, so ensure we start with a clean log - self.target.deleteFiles("/opt/ltp/results/", ltp_group) + self.target.deleteFiles("/opt/ltp/results/", "%s.json" % ltp_group) - cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (ltp_group, ltp_group) starttime = time.time() (status, output) = self.target.run(cmd, timeout=1200) @@ -87,14 +87,14 @@ class LtpTest(LtpTestBase): self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output # Copy the machine-readable test results locally so we can parse it - dst = os.path.join(self.ltptest_log_dir, ltp_group) - remote_src = "/opt/ltp/results/%s" % ltp_group + dst = os.path.join(self.ltptest_log_dir, "%s.json" % ltp_group) + remote_src = "/opt/ltp/results/%s.json" % ltp_group (status, output) = self.target.copyFrom(remote_src, dst, True) if status: msg = 'File could not be copied. Output: %s' % output self.target.logger.warning(msg) - parser = LtpParser() + parser = LtpKirkParser() results, sections = parser.parse(dst) sections['duration'] = int(endtime-starttime) @@ -113,9 +113,9 @@ class LtpTest(LtpTestBase): # LTP runtime tests @OETestDepends(['ssh.SSHTest.test_ssh']) - @OEHasPackage(["ltp"]) + @OEHasPackage(["ltp", "python3-kirk"]) def test_ltp_help(self): - (status, output) = self.target.run('/opt/ltp/runltp --help') + (status, output) = self.target.run('kirk --help') msg = 'Failed to get ltp help. Output: %s' % output self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/ltp_stress.py b/meta/lib/oeqa/runtime/cases/ltp_stress.py index ce6f4bf59d..cf84ec1182 100644 --- a/meta/lib/oeqa/runtime/cases/ltp_stress.py +++ b/meta/lib/oeqa/runtime/cases/ltp_stress.py @@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage from oeqa.core.decorator.data import skipIfQemu -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpStressBase(OERuntimeTestCase): @@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase): class LtpStressTest(LtpStressBase): def runltp(self, stress_group): - cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group, stress_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (stress_group, stress_group) starttime = time.time() (status, output) = self.target.run(cmd) endtime = time.time() @@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase): self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output - parser = LtpParser() - results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group)) + # Copy kirk JSON report from target + dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group) + remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst) runtime = int(endtime-starttime) sections['duration'] = runtime diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py index c479864162..a907421fab 100644 --- a/meta/lib/oeqa/utils/logparser.py +++ b/meta/lib/oeqa/utils/logparser.py @@ -5,6 +5,7 @@ # import enum +import json import os import re @@ -158,6 +159,34 @@ class LtpParser: return results, section +class LtpKirkParser: + """Parse kirk JSON report into the same format as LtpParser.""" + + STATUS_MAP = { + "pass": "PASSED", + "fail": "FAILED", + "brok": "FAILED", + "conf": "SKIPPED", + "warn": "PASSED", + } + + def parse(self, jsonfile): + with open(jsonfile, errors="replace") as f: + report = json.load(f) + + results = {} + section = {"duration": 0, "log": ""} + + for entry in report.get("results", []): + results[entry["test_fqn"]] = self.STATUS_MAP.get(entry.get("status", ""), "FAILED") + test = entry.get("test", {}) + section["log"] += test.get("log", "") + + section["duration"] = int(report.get("stats", {}).get("runtime", 0)) + + return results, section + + # ltp Compliance log parsing class LtpComplianceParser(object): def __init__(self): diff --git a/meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> b/meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> index dcd1e81398..1ff20a7898 100644 --- a/meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> +++ b/meta/recipes-extended/ltp/ltp_20260130.bb<http://ltp_20260130.bb/> @@ -104,6 +104,7 @@ RDEPENDS:${PN} = "\ net-tools \ perl \ python3-core \ + python3-kirk \ procps \ quota \ unzip \ -- - Thou shalt not follow the NULL pointer, for chaos and madness await thee at its end - "Use the force Harry" - Gandalf, Star Trek II -- - Thou shalt not follow the NULL pointer, for chaos and madness await thee at its end - "Use the force Harry" - Gandalf, Star Trek II
On Wed, 2026-03-25 at 09:40 +0100, daniel.turull@ericsson.com wrote: > From: Daniel Turull <daniel.turull@ericsson.com> > > runltp has been removed from ltp and kirk is the official tool to > invoke linux ltp tests. > > See: > https://github.com/linux-test-project/ltp/commit/6efd3605dc005c3ed135b463f182174e24bdce1b > > Signed-off-by: Daniel Turull <daniel.turull@ericsson.com> > Assisted-by: Claude, Anthropic > --- > meta/lib/oeqa/runtime/cases/ltp.py | 16 ++++++------- > meta/lib/oeqa/runtime/cases/ltp_stress.py | 16 +++++++++---- > meta/lib/oeqa/utils/logparser.py | 29 +++++++++++++++++++++++ > meta/recipes-extended/ltp/ltp_20260130.bb | 1 + > 4 files changed, 50 insertions(+), 12 deletions(-) > > diff --git a/meta/lib/oeqa/runtime/cases/ltp.py b/meta/lib/oeqa/runtime/cases/ltp.py > index 0ffdbe23e4..11c4814090 100644 > --- a/meta/lib/oeqa/runtime/cases/ltp.py > +++ b/meta/lib/oeqa/runtime/cases/ltp.py > @@ -12,7 +12,7 @@ import pprint > from oeqa.runtime.case import OERuntimeTestCase > from oeqa.core.decorator.depends import OETestDepends > from oeqa.runtime.decorator.package import OEHasPackage > -from oeqa.utils.logparser import LtpParser > +from oeqa.utils.logparser import LtpKirkParser > > class LtpTestBase(OERuntimeTestCase): > > @@ -66,9 +66,9 @@ class LtpTest(LtpTestBase): > > def runltp(self, ltp_group): > # LTP appends to log files, so ensure we start with a clean log > - self.target.deleteFiles("/opt/ltp/results/", ltp_group) > + self.target.deleteFiles("/opt/ltp/results/", "%s.json" % ltp_group) > > - cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) > + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (ltp_group, ltp_group) > > starttime = time.time() > (status, output) = self.target.run(cmd, timeout=1200) > @@ -87,14 +87,14 @@ class LtpTest(LtpTestBase): > self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output > > # Copy the machine-readable test results locally so we can parse it > - dst = os.path.join(self.ltptest_log_dir, ltp_group) > - remote_src = "/opt/ltp/results/%s" % ltp_group > + dst = os.path.join(self.ltptest_log_dir, "%s.json" % ltp_group) > + remote_src = "/opt/ltp/results/%s.json" % ltp_group > (status, output) = self.target.copyFrom(remote_src, dst, True) > if status: > msg = 'File could not be copied. Output: %s' % output > self.target.logger.warning(msg) > > - parser = LtpParser() > + parser = LtpKirkParser() > results, sections = parser.parse(dst) > > sections['duration'] = int(endtime-starttime) > @@ -113,9 +113,9 @@ class LtpTest(LtpTestBase): > > # LTP runtime tests > @OETestDepends(['ssh.SSHTest.test_ssh']) > - @OEHasPackage(["ltp"]) > + @OEHasPackage(["ltp", "python3-kirk"]) > def test_ltp_help(self): > - (status, output) = self.target.run('/opt/ltp/runltp --help') > + (status, output) = self.target.run('kirk --help') > msg = 'Failed to get ltp help. Output: %s' % output > self.assertEqual(status, 0, msg=msg) > > diff --git a/meta/lib/oeqa/runtime/cases/ltp_stress.py b/meta/lib/oeqa/runtime/cases/ltp_stress.py > index ce6f4bf59d..cf84ec1182 100644 > --- a/meta/lib/oeqa/runtime/cases/ltp_stress.py > +++ b/meta/lib/oeqa/runtime/cases/ltp_stress.py > @@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCase > from oeqa.core.decorator.depends import OETestDepends > from oeqa.runtime.decorator.package import OEHasPackage > from oeqa.core.decorator.data import skipIfQemu > -from oeqa.utils.logparser import LtpParser > +from oeqa.utils.logparser import LtpKirkParser > > class LtpStressBase(OERuntimeTestCase): > > @@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase): > class LtpStressTest(LtpStressBase): > > def runltp(self, stress_group): > - cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group, stress_group) > + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (stress_group, stress_group) > starttime = time.time() > (status, output) = self.target.run(cmd) > endtime = time.time() > @@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase): > > self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output > > - parser = LtpParser() > - results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group)) > + # Copy kirk JSON report from target > + dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group) > + remote_src = "/opt/ltp/results/%s.json" % stress_group > + (status, output) = self.target.copyFrom(remote_src, dst, True) > + if status: > + msg = 'File could not be copied. Output: %s' % output > + self.target.logger.warning(msg) > + > + parser = LtpKirkParser() > + results, sections = parser.parse(dst) > > runtime = int(endtime-starttime) > sections['duration'] = runtime > diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py > index c479864162..a907421fab 100644 > --- a/meta/lib/oeqa/utils/logparser.py > +++ b/meta/lib/oeqa/utils/logparser.py > @@ -5,6 +5,7 @@ > # > > import enum > +import json > import os > import re > > @@ -158,6 +159,34 @@ class LtpParser: > return results, section > > > +class LtpKirkParser: > + """Parse kirk JSON report into the same format as LtpParser.""" > + > + STATUS_MAP = { > + "pass": "PASSED", > + "fail": "FAILED", > + "brok": "FAILED", > + "conf": "SKIPPED", > + "warn": "PASSED", > + } > + > + def parse(self, jsonfile): > + with open(jsonfile, errors="replace") as f: > + report = json.load(f) > + > + results = {} > + section = {"duration": 0, "log": ""} > + > + for entry in report.get("results", []): > + results[entry["test_fqn"]] = self.STATUS_MAP.get(entry.get("status", ""), "FAILED") > + test = entry.get("test", {}) > + section["log"] += test.get("log", "") > + > + section["duration"] = int(report.get("stats", {}).get("runtime", 0)) > + > + return results, section > + > + FWIW we can probably just update the LtpParser class directly rather than creating a wrapper around it since runltp will be no more in the next release anyway. Cheers, Richard
Sure, I’ll try to send a new patch tomorrow with your suggestion and all the comments that I have received. We are running the testimage with ltp for qemuarm64, qemux86-64 and qemuriscv64. Best regards, Daniel
diff --git a/meta/lib/oeqa/runtime/cases/ltp.py b/meta/lib/oeqa/runtime/cases/ltp.py index 0ffdbe23e4..11c4814090 100644 --- a/meta/lib/oeqa/runtime/cases/ltp.py +++ b/meta/lib/oeqa/runtime/cases/ltp.py @@ -12,7 +12,7 @@ import pprint from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpTestBase(OERuntimeTestCase): @@ -66,9 +66,9 @@ class LtpTest(LtpTestBase): def runltp(self, ltp_group): # LTP appends to log files, so ensure we start with a clean log - self.target.deleteFiles("/opt/ltp/results/", ltp_group) + self.target.deleteFiles("/opt/ltp/results/", "%s.json" % ltp_group) - cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (ltp_group, ltp_group) starttime = time.time() (status, output) = self.target.run(cmd, timeout=1200) @@ -87,14 +87,14 @@ class LtpTest(LtpTestBase): self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output # Copy the machine-readable test results locally so we can parse it - dst = os.path.join(self.ltptest_log_dir, ltp_group) - remote_src = "/opt/ltp/results/%s" % ltp_group + dst = os.path.join(self.ltptest_log_dir, "%s.json" % ltp_group) + remote_src = "/opt/ltp/results/%s.json" % ltp_group (status, output) = self.target.copyFrom(remote_src, dst, True) if status: msg = 'File could not be copied. Output: %s' % output self.target.logger.warning(msg) - parser = LtpParser() + parser = LtpKirkParser() results, sections = parser.parse(dst) sections['duration'] = int(endtime-starttime) @@ -113,9 +113,9 @@ class LtpTest(LtpTestBase): # LTP runtime tests @OETestDepends(['ssh.SSHTest.test_ssh']) - @OEHasPackage(["ltp"]) + @OEHasPackage(["ltp", "python3-kirk"]) def test_ltp_help(self): - (status, output) = self.target.run('/opt/ltp/runltp --help') + (status, output) = self.target.run('kirk --help') msg = 'Failed to get ltp help. Output: %s' % output self.assertEqual(status, 0, msg=msg) diff --git a/meta/lib/oeqa/runtime/cases/ltp_stress.py b/meta/lib/oeqa/runtime/cases/ltp_stress.py index ce6f4bf59d..cf84ec1182 100644 --- a/meta/lib/oeqa/runtime/cases/ltp_stress.py +++ b/meta/lib/oeqa/runtime/cases/ltp_stress.py @@ -13,7 +13,7 @@ from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends from oeqa.runtime.decorator.package import OEHasPackage from oeqa.core.decorator.data import skipIfQemu -from oeqa.utils.logparser import LtpParser +from oeqa.utils.logparser import LtpKirkParser class LtpStressBase(OERuntimeTestCase): @@ -60,7 +60,7 @@ class LtpStressBase(OERuntimeTestCase): class LtpStressTest(LtpStressBase): def runltp(self, stress_group): - cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group, stress_group) + cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n' % (stress_group, stress_group) starttime = time.time() (status, output) = self.target.run(cmd) endtime = time.time() @@ -69,8 +69,16 @@ class LtpStressTest(LtpStressBase): self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output - parser = LtpParser() - results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group)) + # Copy kirk JSON report from target + dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group) + remote_src = "/opt/ltp/results/%s.json" % stress_group + (status, output) = self.target.copyFrom(remote_src, dst, True) + if status: + msg = 'File could not be copied. Output: %s' % output + self.target.logger.warning(msg) + + parser = LtpKirkParser() + results, sections = parser.parse(dst) runtime = int(endtime-starttime) sections['duration'] = runtime diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py index c479864162..a907421fab 100644 --- a/meta/lib/oeqa/utils/logparser.py +++ b/meta/lib/oeqa/utils/logparser.py @@ -5,6 +5,7 @@ # import enum +import json import os import re @@ -158,6 +159,34 @@ class LtpParser: return results, section +class LtpKirkParser: + """Parse kirk JSON report into the same format as LtpParser.""" + + STATUS_MAP = { + "pass": "PASSED", + "fail": "FAILED", + "brok": "FAILED", + "conf": "SKIPPED", + "warn": "PASSED", + } + + def parse(self, jsonfile): + with open(jsonfile, errors="replace") as f: + report = json.load(f) + + results = {} + section = {"duration": 0, "log": ""} + + for entry in report.get("results", []): + results[entry["test_fqn"]] = self.STATUS_MAP.get(entry.get("status", ""), "FAILED") + test = entry.get("test", {}) + section["log"] += test.get("log", "") + + section["duration"] = int(report.get("stats", {}).get("runtime", 0)) + + return results, section + + # ltp Compliance log parsing class LtpComplianceParser(object): def __init__(self): diff --git a/meta/recipes-extended/ltp/ltp_20260130.bb b/meta/recipes-extended/ltp/ltp_20260130.bb index dcd1e81398..1ff20a7898 100644 --- a/meta/recipes-extended/ltp/ltp_20260130.bb +++ b/meta/recipes-extended/ltp/ltp_20260130.bb @@ -104,6 +104,7 @@ RDEPENDS:${PN} = "\ net-tools \ perl \ python3-core \ + python3-kirk \ procps \ quota \ unzip \