| Message ID | 20230124173017.134017-4-alexis.lothore@bootlin.com |
|---|---|
| State | New |
| Headers | show |
| Series | generate regression reports against proper releases | expand |
On Tue, 2023-01-24 at 18:30 +0100, Alexis Lothoré via lists.yoctoproject.org wrote: > Instead of only generating regressions reports against HEAD of relevant branch, compute > most relevant tag (ie : release) against which we can check for regressions. General rules > introduced are the following : > - milestone release is checked against previous milestone if possible, otherwise > against major release > - point release is checked against previous point release if possible, > otherwise against major release > - major release is checked against previous major release > - a non release build is checked against base branch > Examples : > - 4.1.2.rc1 is checked against yocto-4.1.1 > - 4.1.2 is checked against yocto-4.1.1 > - 4.1.1.rc1 is checked against yocto-4.1 > - 4.1.1 is checked against yocto-4.1 > - 4.1 is checked against yocto-4.0 > - 4.1.rc4 is checked against yocto-4.0 > - 4.1_M2.rc1 is checked against 4.1_M1 > - 4.1_M2 is checked against 4.1_M1 > - 4.1_M1.rc1 is checked against yocto-4.0 > - 4.1_M1 is checked against yocto-4.0 > > Signed-off-by: Alexis Lothoré <alexis.lothore@bootlin.com> > --- > scripts/send_qa_email.py | 86 +++++++++++++++++++++++++++++++++------- > scripts/utils.py | 47 ++++++++++++++++++++++ > 2 files changed, 118 insertions(+), 15 deletions(-) > > diff --git a/scripts/send_qa_email.py b/scripts/send_qa_email.py > index 4023918..199fe4e 100755 > --- a/scripts/send_qa_email.py > +++ b/scripts/send_qa_email.py > @@ -9,11 +9,79 @@ import json > import os > import sys > import subprocess > -import errno > import tempfile > +import re > > import utils > > +def is_non_release_version(version): > + p = re.compile('\d{8}-\d+') > + return p.match(version) is not None > + > +def get_previous_tag(targetrepodir, version): > + previousversion = None > + previousmilestone = None > + if version: > + if is_non_release_version(version): > + return subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=targetrepodir).decode('utf-8').strip() > + compareversion, comparemilestone, _ = utils.get_version_from_string(version) > + compareversionminor = compareversion[-1] > + # After ignoring rc part, if we get a minor to 0 on point release (e.g 4.0.0), > + # reject last digit since such versions do not exist > + if len(compareversion) == 3 and compareversionminor == 0: > + compareversion = compareversion[:-1] > + > + # Process milestone if not first in current release > + if comparemilestone and comparemilestone > 1: > + previousversion = compareversion > + previousmilestone = comparemilestone-1 > + # Process first milestone or release if not first in major release > + elif compareversionminor > 0: > + previousversion = compareversion[:-1] + [compareversion[-1] - 1] > + # Otherwise : format it as tag (which must exist) and search previous tag > + else: > + comparetagstring = utils.get_tag_from_version(compareversion, comparemilestone) > + return subprocess.check_output(["git", "describe", "--abbrev=0", comparetagstring + "^"], cwd=targetrepodir).decode('utf-8').strip() > + > + return utils.get_tag_from_version(previousversion, previousmilestone) > + > + # All other cases : merely check against latest tag reachable > + defaultbaseversion, _, _ = utils.get_version_from_string(subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=targetrepodir).decode('utf-8').strip()) > + return utils.get_tag_from_version(defaultbaseversion, None) > + > +def get_sha1(targetrepodir, revision): > + return subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=targetrepodir).decode('utf-8').strip() > + > +def fetch_testresults(resultdir, revision): > + rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{revision}*"], cwd=resultdir).decode('utf-8').strip() > + if not rawtags: > + raise Exception(f"No reference found for commit {revision} in {resultdir}") > + for ref in [rawtag.split()[1] for rawtag in rawtags.splitlines()]: > + print(f"Fetching matching revisions: {ref}") > + subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{ref}:{ref}"], cwd=resultdir) > + > + > +def generate_regression_report(resulttool, targetrepodir, basebranch, resultdir, outputdir, yoctoversion): > + baseversion = get_previous_tag(targetrepodir, yoctoversion) > + baserevision = get_sha1(targetrepodir, baseversion) > + comparerevision = get_sha1(targetrepodir, basebranch) > + print(f"Compare version : {basebranch} ({comparerevision})") > + print(f"Base tag : {baseversion} ({baserevision})") > + > + try: > + """ > + Results directory is likely a shallow clone : > + we need to fetch results corresponding to base revision before > + running resulttool > + """ > + fetch_testresults(resultdir, baserevision) > + regreport = subprocess.check_output([resulttool, "regression-git", "-B", basebranch, "--commit", baserevision, "--commit2", comparerevision, resultdir]) > + with open(outputdir + "/testresult-regressions-report.txt", "wb") as f: > + f.write(regreport) > + except subprocess.CalledProcessError as e: > + error = str(e) > + print(f"Error while generating report between {baserevision} and {comparerevision} : {error}") > + > > def send_qa_email(): > parser = utils.ArgParser(description='Process test results and optionally send an email about the build to prompt QA to begin testing.') > @@ -57,16 +125,7 @@ def send_qa_email(): > branch = repos['poky']['branch'] > repo = repos['poky']['url'] > > - extraopts = None > basebranch, comparebranch = utils.getcomparisonbranch(ourconfig, repo, branch) > - if basebranch: > - extraopts = " --branch %s --commit %s" % (branch, revision) > - if comparebranch: > - extraopts = extraopts + " --branch2 %s" % (comparebranch) > - elif basebranch: > - print("No comparision branch found, comparing to %s" % basebranch) > - extraopts = extraopts + " --branch2 %s" % basebranch > - > report = subprocess.check_output([resulttool, "report", args.results_dir]) > with open(args.results_dir + "/testresult-report.txt", "wb") as f: > f.write(report) > There looks to be a potential issue here since you drop the use of comparebranch entirely. This comes from config.json: "BUILD_HISTORY_FORKPUSH" : {"poky-contrib:ross/mut" : "poky:master", "poky:master-next" : "poky:master"}, and basically what it says is that master-next is a force push branch which should be compared against master (and ross/mut should be compared against master too). We need this code so that when we build test branches, we run the regression comparison against the last known master test results. We should probably put Alexandre's test branch in there too. Cheers, Richard
Hi Richard, thanks for the feedback On 1/26/23 23:47, Richard Purdie wrote: > On Tue, 2023-01-24 at 18:30 +0100, Alexis Lothoré via lists.yoctoproject.org wrote: >> report = subprocess.check_output([resulttool, "report", args.results_dir]) >> with open(args.results_dir + "/testresult-report.txt", "wb") as f: >> f.write(report) >> > > There looks to be a potential issue here since you drop the use of > comparebranch entirely. This comes from config.json: > > "BUILD_HISTORY_FORKPUSH" : {"poky-contrib:ross/mut" : "poky:master", "poky:master-next" : "poky:master"}, > > and basically what it says is that master-next is a force push branch > which should be compared against master (and ross/mut should be > compared against master too). > > We need this code so that when we build test branches, we run the > regression comparison against the last known master test results. We > should probably put Alexandre's test branch in there too. I may have missed that. I need to dig a bit to understand exactly what is expected here in order to fix it. Noted, I will take a look at it. Regards
diff --git a/scripts/send_qa_email.py b/scripts/send_qa_email.py index 4023918..199fe4e 100755 --- a/scripts/send_qa_email.py +++ b/scripts/send_qa_email.py @@ -9,11 +9,79 @@ import json import os import sys import subprocess -import errno import tempfile +import re import utils +def is_non_release_version(version): + p = re.compile('\d{8}-\d+') + return p.match(version) is not None + +def get_previous_tag(targetrepodir, version): + previousversion = None + previousmilestone = None + if version: + if is_non_release_version(version): + return subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=targetrepodir).decode('utf-8').strip() + compareversion, comparemilestone, _ = utils.get_version_from_string(version) + compareversionminor = compareversion[-1] + # After ignoring rc part, if we get a minor to 0 on point release (e.g 4.0.0), + # reject last digit since such versions do not exist + if len(compareversion) == 3 and compareversionminor == 0: + compareversion = compareversion[:-1] + + # Process milestone if not first in current release + if comparemilestone and comparemilestone > 1: + previousversion = compareversion + previousmilestone = comparemilestone-1 + # Process first milestone or release if not first in major release + elif compareversionminor > 0: + previousversion = compareversion[:-1] + [compareversion[-1] - 1] + # Otherwise : format it as tag (which must exist) and search previous tag + else: + comparetagstring = utils.get_tag_from_version(compareversion, comparemilestone) + return subprocess.check_output(["git", "describe", "--abbrev=0", comparetagstring + "^"], cwd=targetrepodir).decode('utf-8').strip() + + return utils.get_tag_from_version(previousversion, previousmilestone) + + # All other cases : merely check against latest tag reachable + defaultbaseversion, _, _ = utils.get_version_from_string(subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=targetrepodir).decode('utf-8').strip()) + return utils.get_tag_from_version(defaultbaseversion, None) + +def get_sha1(targetrepodir, revision): + return subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=targetrepodir).decode('utf-8').strip() + +def fetch_testresults(resultdir, revision): + rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{revision}*"], cwd=resultdir).decode('utf-8').strip() + if not rawtags: + raise Exception(f"No reference found for commit {revision} in {resultdir}") + for ref in [rawtag.split()[1] for rawtag in rawtags.splitlines()]: + print(f"Fetching matching revisions: {ref}") + subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{ref}:{ref}"], cwd=resultdir) + + +def generate_regression_report(resulttool, targetrepodir, basebranch, resultdir, outputdir, yoctoversion): + baseversion = get_previous_tag(targetrepodir, yoctoversion) + baserevision = get_sha1(targetrepodir, baseversion) + comparerevision = get_sha1(targetrepodir, basebranch) + print(f"Compare version : {basebranch} ({comparerevision})") + print(f"Base tag : {baseversion} ({baserevision})") + + try: + """ + Results directory is likely a shallow clone : + we need to fetch results corresponding to base revision before + running resulttool + """ + fetch_testresults(resultdir, baserevision) + regreport = subprocess.check_output([resulttool, "regression-git", "-B", basebranch, "--commit", baserevision, "--commit2", comparerevision, resultdir]) + with open(outputdir + "/testresult-regressions-report.txt", "wb") as f: + f.write(regreport) + except subprocess.CalledProcessError as e: + error = str(e) + print(f"Error while generating report between {baserevision} and {comparerevision} : {error}") + def send_qa_email(): parser = utils.ArgParser(description='Process test results and optionally send an email about the build to prompt QA to begin testing.') @@ -57,16 +125,7 @@ def send_qa_email(): branch = repos['poky']['branch'] repo = repos['poky']['url'] - extraopts = None basebranch, comparebranch = utils.getcomparisonbranch(ourconfig, repo, branch) - if basebranch: - extraopts = " --branch %s --commit %s" % (branch, revision) - if comparebranch: - extraopts = extraopts + " --branch2 %s" % (comparebranch) - elif basebranch: - print("No comparision branch found, comparing to %s" % basebranch) - extraopts = extraopts + " --branch2 %s" % basebranch - report = subprocess.check_output([resulttool, "report", args.results_dir]) with open(args.results_dir + "/testresult-report.txt", "wb") as f: f.write(report) @@ -95,7 +154,6 @@ def send_qa_email(): subprocess.check_call(["git", "checkout", "master"], cwd=tempdir) subprocess.check_call(["git", "branch", basebranch], cwd=tempdir) subprocess.check_call(["git", "checkout", basebranch], cwd=tempdir) - extraopts = None subprocess.check_call([resulttool, "store", args.results_dir, tempdir]) if comparebranch: @@ -105,10 +163,8 @@ def send_qa_email(): subprocess.check_call(["git", "push", "--all"], cwd=tempdir) subprocess.check_call(["git", "push", "--tags"], cwd=tempdir) - if extraopts: - regreport = subprocess.check_output([resulttool, "regression-git", tempdir] + extraopts.split()) - with open(args.results_dir + "/testresult-regressions-report.txt", "wb") as f: - f.write(regreport) + if basebranch: + generate_regression_report(resulttool, targetrepodir, basebranch, tempdir, args.results_dir, args.release) finally: subprocess.check_call(["rm", "-rf", tempdir]) diff --git a/scripts/utils.py b/scripts/utils.py index c0ad14e..444b3ab 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -478,3 +478,50 @@ def setup_buildtools_tarball(ourconfig, workername, btdir, checkonly=False): pass subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"]) enable_buildtools_tarball(btdir) + +def get_string_from_version(version, milestone=None, rc=None): + """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists, + those are major releases + """ + if len(version) == 3 and version[-1] == 0: + version = version[:-1] + + result = ".".join(list(map(str, version))) + if milestone: + result += "_M" + str(milestone) + if rc: + result += ".rc" + str(rc) + return result + +def get_tag_from_version(version, milestone): + if not milestone: + return "yocto-" + get_string_from_version(version, milestone) + return get_string_from_version(version, milestone) + + +def get_version_from_string(raw_version): + """ Get version as list of int from raw_version. + + Raw version _can_ be prefixed by "yocto-", + Raw version _can_ be suffixed by "_MX" + Raw version _can_ be suffixed by ".rcY" + """ + version = None + milestone = None + rc = None + if raw_version[:6] == "yocto-": + raw_version = raw_version[6:] + raw_version = raw_version.split(".") + if raw_version[-1][:2] == "rc": + rc = int(raw_version[-1][-1]) + raw_version = raw_version[:-1] + if raw_version[-1][-3:-1] == "_M": + milestone = int(raw_version[-1][-1]) + raw_version = raw_version[:-1] + [raw_version[-1][:-3]] + version = list(map(int, raw_version)) + """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists, + those are major releases + """ + if len(version) == 3 and version[-1] == 0: + version = version[:-1] + return version, milestone, rc \ No newline at end of file
Instead of only generating regressions reports against HEAD of relevant branch, compute most relevant tag (ie : release) against which we can check for regressions. General rules introduced are the following : - milestone release is checked against previous milestone if possible, otherwise against major release - point release is checked against previous point release if possible, otherwise against major release - major release is checked against previous major release - a non release build is checked against base branch Examples : - 4.1.2.rc1 is checked against yocto-4.1.1 - 4.1.2 is checked against yocto-4.1.1 - 4.1.1.rc1 is checked against yocto-4.1 - 4.1.1 is checked against yocto-4.1 - 4.1 is checked against yocto-4.0 - 4.1.rc4 is checked against yocto-4.0 - 4.1_M2.rc1 is checked against 4.1_M1 - 4.1_M2 is checked against 4.1_M1 - 4.1_M1.rc1 is checked against yocto-4.0 - 4.1_M1 is checked against yocto-4.0 Signed-off-by: Alexis Lothoré <alexis.lothore@bootlin.com> --- scripts/send_qa_email.py | 86 +++++++++++++++++++++++++++++++++------- scripts/utils.py | 47 ++++++++++++++++++++++ 2 files changed, 118 insertions(+), 15 deletions(-)