@@ -9,11 +9,79 @@ import json
import os
import sys
import subprocess
-import errno
import tempfile
+import re
import utils
+def is_non_release_version(version):
+ p = re.compile('\d{8}-\d+')
+ return p.match(version) is not None
+
+def get_previous_tag(targetrepodir, version):
+ previousversion = None
+ previousmilestone = None
+ if version:
+ if is_non_release_version(version):
+ return subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=targetrepodir).decode('utf-8').strip()
+ compareversion, comparemilestone, _ = utils.get_version_from_string(version)
+ compareversionminor = compareversion[-1]
+ # After ignoring rc part, if we get a minor to 0 on point release (e.g 4.0.0),
+ # reject last digit since such versions do not exist
+ if len(compareversion) == 3 and compareversionminor == 0:
+ compareversion = compareversion[:-1]
+
+ # Process milestone if not first in current release
+ if comparemilestone and comparemilestone > 1:
+ previousversion = compareversion
+ previousmilestone = comparemilestone-1
+ # Process first milestone or release if not first in major release
+ elif compareversionminor > 0:
+ previousversion = compareversion[:-1] + [compareversion[-1] - 1]
+ # Otherwise : format it as tag (which must exist) and search previous tag
+ else:
+ comparetagstring = utils.get_tag_from_version(compareversion, comparemilestone)
+ return subprocess.check_output(["git", "describe", "--abbrev=0", comparetagstring + "^"], cwd=targetrepodir).decode('utf-8').strip()
+
+ return utils.get_tag_from_version(previousversion, previousmilestone)
+
+ # All other cases : merely check against latest tag reachable
+ defaultbaseversion, _, _ = utils.get_version_from_string(subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=targetrepodir).decode('utf-8').strip())
+ return utils.get_tag_from_version(defaultbaseversion, None)
+
+def get_sha1(targetrepodir, revision):
+ return subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=targetrepodir).decode('utf-8').strip()
+
+def fetch_testresults(resultdir, revision):
+ rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{revision}*"], cwd=resultdir).decode('utf-8').strip()
+ if not rawtags:
+ raise Exception(f"No reference found for commit {revision} in {resultdir}")
+ for ref in [rawtag.split()[1] for rawtag in rawtags.splitlines()]:
+ print(f"Fetching matching revisions: {ref}")
+ subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{ref}:{ref}"], cwd=resultdir)
+
+
+def generate_regression_report(resulttool, targetrepodir, basebranch, resultdir, outputdir, yoctoversion):
+ baseversion = get_previous_tag(targetrepodir, yoctoversion)
+ baserevision = get_sha1(targetrepodir, baseversion)
+ comparerevision = get_sha1(targetrepodir, basebranch)
+ print(f"Compare version : {basebranch} ({comparerevision})")
+ print(f"Base tag : {baseversion} ({baserevision})")
+
+ try:
+ """
+ Results directory is likely a shallow clone :
+ we need to fetch results corresponding to base revision before
+ running resulttool
+ """
+ fetch_testresults(resultdir, baserevision)
+ regreport = subprocess.check_output([resulttool, "regression-git", "-B", basebranch, "--commit", baserevision, "--commit2", comparerevision, resultdir])
+ with open(outputdir + "/testresult-regressions-report.txt", "wb") as f:
+ f.write(regreport)
+ except subprocess.CalledProcessError as e:
+ error = str(e)
+ print(f"Error while generating report between {baserevision} and {comparerevision} : {error}")
+
def send_qa_email():
parser = utils.ArgParser(description='Process test results and optionally send an email about the build to prompt QA to begin testing.')
@@ -57,16 +125,7 @@ def send_qa_email():
branch = repos['poky']['branch']
repo = repos['poky']['url']
- extraopts = None
basebranch, comparebranch = utils.getcomparisonbranch(ourconfig, repo, branch)
- if basebranch:
- extraopts = " --branch %s --commit %s" % (branch, revision)
- if comparebranch:
- extraopts = extraopts + " --branch2 %s" % (comparebranch)
- elif basebranch:
- print("No comparision branch found, comparing to %s" % basebranch)
- extraopts = extraopts + " --branch2 %s" % basebranch
-
report = subprocess.check_output([resulttool, "report", args.results_dir])
with open(args.results_dir + "/testresult-report.txt", "wb") as f:
f.write(report)
@@ -95,7 +154,6 @@ def send_qa_email():
subprocess.check_call(["git", "checkout", "master"], cwd=tempdir)
subprocess.check_call(["git", "branch", basebranch], cwd=tempdir)
subprocess.check_call(["git", "checkout", basebranch], cwd=tempdir)
- extraopts = None
subprocess.check_call([resulttool, "store", args.results_dir, tempdir])
if comparebranch:
@@ -105,10 +163,8 @@ def send_qa_email():
subprocess.check_call(["git", "push", "--all"], cwd=tempdir)
subprocess.check_call(["git", "push", "--tags"], cwd=tempdir)
- if extraopts:
- regreport = subprocess.check_output([resulttool, "regression-git", tempdir] + extraopts.split())
- with open(args.results_dir + "/testresult-regressions-report.txt", "wb") as f:
- f.write(regreport)
+ if basebranch:
+ generate_regression_report(resulttool, targetrepodir, basebranch, tempdir, args.results_dir, args.release)
finally:
subprocess.check_call(["rm", "-rf", tempdir])
@@ -478,3 +478,50 @@ def setup_buildtools_tarball(ourconfig, workername, btdir, checkonly=False):
pass
subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"])
enable_buildtools_tarball(btdir)
+
+def get_string_from_version(version, milestone=None, rc=None):
+ """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists,
+ those are major releases
+ """
+ if len(version) == 3 and version[-1] == 0:
+ version = version[:-1]
+
+ result = ".".join(list(map(str, version)))
+ if milestone:
+ result += "_M" + str(milestone)
+ if rc:
+ result += ".rc" + str(rc)
+ return result
+
+def get_tag_from_version(version, milestone):
+ if not milestone:
+ return "yocto-" + get_string_from_version(version, milestone)
+ return get_string_from_version(version, milestone)
+
+
+def get_version_from_string(raw_version):
+ """ Get version as list of int from raw_version.
+
+ Raw version _can_ be prefixed by "yocto-",
+ Raw version _can_ be suffixed by "_MX"
+ Raw version _can_ be suffixed by ".rcY"
+ """
+ version = None
+ milestone = None
+ rc = None
+ if raw_version[:6] == "yocto-":
+ raw_version = raw_version[6:]
+ raw_version = raw_version.split(".")
+ if raw_version[-1][:2] == "rc":
+ rc = int(raw_version[-1][-1])
+ raw_version = raw_version[:-1]
+ if raw_version[-1][-3:-1] == "_M":
+ milestone = int(raw_version[-1][-1])
+ raw_version = raw_version[:-1] + [raw_version[-1][:-3]]
+ version = list(map(int, raw_version))
+ """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists,
+ those are major releases
+ """
+ if len(version) == 3 and version[-1] == 0:
+ version = version[:-1]
+ return version, milestone, rc
\ No newline at end of file
Instead of only generating regressions reports against HEAD of relevant branch, compute most relevant tag (ie : release) against which we can check for regressions. General rules introduced are the following : - milestone release is checked against previous milestone if possible, otherwise against major release - point release is checked against previous point release if possible, otherwise against major release - major release is checked against previous major release - a non release build is checked against base branch Examples : - 4.1.2.rc1 is checked against yocto-4.1.1 - 4.1.2 is checked against yocto-4.1.1 - 4.1.1.rc1 is checked against yocto-4.1 - 4.1.1 is checked against yocto-4.1 - 4.1 is checked against yocto-4.0 - 4.1.rc4 is checked against yocto-4.0 - 4.1_M2.rc1 is checked against 4.1_M1 - 4.1_M2 is checked against 4.1_M1 - 4.1_M1.rc1 is checked against yocto-4.0 - 4.1_M1 is checked against yocto-4.0 Signed-off-by: Alexis Lothoré <alexis.lothore@bootlin.com> --- scripts/send_qa_email.py | 86 +++++++++++++++++++++++++++++++++------- scripts/utils.py | 47 ++++++++++++++++++++++ 2 files changed, 118 insertions(+), 15 deletions(-)