summaryrefslogtreecommitdiffstats
path: root/scripts/lib/resulttool/regression.py
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/lib/resulttool/regression.py')
-rw-r--r--scripts/lib/resulttool/regression.py19
1 files changed, 14 insertions, 5 deletions
diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py
index d7f0b16ca9..1facbcd85e 100644
--- a/scripts/lib/resulttool/regression.py
+++ b/scripts/lib/resulttool/regression.py
@@ -190,11 +190,20 @@ def compare_result(logger, base_name, target_name, base_result, target_result):
190 else: 190 else:
191 logger.error('Failed to retrieved base test case status: %s' % k) 191 logger.error('Failed to retrieved base test case status: %s' % k)
192 if result: 192 if result:
193 resultstring = "Regression: %s\n %s\n" % (base_name, target_name) 193 new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values())
194 for k in sorted(result): 194 # Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...)
195 resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target']) 195 if new_pass_count < len(result):
196 resultstring = "Regression: %s\n %s\n" % (base_name, target_name)
197 for k in sorted(result):
198 if not result[k]['target'] or not result[k]['target'].startswith("PASS"):
199 resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target'])
200 if new_pass_count > 0:
201 resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n'
202 else:
203 resultstring = "Improvement: %s\n %s\n (+%d test(s) passing)" % (base_name, target_name, new_pass_count)
204 result = None
196 else: 205 else:
197 resultstring = "Match: %s\n %s" % (base_name, target_name) 206 resultstring = "Match: %s\n %s" % (base_name, target_name)
198 return result, resultstring 207 return result, resultstring
199 208
200def get_results(logger, source): 209def get_results(logger, source):
@@ -269,9 +278,9 @@ def regression_common(args, logger, base_results, target_results):
269 else: 278 else:
270 notfound.append("%s not found in target" % a) 279 notfound.append("%s not found in target" % a)
271 print("\n".join(sorted(matches))) 280 print("\n".join(sorted(matches)))
281 print("\n")
272 print("\n".join(sorted(regressions))) 282 print("\n".join(sorted(regressions)))
273 print("\n".join(sorted(notfound))) 283 print("\n".join(sorted(notfound)))
274
275 return 0 284 return 0
276 285
277def regression_git(args, logger): 286def regression_git(args, logger):