diff options
| -rw-r--r-- | meta/lib/oeqa/utils/gitarchive.py | 71 | ||||
| -rwxr-xr-x | scripts/oe-build-perf-report | 90 |
2 files changed, 81 insertions, 80 deletions
diff --git a/meta/lib/oeqa/utils/gitarchive.py b/meta/lib/oeqa/utils/gitarchive.py index fddd608593..ff614d06bb 100644 --- a/meta/lib/oeqa/utils/gitarchive.py +++ b/meta/lib/oeqa/utils/gitarchive.py | |||
| @@ -17,6 +17,8 @@ | |||
| 17 | import os | 17 | import os |
| 18 | import re | 18 | import re |
| 19 | import sys | 19 | import sys |
| 20 | from operator import attrgetter | ||
| 21 | from collections import namedtuple | ||
| 20 | from oeqa.utils.git import GitRepo, GitError | 22 | from oeqa.utils.git import GitRepo, GitError |
| 21 | 23 | ||
| 22 | class ArchiveError(Exception): | 24 | class ArchiveError(Exception): |
| @@ -171,3 +173,72 @@ def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_ms | |||
| 171 | log.info("Pushing data to remote") | 173 | log.info("Pushing data to remote") |
| 172 | data_repo.run_cmd(cmd) | 174 | data_repo.run_cmd(cmd) |
| 173 | 175 | ||
| 176 | # Container class for tester revisions | ||
| 177 | TestedRev = namedtuple('TestedRev', 'commit commit_number tags') | ||
| 178 | |||
| 179 | def get_test_runs(log, repo, tag_name, **kwargs): | ||
| 180 | """Get a sorted list of test runs, matching given pattern""" | ||
| 181 | # First, get field names from the tag name pattern | ||
| 182 | field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)] | ||
| 183 | undef_fields = [f for f in field_names if f not in kwargs.keys()] | ||
| 184 | |||
| 185 | # Fields for formatting tag name pattern | ||
| 186 | str_fields = dict([(f, '*') for f in field_names]) | ||
| 187 | str_fields.update(kwargs) | ||
| 188 | |||
| 189 | # Get a list of all matching tags | ||
| 190 | tag_pattern = tag_name.format(**str_fields) | ||
| 191 | tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines() | ||
| 192 | log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern) | ||
| 193 | |||
| 194 | # Parse undefined fields from tag names | ||
| 195 | str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names]) | ||
| 196 | str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)' | ||
| 197 | str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})' | ||
| 198 | str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})' | ||
| 199 | str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})' | ||
| 200 | # escape parenthesis in fields in order to not messa up the regexp | ||
| 201 | fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()]) | ||
| 202 | str_fields.update(fixed_fields) | ||
| 203 | tag_re = re.compile(tag_name.format(**str_fields)) | ||
| 204 | |||
| 205 | # Parse fields from tags | ||
| 206 | revs = [] | ||
| 207 | for tag in tags: | ||
| 208 | m = tag_re.match(tag) | ||
| 209 | groups = m.groupdict() | ||
| 210 | revs.append([groups[f] for f in undef_fields] + [tag]) | ||
| 211 | |||
| 212 | # Return field names and a sorted list of revs | ||
| 213 | return undef_fields, sorted(revs) | ||
| 214 | |||
| 215 | def get_test_revs(log, repo, tag_name, **kwargs): | ||
| 216 | """Get list of all tested revisions""" | ||
| 217 | fields, runs = get_test_runs(log, repo, tag_name, **kwargs) | ||
| 218 | |||
| 219 | revs = {} | ||
| 220 | commit_i = fields.index('commit') | ||
| 221 | commit_num_i = fields.index('commit_number') | ||
| 222 | for run in runs: | ||
| 223 | commit = run[commit_i] | ||
| 224 | commit_num = run[commit_num_i] | ||
| 225 | tag = run[-1] | ||
| 226 | if not commit in revs: | ||
| 227 | revs[commit] = TestedRev(commit, commit_num, [tag]) | ||
| 228 | else: | ||
| 229 | assert commit_num == revs[commit].commit_number, "Commit numbers do not match" | ||
| 230 | revs[commit].tags.append(tag) | ||
| 231 | |||
| 232 | # Return in sorted table | ||
| 233 | revs = sorted(revs.values(), key=attrgetter('commit_number')) | ||
| 234 | log.debug("Found %d tested revisions:\n %s", len(revs), | ||
| 235 | "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs])) | ||
| 236 | return revs | ||
| 237 | |||
| 238 | def rev_find(revs, attr, val): | ||
| 239 | """Search from a list of TestedRev""" | ||
| 240 | for i, rev in enumerate(revs): | ||
| 241 | if getattr(rev, attr) == val: | ||
| 242 | return i | ||
| 243 | raise ValueError("Unable to find '{}' value '{}'".format(attr, val)) | ||
| 244 | |||
diff --git a/scripts/oe-build-perf-report b/scripts/oe-build-perf-report index af4f0469f6..f6fb458c2e 100755 --- a/scripts/oe-build-perf-report +++ b/scripts/oe-build-perf-report | |||
| @@ -37,58 +37,18 @@ from buildstats import BuildStats, diff_buildstats, BSVerDiff | |||
| 37 | scriptpath.add_oe_lib_path() | 37 | scriptpath.add_oe_lib_path() |
| 38 | 38 | ||
| 39 | from oeqa.utils.git import GitRepo, GitError | 39 | from oeqa.utils.git import GitRepo, GitError |
| 40 | import oeqa.utils.gitarchive as gitarchive | ||
| 40 | 41 | ||
| 41 | 42 | ||
| 42 | # Setup logging | 43 | # Setup logging |
| 43 | logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") | 44 | logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") |
| 44 | log = logging.getLogger('oe-build-perf-report') | 45 | log = logging.getLogger('oe-build-perf-report') |
| 45 | 46 | ||
| 46 | |||
| 47 | # Container class for tester revisions | ||
| 48 | TestedRev = namedtuple('TestedRev', 'commit commit_number tags') | ||
| 49 | |||
| 50 | |||
| 51 | def get_test_runs(repo, tag_name, **kwargs): | ||
| 52 | """Get a sorted list of test runs, matching given pattern""" | ||
| 53 | # First, get field names from the tag name pattern | ||
| 54 | field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)] | ||
| 55 | undef_fields = [f for f in field_names if f not in kwargs.keys()] | ||
| 56 | |||
| 57 | # Fields for formatting tag name pattern | ||
| 58 | str_fields = dict([(f, '*') for f in field_names]) | ||
| 59 | str_fields.update(kwargs) | ||
| 60 | |||
| 61 | # Get a list of all matching tags | ||
| 62 | tag_pattern = tag_name.format(**str_fields) | ||
| 63 | tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines() | ||
| 64 | log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern) | ||
| 65 | |||
| 66 | # Parse undefined fields from tag names | ||
| 67 | str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names]) | ||
| 68 | str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)' | ||
| 69 | str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})' | ||
| 70 | str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})' | ||
| 71 | str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})' | ||
| 72 | # escape parenthesis in fields in order to not messa up the regexp | ||
| 73 | fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()]) | ||
| 74 | str_fields.update(fixed_fields) | ||
| 75 | tag_re = re.compile(tag_name.format(**str_fields)) | ||
| 76 | |||
| 77 | # Parse fields from tags | ||
| 78 | revs = [] | ||
| 79 | for tag in tags: | ||
| 80 | m = tag_re.match(tag) | ||
| 81 | groups = m.groupdict() | ||
| 82 | revs.append([groups[f] for f in undef_fields] + [tag]) | ||
| 83 | |||
| 84 | # Return field names and a sorted list of revs | ||
| 85 | return undef_fields, sorted(revs) | ||
| 86 | |||
| 87 | def list_test_revs(repo, tag_name, verbosity, **kwargs): | 47 | def list_test_revs(repo, tag_name, verbosity, **kwargs): |
| 88 | """Get list of all tested revisions""" | 48 | """Get list of all tested revisions""" |
| 89 | valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None]) | 49 | valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None]) |
| 90 | 50 | ||
| 91 | fields, revs = get_test_runs(repo, tag_name, **valid_kwargs) | 51 | fields, revs = gitarchive.get_test_runs(log, repo, tag_name, **valid_kwargs) |
| 92 | ignore_fields = ['tag_number'] | 52 | ignore_fields = ['tag_number'] |
| 93 | if verbosity < 2: | 53 | if verbosity < 2: |
| 94 | extra_fields = ['COMMITS', 'TEST RUNS'] | 54 | extra_fields = ['COMMITS', 'TEST RUNS'] |
| @@ -133,36 +93,6 @@ def list_test_revs(repo, tag_name, verbosity, **kwargs): | |||
| 133 | 93 | ||
| 134 | print_table(rows) | 94 | print_table(rows) |
| 135 | 95 | ||
| 136 | def get_test_revs(repo, tag_name, **kwargs): | ||
| 137 | """Get list of all tested revisions""" | ||
| 138 | fields, runs = get_test_runs(repo, tag_name, **kwargs) | ||
| 139 | |||
| 140 | revs = {} | ||
| 141 | commit_i = fields.index('commit') | ||
| 142 | commit_num_i = fields.index('commit_number') | ||
| 143 | for run in runs: | ||
| 144 | commit = run[commit_i] | ||
| 145 | commit_num = run[commit_num_i] | ||
| 146 | tag = run[-1] | ||
| 147 | if not commit in revs: | ||
| 148 | revs[commit] = TestedRev(commit, commit_num, [tag]) | ||
| 149 | else: | ||
| 150 | assert commit_num == revs[commit].commit_number, "Commit numbers do not match" | ||
| 151 | revs[commit].tags.append(tag) | ||
| 152 | |||
| 153 | # Return in sorted table | ||
| 154 | revs = sorted(revs.values(), key=attrgetter('commit_number')) | ||
| 155 | log.debug("Found %d tested revisions:\n %s", len(revs), | ||
| 156 | "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs])) | ||
| 157 | return revs | ||
| 158 | |||
| 159 | def rev_find(revs, attr, val): | ||
| 160 | """Search from a list of TestedRev""" | ||
| 161 | for i, rev in enumerate(revs): | ||
| 162 | if getattr(rev, attr) == val: | ||
| 163 | return i | ||
| 164 | raise ValueError("Unable to find '{}' value '{}'".format(attr, val)) | ||
| 165 | |||
| 166 | def is_xml_format(repo, commit): | 96 | def is_xml_format(repo, commit): |
| 167 | """Check if the commit contains xml (or json) data""" | 97 | """Check if the commit contains xml (or json) data""" |
| 168 | if repo.rev_parse(commit + ':results.xml'): | 98 | if repo.rev_parse(commit + ':results.xml'): |
| @@ -578,11 +508,11 @@ def main(argv=None): | |||
| 578 | if not args.hostname: | 508 | if not args.hostname: |
| 579 | auto_args(repo, args) | 509 | auto_args(repo, args) |
| 580 | 510 | ||
| 581 | revs = get_test_revs(repo, args.tag_name, hostname=args.hostname, | 511 | revs = gitarchive.get_test_revs(log, repo, args.tag_name, hostname=args.hostname, |
| 582 | branch=args.branch, machine=args.machine) | 512 | branch=args.branch, machine=args.machine) |
| 583 | if args.branch2: | 513 | if args.branch2: |
| 584 | revs2 = get_test_revs(repo, args.tag_name, hostname=args.hostname, | 514 | revs2 = gitarchive.get_test_revs(log, repo, args.tag_name, hostname=args.hostname, |
| 585 | branch=args.branch2, machine=args.machine) | 515 | branch=args.branch2, machine=args.machine) |
| 586 | if not len(revs2): | 516 | if not len(revs2): |
| 587 | log.error("No revisions found to compare against") | 517 | log.error("No revisions found to compare against") |
| 588 | return 1 | 518 | return 1 |
| @@ -598,9 +528,9 @@ def main(argv=None): | |||
| 598 | if args.commit: | 528 | if args.commit: |
| 599 | if args.commit_number: | 529 | if args.commit_number: |
| 600 | log.warning("Ignoring --commit-number as --commit was specified") | 530 | log.warning("Ignoring --commit-number as --commit was specified") |
| 601 | index1 = rev_find(revs, 'commit', args.commit) | 531 | index1 = gitarchive.rev_find(revs, 'commit', args.commit) |
| 602 | elif args.commit_number: | 532 | elif args.commit_number: |
| 603 | index1 = rev_find(revs, 'commit_number', args.commit_number) | 533 | index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number) |
| 604 | else: | 534 | else: |
| 605 | index1 = len(revs) - 1 | 535 | index1 = len(revs) - 1 |
| 606 | 536 | ||
| @@ -612,9 +542,9 @@ def main(argv=None): | |||
| 612 | if args.commit2: | 542 | if args.commit2: |
| 613 | if args.commit_number2: | 543 | if args.commit_number2: |
| 614 | log.warning("Ignoring --commit-number2 as --commit2 was specified") | 544 | log.warning("Ignoring --commit-number2 as --commit2 was specified") |
| 615 | index2 = rev_find(revs, 'commit', args.commit2) | 545 | index2 = gitarchive.rev_find(revs, 'commit', args.commit2) |
| 616 | elif args.commit_number2: | 546 | elif args.commit_number2: |
| 617 | index2 = rev_find(revs, 'commit_number', args.commit_number2) | 547 | index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2) |
| 618 | else: | 548 | else: |
| 619 | if index1 > 0: | 549 | if index1 > 0: |
| 620 | index2 = index1 - 1 | 550 | index2 = index1 - 1 |
