1 files changed, 244 insertions, 0 deletions

diff --git a/scripts/oe-git-archive b/scripts/oe-git-archive
new file mode 100755
index 0000000000..419332ded1
--- /dev/null
+++ b/scripts/oe-git-archive
@@ -0,0 +1,244 @@
+#!/usr/bin/python3
+#
+# Helper script for committing data to git and pushing upstream
+#
+# Copyright (c) 2017, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+import argparse
+import glob
+import json
+import logging
+import math
+import os
+import re
+import sys
+from collections import namedtuple, OrderedDict
+from datetime import datetime, timedelta, tzinfo
+from operator import attrgetter
+
+# Import oe and bitbake libs
+scripts_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(scripts_path, 'lib'))
+import scriptpath
+scriptpath.add_bitbake_lib_path()
+scriptpath.add_oe_lib_path()
+
+from oeqa.utils.git import GitRepo, GitError
+from oeqa.utils.metadata import metadata_from_bb
+
+
+# Setup logging
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+log = logging.getLogger()
+
+
+class ArchiveError(Exception):
+    """Internal error handling of this script"""
+
+
+def format_str(string, fields):
+    """Format string using the given fields (dict)"""
+    try:
+        return string.format(**fields)
+    except KeyError as err:
+        raise ArchiveError("Unable to expand string '{}': unknown field {} "
+                           "(valid fields are: {})".format(
+                               string, err, ', '.join(sorted(fields.keys()))))
+
+
+def init_git_repo(path, no_create):
+    """Initialize local Git repository"""
+    path = os.path.abspath(path)
+    if os.path.isfile(path):
+        raise ArchiveError("Invalid Git repo at {}: path exists but is not a "
+                           "directory".format(path))
+    if not os.path.isdir(path) or not os.listdir(path):
+        if no_create:
+            raise ArchiveError("No git repo at {}, refusing to create "
+                               "one".format(path))
+        if not os.path.isdir(path):
+            try:
+                os.mkdir(path)
+            except (FileNotFoundError, PermissionError) as err:
+                raise ArchiveError("Failed to mkdir {}: {}".format(path, err))
+        if not os.listdir(path):
+            log.info("Initializing a new Git repo at %s", path)
+            repo = GitRepo.init(path)
+    try:
+        repo = GitRepo(path, is_topdir=True)
+    except GitError:
+        raise ArchiveError("Non-empty directory that is not a Git repository "
+                           "at {}\nPlease specify an existing Git repository, "
+                           "an empty directory or a non-existing directory "
+                           "path.".format(path))
+    return repo
+
+
+def git_commit_data(repo, data_dir, branch, message):
+    """Commit data into a Git repository"""
+    log.info("Committing data into to branch %s", branch)
+    tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive')
+    try:
+        # Create new tree object from the data
+        env_update = {'GIT_INDEX_FILE': tmp_index,
+                      'GIT_WORK_TREE': os.path.abspath(data_dir)}
+        repo.run_cmd('add .', env_update)
+        tree = repo.run_cmd('write-tree', env_update)
+
+        # Create new commit object from the tree
+        parent = repo.rev_parse(branch)
+        git_cmd = ['commit-tree', tree, '-m', message]
+        if parent:
+            git_cmd += ['-p', parent]
+        commit = repo.run_cmd(git_cmd, env_update)
+
+        # Update branch head
+        git_cmd = ['update-ref', 'refs/heads/' + branch, commit]
+        if parent:
+            git_cmd.append(parent)
+        repo.run_cmd(git_cmd)
+
+        # Update current HEAD, if we're on branch 'branch'
+        if repo.get_current_branch() == branch:
+            log.info("Updating %s HEAD to latest commit", repo.top_dir)
+            repo.run_cmd('reset --hard')
+
+        return commit
+    finally:
+        if os.path.exists(tmp_index):
+            os.unlink(tmp_index)
+
+
+def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern,
+                       keywords):
+    """Generate tag name and message, with support for running id number"""
+    keyws = keywords.copy()
+    # Tag number is handled specially: if not defined, we autoincrement it
+    if 'tag_number' not in keyws:
+        # Fill in all other fields than 'tag_number'
+        keyws['tag_number'] = '{tag_number}'
+        tag_re = format_str(name_pattern, keyws)
+        # Replace parentheses for proper regex matching
+        tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$'
+        # Inject regex group pattern for 'tag_number'
+        tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})')
+
+        keyws['tag_number'] = 0
+        for existing_tag in repo.run_cmd('tag').splitlines():
+            match = re.match(tag_re, existing_tag)
+
+            if match and int(match.group('tag_number')) >= keyws['tag_number']:
+                keyws['tag_number'] = int(match.group('tag_number')) + 1
+
+    tag_name = format_str(name_pattern, keyws)
+    msg_subj= format_str(msg_subj_pattern.strip(), keyws)
+    msg_body = format_str(msg_body_pattern, keyws)
+    return tag_name, msg_subj + '\n\n' + msg_body
+
+
+def parse_args(argv):
+    """Parse command line arguments"""
+    parser = argparse.ArgumentParser(
+            description="Commit data to git and push upstream",
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument('--debug', '-D', action='store_true',
+                        help="Verbose logging")
+    parser.add_argument('--git-dir', '-g', required=True,
+                        help="Local git directory to use")
+    parser.add_argument('--no-create', action='store_true',
+                        help="If GIT_DIR is not a valid Git repository, do not "
+                             "try to create one")
+    parser.add_argument('--push', '-p', nargs='?', default=False, const=True,
+                        help="Push to remote")
+    parser.add_argument('--branch-name', '-b',
+                        default='{hostname}/{branch}/{machine}',
+                        help="Git branch name (pattern) to use")
+    parser.add_argument('--no-tag', action='store_true',
+                        help="Do not create Git tag")
+    parser.add_argument('--tag-name', '-t',
+                        default='{hostname}/{branch}/{machine}/{commit_count}-g{commit}/{tag_number}',
+                        help="Tag name (pattern) to use")
+    parser.add_argument('--commit-msg-subject',
+                        default='Results of {branch}:{commit} on {hostname}',
+                        help="Subject line (pattern) to use in the commit message")
+    parser.add_argument('--commit-msg-body',
+                        default='branch: {branch}\ncommit: {commit}\nhostname: {hostname}',
+                        help="Commit message body (pattern)")
+    parser.add_argument('--tag-msg-subject',
+                        default='Test run #{tag_number} of {branch}:{commit} on {hostname}',
+                        help="Subject line (pattern) of the tag message")
+    parser.add_argument('--tag-msg-body',
+                        default='',
+                        help="Tag message body (pattern)")
+    parser.add_argument('data_dir', metavar='DATA_DIR',
+                        help="Data to commit")
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    """Script entry point"""
+    args = parse_args(argv)
+    if args.debug:
+        log.setLevel(logging.DEBUG)
+
+    try:
+        if not os.path.isdir(args.data_dir):
+            raise ArchiveError("Not a directory: {}".format(args.data_dir))
+
+        data_repo = init_git_repo(args.git_dir, args.no_create)
+
+        # Get keywords to be used in tag and branch names and messages
+        metadata = metadata_from_bb()
+        keywords = {'hostname': metadata['hostname'],
+                    'branch': metadata['layers']['meta']['branch'],
+                    'commit': metadata['layers']['meta']['commit'],
+                    'commit_count': metadata['layers']['meta']['commit_count'],
+                    'machine': metadata['config']['MACHINE']}
+
+        # Expand strings early in order to avoid getting into inconsistent
+        # state (e.g. no tag even if data was committed)
+        commit_msg = format_str(args.commit_msg_subject.strip(), keywords)
+        commit_msg += '\n\n' + format_str(args.commit_msg_body, keywords)
+        branch_name = format_str(args.branch_name, keywords)
+        tag_name = None
+        if not args.no_tag and args.tag_name:
+            tag_name, tag_msg = expand_tag_strings(data_repo, args.tag_name,
+                                                   args.tag_msg_subject,
+                                                   args.tag_msg_body, keywords)
+
+        # Commit data
+        commit = git_commit_data(data_repo, args.data_dir, branch_name,
+                                 commit_msg)
+
+        # Create tag
+        if tag_name:
+            log.info("Creating tag %s", tag_name)
+            data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit])
+
+        # Push data to remote
+        if args.push:
+            cmd = ['push', '--tags']
+            if args.push is not True:
+                cmd.extend(['--repo', args.push])
+                cmd.append(branch_name)
+            log.info("Pushing data to remote")
+            data_repo.run_cmd(cmd)
+
+    except ArchiveError as err:
+        log.error(str(err))
+        return 1
+
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())