From 7954de13b79182e299163307c430ad96f9cdbd38 Mon Sep 17 00:00:00 2001 From: Raman Tenneti Date: Wed, 28 Jul 2021 14:36:49 -0700 Subject: sync: Added logging of repo sync state and config options for analysis. git_config.py: + Added SyncAnalysisState class, which saves the following data into the config object. ++ sys.argv, options, superproject's logging data. ++ repo.*, branch.* and remote.* parameters from config object. ++ current time as synctime. ++ Version number of the object. + All the keys for the above data are prepended with 'repo.syncstate.' + Added GetSyncAnalysisStateData and UpdateSyncAnalysisState methods to GitConfig object to save/get the above data. git_trace2_event_log.py: + Added LogConfigEvents method with code from DefParamRepoEvents to log events. sync.py: + superproject_logging_data is a dictionary that collects all the superproject data that is to be logged as trace2 event. + Sync at the end logs the previously saved syncstate.* parameters as previous_sync_state. Then it calls config's UpdateSyncAnalysisState to save and log all the current options, superproject logged data. docs/internal-fs-layout.md: + Added doc string explaining [repo.syncstate ...] sections of .repo/manifests.git/config file. test_git_config.py: + Added unit test for the new methods of GitConfig object. Tested: $ ./run_tests $ repo_dev init --use-superproject -u https://android.googlesource.com/platform/manifest Tested it by running the following command multiple times. $ repo_dev sync -j 20 repo sync has finished successfully Verified config file has [syncstate ...] data saved. Bug: [google internal] b/188573450 Change-Id: I1f914ce50f3382111b72940ca56de7c41b53d460 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/313123 Tested-by: Raman Tenneti Reviewed-by: Mike Frysinger Reviewed-by: Xin Li --- git_config.py | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) (limited to 'git_config.py') diff --git a/git_config.py b/git_config.py index 3eaf201c..05d824cb 100644 --- a/git_config.py +++ b/git_config.py @@ -13,6 +13,7 @@ # limitations under the License. import contextlib +import datetime import errno from http.client import HTTPException import json @@ -30,6 +31,10 @@ from repo_trace import Trace from git_command import GitCommand from git_refs import R_CHANGES, R_HEADS, R_TAGS +# Prefix that is prepended to all the keys of SyncAnalysisState's data +# that is saved in the config. +SYNC_STATE_PREFIX = 'repo.syncstate.' + ID_RE = re.compile(r'^[0-9a-f]{40}$') REVIEW_CACHE = dict() @@ -262,6 +267,22 @@ class GitConfig(object): self._branches[b.name] = b return b + def GetSyncAnalysisStateData(self): + """Returns data to be logged for the analysis of sync performance.""" + return {k: v for k, v in self.DumpConfigDict().items() if k.startswith(SYNC_STATE_PREFIX)} + + def UpdateSyncAnalysisState(self, options, superproject_logging_data): + """Update Config's SYNC_STATE_PREFIX* data with the latest sync data. + + Args: + options: Options passed to sync returned from optparse. See _Options(). + superproject_logging_data: A dictionary of superproject data that is to be logged. + + Returns: + SyncAnalysisState object. + """ + return SyncAnalysisState(self, options, superproject_logging_data) + def GetSubSections(self, section): """List all subsection names matching $section.*.* """ @@ -717,3 +738,69 @@ class Branch(object): def _Get(self, key, all_keys=False): key = 'branch.%s.%s' % (self.name, key) return self._config.GetString(key, all_keys=all_keys) + + +class SyncAnalysisState: + """Configuration options related to logging of sync state for analysis. + + This object is versioned. + """ + def __init__(self, config, options, superproject_logging_data): + """Initializes SyncAnalysisState. + + Saves the following data into the |config| object. + - sys.argv, options, superproject's logging data. + - repo.*, branch.* and remote.* parameters from config object. + - Current time as synctime. + - Version number of the object. + + All the keys saved by this object are prepended with SYNC_STATE_PREFIX. + + Args: + config: GitConfig object to store all options. + options: Options passed to sync returned from optparse. See _Options(). + superproject_logging_data: A dictionary of superproject data that is to be logged. + """ + self._config = config + now = datetime.datetime.utcnow() + self._Set('main.synctime', now.isoformat() + 'Z') + self._Set('main.version', '1') + self._Set('sys.argv', sys.argv) + for key, value in superproject_logging_data.items(): + self._Set(f'superproject.{key}', value) + for key, value in options.__dict__.items(): + self._Set(f'options.{key}', value) + config_items = config.DumpConfigDict().items() + EXTRACT_NAMESPACES = {'repo', 'branch', 'remote'} + self._SetDictionary({k: v for k, v in config_items + if not k.startswith(SYNC_STATE_PREFIX) and + k.split('.', 1)[0] in EXTRACT_NAMESPACES}) + + def _SetDictionary(self, data): + """Save all key/value pairs of |data| dictionary. + + Args: + data: A dictionary whose key/value are to be saved. + """ + for key, value in data.items(): + self._Set(key, value) + + def _Set(self, key, value): + """Set the |value| for a |key| in the |_config| member. + + |key| is prepended with the value of SYNC_STATE_PREFIX constant. + + Args: + key: Name of the key. + value: |value| could be of any type. If it is 'bool', it will be saved + as a Boolean and for all other types, it will be saved as a String. + """ + if value is None: + return + sync_key = f'{SYNC_STATE_PREFIX}{key}' + if isinstance(value, str): + self._config.SetString(sync_key, value) + elif isinstance(value, bool): + self._config.SetBoolean(sync_key, value) + else: + self._config.SetString(sync_key, str(value)) -- cgit v1.2.3-54-g00ecf