diff options
| -rwxr-xr-x | main.py | 3 | ||||
| -rw-r--r-- | subcmds/sync.py | 87 | ||||
| -rw-r--r-- | tests/test_subcmds_sync.py | 75 |
3 files changed, 148 insertions, 17 deletions
| @@ -427,7 +427,8 @@ class _Repo(object): | |||
| 427 | if not ok: | 427 | if not ok: |
| 428 | exception_name = type(e).__name__ | 428 | exception_name = type(e).__name__ |
| 429 | git_trace2_event_log.ErrorEvent( | 429 | git_trace2_event_log.ErrorEvent( |
| 430 | f"RepoExitError:{exception_name}") | 430 | f"RepoExitError:{exception_name}" |
| 431 | ) | ||
| 431 | raise | 432 | raise |
| 432 | 433 | ||
| 433 | try: | 434 | try: |
diff --git a/subcmds/sync.py b/subcmds/sync.py index a2cc1f89..5f8bc2f0 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
| @@ -737,6 +737,7 @@ later is required to fix a server side protocol bug. | |||
| 737 | start = result.start | 737 | start = result.start |
| 738 | finish = result.finish | 738 | finish = result.finish |
| 739 | self._fetch_times.Set(project, finish - start) | 739 | self._fetch_times.Set(project, finish - start) |
| 740 | self._local_sync_state.SetFetchTime(project) | ||
| 740 | self.event_log.AddSync( | 741 | self.event_log.AddSync( |
| 741 | project, | 742 | project, |
| 742 | event_log.TASK_SYNC_NETWORK, | 743 | event_log.TASK_SYNC_NETWORK, |
| @@ -807,6 +808,7 @@ later is required to fix a server side protocol bug. | |||
| 807 | sync_event.set() | 808 | sync_event.set() |
| 808 | pm.end() | 809 | pm.end() |
| 809 | self._fetch_times.Save() | 810 | self._fetch_times.Save() |
| 811 | self._local_sync_state.Save() | ||
| 810 | 812 | ||
| 811 | if not self.outer_client.manifest.IsArchive: | 813 | if not self.outer_client.manifest.IsArchive: |
| 812 | self._GCProjects(projects, opt, err_event) | 814 | self._GCProjects(projects, opt, err_event) |
| @@ -949,7 +951,9 @@ later is required to fix a server side protocol bug. | |||
| 949 | ) | 951 | ) |
| 950 | # Check for any errors before running any more tasks. | 952 | # Check for any errors before running any more tasks. |
| 951 | # ...we'll let existing jobs finish, though. | 953 | # ...we'll let existing jobs finish, though. |
| 952 | if not success: | 954 | if success: |
| 955 | self._local_sync_state.SetCheckoutTime(project) | ||
| 956 | else: | ||
| 953 | ret = False | 957 | ret = False |
| 954 | err_results.append( | 958 | err_results.append( |
| 955 | project.RelPath(local=opt.this_manifest_only) | 959 | project.RelPath(local=opt.this_manifest_only) |
| @@ -961,21 +965,19 @@ later is required to fix a server side protocol bug. | |||
| 961 | pm.update(msg=project.name) | 965 | pm.update(msg=project.name) |
| 962 | return ret | 966 | return ret |
| 963 | 967 | ||
| 964 | return ( | 968 | proc_res = self.ExecuteInParallel( |
| 965 | self.ExecuteInParallel( | 969 | opt.jobs_checkout, |
| 966 | opt.jobs_checkout, | 970 | functools.partial( |
| 967 | functools.partial( | 971 | self._CheckoutOne, opt.detach_head, opt.force_sync |
| 968 | self._CheckoutOne, opt.detach_head, opt.force_sync | 972 | ), |
| 969 | ), | 973 | all_projects, |
| 970 | all_projects, | 974 | callback=_ProcessResults, |
| 971 | callback=_ProcessResults, | 975 | output=Progress("Checking out", len(all_projects), quiet=opt.quiet), |
| 972 | output=Progress( | ||
| 973 | "Checking out", len(all_projects), quiet=opt.quiet | ||
| 974 | ), | ||
| 975 | ) | ||
| 976 | and not err_results | ||
| 977 | ) | 976 | ) |
| 978 | 977 | ||
| 978 | self._local_sync_state.Save() | ||
| 979 | return proc_res and not err_results | ||
| 980 | |||
| 979 | @staticmethod | 981 | @staticmethod |
| 980 | def _GetPreciousObjectsState(project: Project, opt): | 982 | def _GetPreciousObjectsState(project: Project, opt): |
| 981 | """Get the preciousObjects state for the project. | 983 | """Get the preciousObjects state for the project. |
| @@ -1684,6 +1686,7 @@ later is required to fix a server side protocol bug. | |||
| 1684 | ) | 1686 | ) |
| 1685 | 1687 | ||
| 1686 | self._fetch_times = _FetchTimes(manifest) | 1688 | self._fetch_times = _FetchTimes(manifest) |
| 1689 | self._local_sync_state = _LocalSyncState(manifest) | ||
| 1687 | if not opt.local_only: | 1690 | if not opt.local_only: |
| 1688 | with multiprocessing.Manager() as manager: | 1691 | with multiprocessing.Manager() as manager: |
| 1689 | with ssh.ProxyManager(manager) as ssh_proxy: | 1692 | with ssh.ProxyManager(manager) as ssh_proxy: |
| @@ -1898,12 +1901,64 @@ class _FetchTimes(object): | |||
| 1898 | platform_utils.remove(self._path, missing_ok=True) | 1901 | platform_utils.remove(self._path, missing_ok=True) |
| 1899 | 1902 | ||
| 1900 | 1903 | ||
| 1904 | class _LocalSyncState(object): | ||
| 1905 | _LAST_FETCH = "last_fetch" | ||
| 1906 | _LAST_CHECKOUT = "last_checkout" | ||
| 1907 | |||
| 1908 | def __init__(self, manifest): | ||
| 1909 | self._path = os.path.join(manifest.repodir, ".repo_localsyncstate.json") | ||
| 1910 | self._time = time.time() | ||
| 1911 | self._state = None | ||
| 1912 | self._Load() | ||
| 1913 | |||
| 1914 | def SetFetchTime(self, project): | ||
| 1915 | self._Set(project, self._LAST_FETCH) | ||
| 1916 | |||
| 1917 | def SetCheckoutTime(self, project): | ||
| 1918 | self._Set(project, self._LAST_CHECKOUT) | ||
| 1919 | |||
| 1920 | def GetFetchTime(self, project): | ||
| 1921 | return self._Get(project, self._LAST_FETCH) | ||
| 1922 | |||
| 1923 | def GetCheckoutTime(self, project): | ||
| 1924 | return self._Get(project, self._LAST_CHECKOUT) | ||
| 1925 | |||
| 1926 | def _Get(self, project, key): | ||
| 1927 | self._Load() | ||
| 1928 | p = project.relpath | ||
| 1929 | if p not in self._state: | ||
| 1930 | return | ||
| 1931 | return self._state[p].get(key) | ||
| 1932 | |||
| 1933 | def _Set(self, project, key): | ||
| 1934 | p = project.relpath | ||
| 1935 | if p not in self._state: | ||
| 1936 | self._state[p] = {} | ||
| 1937 | self._state[p][key] = self._time | ||
| 1938 | |||
| 1939 | def _Load(self): | ||
| 1940 | if self._state is None: | ||
| 1941 | try: | ||
| 1942 | with open(self._path) as f: | ||
| 1943 | self._state = json.load(f) | ||
| 1944 | except (IOError, ValueError): | ||
| 1945 | platform_utils.remove(self._path, missing_ok=True) | ||
| 1946 | self._state = {} | ||
| 1947 | |||
| 1948 | def Save(self): | ||
| 1949 | if not self._state: | ||
| 1950 | return | ||
| 1951 | try: | ||
| 1952 | with open(self._path, "w") as f: | ||
| 1953 | json.dump(self._state, f, indent=2) | ||
| 1954 | except (IOError, TypeError): | ||
| 1955 | platform_utils.remove(self._path, missing_ok=True) | ||
| 1956 | |||
| 1957 | |||
| 1901 | # This is a replacement for xmlrpc.client.Transport using urllib2 | 1958 | # This is a replacement for xmlrpc.client.Transport using urllib2 |
| 1902 | # and supporting persistent-http[s]. It cannot change hosts from | 1959 | # and supporting persistent-http[s]. It cannot change hosts from |
| 1903 | # request to request like the normal transport, the real url | 1960 | # request to request like the normal transport, the real url |
| 1904 | # is passed during initialization. | 1961 | # is passed during initialization. |
| 1905 | |||
| 1906 | |||
| 1907 | class PersistentTransport(xmlrpc.client.Transport): | 1962 | class PersistentTransport(xmlrpc.client.Transport): |
| 1908 | def __init__(self, orig_host): | 1963 | def __init__(self, orig_host): |
| 1909 | self.orig_host = orig_host | 1964 | self.orig_host = orig_host |
diff --git a/tests/test_subcmds_sync.py b/tests/test_subcmds_sync.py index 5c8e606e..057478ef 100644 --- a/tests/test_subcmds_sync.py +++ b/tests/test_subcmds_sync.py | |||
| @@ -14,6 +14,8 @@ | |||
| 14 | """Unittests for the subcmds/sync.py module.""" | 14 | """Unittests for the subcmds/sync.py module.""" |
| 15 | 15 | ||
| 16 | import os | 16 | import os |
| 17 | import shutil | ||
| 18 | import tempfile | ||
| 17 | import unittest | 19 | import unittest |
| 18 | from unittest import mock | 20 | from unittest import mock |
| 19 | 21 | ||
| @@ -104,6 +106,79 @@ def test_cli_jobs(argv, jobs_manifest, jobs, jobs_net, jobs_check): | |||
| 104 | assert opts.jobs_checkout == jobs_check | 106 | assert opts.jobs_checkout == jobs_check |
| 105 | 107 | ||
| 106 | 108 | ||
| 109 | class LocalSyncState(unittest.TestCase): | ||
| 110 | """Tests for _LocalSyncState.""" | ||
| 111 | |||
| 112 | _TIME = 10 | ||
| 113 | |||
| 114 | def setUp(self): | ||
| 115 | """Common setup.""" | ||
| 116 | self.repodir = tempfile.mkdtemp(".repo") | ||
| 117 | self.manifest = mock.MagicMock( | ||
| 118 | repodir=self.repodir, | ||
| 119 | ) | ||
| 120 | self.state = self._new_state() | ||
| 121 | |||
| 122 | def tearDown(self): | ||
| 123 | """Common teardown.""" | ||
| 124 | shutil.rmtree(self.repodir) | ||
| 125 | |||
| 126 | def _new_state(self): | ||
| 127 | with mock.patch("time.time", return_value=self._TIME): | ||
| 128 | return sync._LocalSyncState(self.manifest) | ||
| 129 | |||
| 130 | def test_set(self): | ||
| 131 | """Times are set.""" | ||
| 132 | p = mock.MagicMock(relpath="projA") | ||
| 133 | self.state.SetFetchTime(p) | ||
| 134 | self.state.SetCheckoutTime(p) | ||
| 135 | self.assertEqual(self.state.GetFetchTime(p), self._TIME) | ||
| 136 | self.assertEqual(self.state.GetCheckoutTime(p), self._TIME) | ||
| 137 | |||
| 138 | def test_update(self): | ||
| 139 | """Times are updated.""" | ||
| 140 | with open(self.state._path, "w") as f: | ||
| 141 | f.write( | ||
| 142 | """ | ||
| 143 | { | ||
| 144 | "projB": { | ||
| 145 | "last_fetch": 5, | ||
| 146 | "last_checkout": 7 | ||
| 147 | } | ||
| 148 | } | ||
| 149 | """ | ||
| 150 | ) | ||
| 151 | |||
| 152 | # Initialize state to read from the new file. | ||
| 153 | self.state = self._new_state() | ||
| 154 | projA = mock.MagicMock(relpath="projA") | ||
| 155 | projB = mock.MagicMock(relpath="projB") | ||
| 156 | self.assertEqual(self.state.GetFetchTime(projA), None) | ||
| 157 | self.assertEqual(self.state.GetFetchTime(projB), 5) | ||
| 158 | self.assertEqual(self.state.GetCheckoutTime(projB), 7) | ||
| 159 | |||
| 160 | self.state.SetFetchTime(projA) | ||
| 161 | self.state.SetFetchTime(projB) | ||
| 162 | self.assertEqual(self.state.GetFetchTime(projA), self._TIME) | ||
| 163 | self.assertEqual(self.state.GetFetchTime(projB), self._TIME) | ||
| 164 | self.assertEqual(self.state.GetCheckoutTime(projB), 7) | ||
| 165 | |||
| 166 | def test_save_to_file(self): | ||
| 167 | """Data is saved under repodir.""" | ||
| 168 | p = mock.MagicMock(relpath="projA") | ||
| 169 | self.state.SetFetchTime(p) | ||
| 170 | self.state.Save() | ||
| 171 | self.assertEqual( | ||
| 172 | os.listdir(self.repodir), [".repo_localsyncstate.json"] | ||
| 173 | ) | ||
| 174 | |||
| 175 | def test_nonexistent_project(self): | ||
| 176 | """Unsaved projects don't have data.""" | ||
| 177 | p = mock.MagicMock(relpath="projC") | ||
| 178 | self.assertEqual(self.state.GetFetchTime(p), None) | ||
| 179 | self.assertEqual(self.state.GetCheckoutTime(p), None) | ||
| 180 | |||
| 181 | |||
| 107 | class GetPreciousObjectsState(unittest.TestCase): | 182 | class GetPreciousObjectsState(unittest.TestCase): |
| 108 | """Tests for _GetPreciousObjectsState.""" | 183 | """Tests for _GetPreciousObjectsState.""" |
| 109 | 184 | ||
