diff options
-rwxr-xr-x | main.py | 3 | ||||
-rw-r--r-- | subcmds/sync.py | 87 | ||||
-rw-r--r-- | tests/test_subcmds_sync.py | 75 |
3 files changed, 148 insertions, 17 deletions
@@ -427,7 +427,8 @@ class _Repo(object): | |||
427 | if not ok: | 427 | if not ok: |
428 | exception_name = type(e).__name__ | 428 | exception_name = type(e).__name__ |
429 | git_trace2_event_log.ErrorEvent( | 429 | git_trace2_event_log.ErrorEvent( |
430 | f"RepoExitError:{exception_name}") | 430 | f"RepoExitError:{exception_name}" |
431 | ) | ||
431 | raise | 432 | raise |
432 | 433 | ||
433 | try: | 434 | try: |
diff --git a/subcmds/sync.py b/subcmds/sync.py index a2cc1f89..5f8bc2f0 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
@@ -737,6 +737,7 @@ later is required to fix a server side protocol bug. | |||
737 | start = result.start | 737 | start = result.start |
738 | finish = result.finish | 738 | finish = result.finish |
739 | self._fetch_times.Set(project, finish - start) | 739 | self._fetch_times.Set(project, finish - start) |
740 | self._local_sync_state.SetFetchTime(project) | ||
740 | self.event_log.AddSync( | 741 | self.event_log.AddSync( |
741 | project, | 742 | project, |
742 | event_log.TASK_SYNC_NETWORK, | 743 | event_log.TASK_SYNC_NETWORK, |
@@ -807,6 +808,7 @@ later is required to fix a server side protocol bug. | |||
807 | sync_event.set() | 808 | sync_event.set() |
808 | pm.end() | 809 | pm.end() |
809 | self._fetch_times.Save() | 810 | self._fetch_times.Save() |
811 | self._local_sync_state.Save() | ||
810 | 812 | ||
811 | if not self.outer_client.manifest.IsArchive: | 813 | if not self.outer_client.manifest.IsArchive: |
812 | self._GCProjects(projects, opt, err_event) | 814 | self._GCProjects(projects, opt, err_event) |
@@ -949,7 +951,9 @@ later is required to fix a server side protocol bug. | |||
949 | ) | 951 | ) |
950 | # Check for any errors before running any more tasks. | 952 | # Check for any errors before running any more tasks. |
951 | # ...we'll let existing jobs finish, though. | 953 | # ...we'll let existing jobs finish, though. |
952 | if not success: | 954 | if success: |
955 | self._local_sync_state.SetCheckoutTime(project) | ||
956 | else: | ||
953 | ret = False | 957 | ret = False |
954 | err_results.append( | 958 | err_results.append( |
955 | project.RelPath(local=opt.this_manifest_only) | 959 | project.RelPath(local=opt.this_manifest_only) |
@@ -961,21 +965,19 @@ later is required to fix a server side protocol bug. | |||
961 | pm.update(msg=project.name) | 965 | pm.update(msg=project.name) |
962 | return ret | 966 | return ret |
963 | 967 | ||
964 | return ( | 968 | proc_res = self.ExecuteInParallel( |
965 | self.ExecuteInParallel( | 969 | opt.jobs_checkout, |
966 | opt.jobs_checkout, | 970 | functools.partial( |
967 | functools.partial( | 971 | self._CheckoutOne, opt.detach_head, opt.force_sync |
968 | self._CheckoutOne, opt.detach_head, opt.force_sync | 972 | ), |
969 | ), | 973 | all_projects, |
970 | all_projects, | 974 | callback=_ProcessResults, |
971 | callback=_ProcessResults, | 975 | output=Progress("Checking out", len(all_projects), quiet=opt.quiet), |
972 | output=Progress( | ||
973 | "Checking out", len(all_projects), quiet=opt.quiet | ||
974 | ), | ||
975 | ) | ||
976 | and not err_results | ||
977 | ) | 976 | ) |
978 | 977 | ||
978 | self._local_sync_state.Save() | ||
979 | return proc_res and not err_results | ||
980 | |||
979 | @staticmethod | 981 | @staticmethod |
980 | def _GetPreciousObjectsState(project: Project, opt): | 982 | def _GetPreciousObjectsState(project: Project, opt): |
981 | """Get the preciousObjects state for the project. | 983 | """Get the preciousObjects state for the project. |
@@ -1684,6 +1686,7 @@ later is required to fix a server side protocol bug. | |||
1684 | ) | 1686 | ) |
1685 | 1687 | ||
1686 | self._fetch_times = _FetchTimes(manifest) | 1688 | self._fetch_times = _FetchTimes(manifest) |
1689 | self._local_sync_state = _LocalSyncState(manifest) | ||
1687 | if not opt.local_only: | 1690 | if not opt.local_only: |
1688 | with multiprocessing.Manager() as manager: | 1691 | with multiprocessing.Manager() as manager: |
1689 | with ssh.ProxyManager(manager) as ssh_proxy: | 1692 | with ssh.ProxyManager(manager) as ssh_proxy: |
@@ -1898,12 +1901,64 @@ class _FetchTimes(object): | |||
1898 | platform_utils.remove(self._path, missing_ok=True) | 1901 | platform_utils.remove(self._path, missing_ok=True) |
1899 | 1902 | ||
1900 | 1903 | ||
1904 | class _LocalSyncState(object): | ||
1905 | _LAST_FETCH = "last_fetch" | ||
1906 | _LAST_CHECKOUT = "last_checkout" | ||
1907 | |||
1908 | def __init__(self, manifest): | ||
1909 | self._path = os.path.join(manifest.repodir, ".repo_localsyncstate.json") | ||
1910 | self._time = time.time() | ||
1911 | self._state = None | ||
1912 | self._Load() | ||
1913 | |||
1914 | def SetFetchTime(self, project): | ||
1915 | self._Set(project, self._LAST_FETCH) | ||
1916 | |||
1917 | def SetCheckoutTime(self, project): | ||
1918 | self._Set(project, self._LAST_CHECKOUT) | ||
1919 | |||
1920 | def GetFetchTime(self, project): | ||
1921 | return self._Get(project, self._LAST_FETCH) | ||
1922 | |||
1923 | def GetCheckoutTime(self, project): | ||
1924 | return self._Get(project, self._LAST_CHECKOUT) | ||
1925 | |||
1926 | def _Get(self, project, key): | ||
1927 | self._Load() | ||
1928 | p = project.relpath | ||
1929 | if p not in self._state: | ||
1930 | return | ||
1931 | return self._state[p].get(key) | ||
1932 | |||
1933 | def _Set(self, project, key): | ||
1934 | p = project.relpath | ||
1935 | if p not in self._state: | ||
1936 | self._state[p] = {} | ||
1937 | self._state[p][key] = self._time | ||
1938 | |||
1939 | def _Load(self): | ||
1940 | if self._state is None: | ||
1941 | try: | ||
1942 | with open(self._path) as f: | ||
1943 | self._state = json.load(f) | ||
1944 | except (IOError, ValueError): | ||
1945 | platform_utils.remove(self._path, missing_ok=True) | ||
1946 | self._state = {} | ||
1947 | |||
1948 | def Save(self): | ||
1949 | if not self._state: | ||
1950 | return | ||
1951 | try: | ||
1952 | with open(self._path, "w") as f: | ||
1953 | json.dump(self._state, f, indent=2) | ||
1954 | except (IOError, TypeError): | ||
1955 | platform_utils.remove(self._path, missing_ok=True) | ||
1956 | |||
1957 | |||
1901 | # This is a replacement for xmlrpc.client.Transport using urllib2 | 1958 | # This is a replacement for xmlrpc.client.Transport using urllib2 |
1902 | # and supporting persistent-http[s]. It cannot change hosts from | 1959 | # and supporting persistent-http[s]. It cannot change hosts from |
1903 | # request to request like the normal transport, the real url | 1960 | # request to request like the normal transport, the real url |
1904 | # is passed during initialization. | 1961 | # is passed during initialization. |
1905 | |||
1906 | |||
1907 | class PersistentTransport(xmlrpc.client.Transport): | 1962 | class PersistentTransport(xmlrpc.client.Transport): |
1908 | def __init__(self, orig_host): | 1963 | def __init__(self, orig_host): |
1909 | self.orig_host = orig_host | 1964 | self.orig_host = orig_host |
diff --git a/tests/test_subcmds_sync.py b/tests/test_subcmds_sync.py index 5c8e606e..057478ef 100644 --- a/tests/test_subcmds_sync.py +++ b/tests/test_subcmds_sync.py | |||
@@ -14,6 +14,8 @@ | |||
14 | """Unittests for the subcmds/sync.py module.""" | 14 | """Unittests for the subcmds/sync.py module.""" |
15 | 15 | ||
16 | import os | 16 | import os |
17 | import shutil | ||
18 | import tempfile | ||
17 | import unittest | 19 | import unittest |
18 | from unittest import mock | 20 | from unittest import mock |
19 | 21 | ||
@@ -104,6 +106,79 @@ def test_cli_jobs(argv, jobs_manifest, jobs, jobs_net, jobs_check): | |||
104 | assert opts.jobs_checkout == jobs_check | 106 | assert opts.jobs_checkout == jobs_check |
105 | 107 | ||
106 | 108 | ||
109 | class LocalSyncState(unittest.TestCase): | ||
110 | """Tests for _LocalSyncState.""" | ||
111 | |||
112 | _TIME = 10 | ||
113 | |||
114 | def setUp(self): | ||
115 | """Common setup.""" | ||
116 | self.repodir = tempfile.mkdtemp(".repo") | ||
117 | self.manifest = mock.MagicMock( | ||
118 | repodir=self.repodir, | ||
119 | ) | ||
120 | self.state = self._new_state() | ||
121 | |||
122 | def tearDown(self): | ||
123 | """Common teardown.""" | ||
124 | shutil.rmtree(self.repodir) | ||
125 | |||
126 | def _new_state(self): | ||
127 | with mock.patch("time.time", return_value=self._TIME): | ||
128 | return sync._LocalSyncState(self.manifest) | ||
129 | |||
130 | def test_set(self): | ||
131 | """Times are set.""" | ||
132 | p = mock.MagicMock(relpath="projA") | ||
133 | self.state.SetFetchTime(p) | ||
134 | self.state.SetCheckoutTime(p) | ||
135 | self.assertEqual(self.state.GetFetchTime(p), self._TIME) | ||
136 | self.assertEqual(self.state.GetCheckoutTime(p), self._TIME) | ||
137 | |||
138 | def test_update(self): | ||
139 | """Times are updated.""" | ||
140 | with open(self.state._path, "w") as f: | ||
141 | f.write( | ||
142 | """ | ||
143 | { | ||
144 | "projB": { | ||
145 | "last_fetch": 5, | ||
146 | "last_checkout": 7 | ||
147 | } | ||
148 | } | ||
149 | """ | ||
150 | ) | ||
151 | |||
152 | # Initialize state to read from the new file. | ||
153 | self.state = self._new_state() | ||
154 | projA = mock.MagicMock(relpath="projA") | ||
155 | projB = mock.MagicMock(relpath="projB") | ||
156 | self.assertEqual(self.state.GetFetchTime(projA), None) | ||
157 | self.assertEqual(self.state.GetFetchTime(projB), 5) | ||
158 | self.assertEqual(self.state.GetCheckoutTime(projB), 7) | ||
159 | |||
160 | self.state.SetFetchTime(projA) | ||
161 | self.state.SetFetchTime(projB) | ||
162 | self.assertEqual(self.state.GetFetchTime(projA), self._TIME) | ||
163 | self.assertEqual(self.state.GetFetchTime(projB), self._TIME) | ||
164 | self.assertEqual(self.state.GetCheckoutTime(projB), 7) | ||
165 | |||
166 | def test_save_to_file(self): | ||
167 | """Data is saved under repodir.""" | ||
168 | p = mock.MagicMock(relpath="projA") | ||
169 | self.state.SetFetchTime(p) | ||
170 | self.state.Save() | ||
171 | self.assertEqual( | ||
172 | os.listdir(self.repodir), [".repo_localsyncstate.json"] | ||
173 | ) | ||
174 | |||
175 | def test_nonexistent_project(self): | ||
176 | """Unsaved projects don't have data.""" | ||
177 | p = mock.MagicMock(relpath="projC") | ||
178 | self.assertEqual(self.state.GetFetchTime(p), None) | ||
179 | self.assertEqual(self.state.GetCheckoutTime(p), None) | ||
180 | |||
181 | |||
107 | class GetPreciousObjectsState(unittest.TestCase): | 182 | class GetPreciousObjectsState(unittest.TestCase): |
108 | """Tests for _GetPreciousObjectsState.""" | 183 | """Tests for _GetPreciousObjectsState.""" |
109 | 184 | ||