diff options
-rw-r--r-- | subcmds/sync.py | 64 | ||||
-rw-r--r-- | tests/test_subcmds_sync.py | 26 |
2 files changed, 81 insertions, 9 deletions
diff --git a/subcmds/sync.py b/subcmds/sync.py index bf1171dd..c6682a5b 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
@@ -21,6 +21,7 @@ import multiprocessing | |||
21 | import netrc | 21 | import netrc |
22 | import optparse | 22 | import optparse |
23 | import os | 23 | import os |
24 | from pathlib import Path | ||
24 | import sys | 25 | import sys |
25 | import tempfile | 26 | import tempfile |
26 | import time | 27 | import time |
@@ -87,6 +88,45 @@ _REPO_ALLOW_SHALLOW = os.environ.get("REPO_ALLOW_SHALLOW") | |||
87 | logger = RepoLogger(__file__) | 88 | logger = RepoLogger(__file__) |
88 | 89 | ||
89 | 90 | ||
91 | def _SafeCheckoutOrder(checkouts: List[Project]) -> List[List[Project]]: | ||
92 | """Generate a sequence of checkouts that is safe to perform. The client | ||
93 | should checkout everything from n-th index before moving to n+1. | ||
94 | |||
95 | This is only useful if manifest contains nested projects. | ||
96 | |||
97 | E.g. if foo, foo/bar and foo/bar/baz are project paths, then foo needs to | ||
98 | finish before foo/bar can proceed, and foo/bar needs to finish before | ||
99 | foo/bar/baz.""" | ||
100 | res = [[]] | ||
101 | current = res[0] | ||
102 | |||
103 | # depth_stack contains a current stack of parent paths. | ||
104 | depth_stack = [] | ||
105 | # checkouts are iterated in asc order by relpath. That way, it can easily be | ||
106 | # determined if the previous checkout is parent of the current checkout. | ||
107 | for checkout in sorted(checkouts, key=lambda x: x.relpath): | ||
108 | checkout_path = Path(checkout.relpath) | ||
109 | while depth_stack: | ||
110 | try: | ||
111 | checkout_path.relative_to(depth_stack[-1]) | ||
112 | except ValueError: | ||
113 | # Path.relative_to returns ValueError if paths are not relative. | ||
114 | # TODO(sokcevic): Switch to is_relative_to once min supported | ||
115 | # version is py3.9. | ||
116 | depth_stack.pop() | ||
117 | else: | ||
118 | if len(depth_stack) >= len(res): | ||
119 | # Another depth created. | ||
120 | res.append([]) | ||
121 | break | ||
122 | |||
123 | current = res[len(depth_stack)] | ||
124 | current.append(checkout) | ||
125 | depth_stack.append(checkout_path) | ||
126 | |||
127 | return res | ||
128 | |||
129 | |||
90 | class _FetchOneResult(NamedTuple): | 130 | class _FetchOneResult(NamedTuple): |
91 | """_FetchOne return value. | 131 | """_FetchOne return value. |
92 | 132 | ||
@@ -1035,15 +1075,21 @@ later is required to fix a server side protocol bug. | |||
1035 | pm.update(msg=project.name) | 1075 | pm.update(msg=project.name) |
1036 | return ret | 1076 | return ret |
1037 | 1077 | ||
1038 | proc_res = self.ExecuteInParallel( | 1078 | for projects in _SafeCheckoutOrder(all_projects): |
1039 | opt.jobs_checkout, | 1079 | proc_res = self.ExecuteInParallel( |
1040 | functools.partial( | 1080 | opt.jobs_checkout, |
1041 | self._CheckoutOne, opt.detach_head, opt.force_sync, opt.verbose | 1081 | functools.partial( |
1042 | ), | 1082 | self._CheckoutOne, |
1043 | all_projects, | 1083 | opt.detach_head, |
1044 | callback=_ProcessResults, | 1084 | opt.force_sync, |
1045 | output=Progress("Checking out", len(all_projects), quiet=opt.quiet), | 1085 | opt.verbose, |
1046 | ) | 1086 | ), |
1087 | projects, | ||
1088 | callback=_ProcessResults, | ||
1089 | output=Progress( | ||
1090 | "Checking out", len(all_projects), quiet=opt.quiet | ||
1091 | ), | ||
1092 | ) | ||
1047 | 1093 | ||
1048 | self._local_sync_state.Save() | 1094 | self._local_sync_state.Save() |
1049 | return proc_res and not err_results | 1095 | return proc_res and not err_results |
diff --git a/tests/test_subcmds_sync.py b/tests/test_subcmds_sync.py index af6bbef7..13e23e34 100644 --- a/tests/test_subcmds_sync.py +++ b/tests/test_subcmds_sync.py | |||
@@ -304,6 +304,32 @@ class LocalSyncState(unittest.TestCase): | |||
304 | self.assertEqual(self.state.GetFetchTime(projA), 5) | 304 | self.assertEqual(self.state.GetFetchTime(projA), 5) |
305 | 305 | ||
306 | 306 | ||
307 | class SafeCheckoutOrder(unittest.TestCase): | ||
308 | def test_no_nested(self): | ||
309 | p_f = mock.MagicMock(relpath="f") | ||
310 | p_foo = mock.MagicMock(relpath="foo") | ||
311 | out = sync._SafeCheckoutOrder([p_f, p_foo]) | ||
312 | self.assertEqual(out, [[p_f, p_foo]]) | ||
313 | |||
314 | def test_basic_nested(self): | ||
315 | p_foo = p_foo = mock.MagicMock(relpath="foo") | ||
316 | p_foo_bar = mock.MagicMock(relpath="foo/bar") | ||
317 | out = sync._SafeCheckoutOrder([p_foo, p_foo_bar]) | ||
318 | self.assertEqual(out, [[p_foo], [p_foo_bar]]) | ||
319 | |||
320 | def test_complex_nested(self): | ||
321 | p_foo = mock.MagicMock(relpath="foo") | ||
322 | p_foo_bar = mock.MagicMock(relpath="foo/bar") | ||
323 | p_foo_bar_baz_baq = mock.MagicMock(relpath="foo/bar/baz/baq") | ||
324 | p_bar = mock.MagicMock(relpath="bar") | ||
325 | out = sync._SafeCheckoutOrder( | ||
326 | [p_foo_bar_baz_baq, p_foo, p_foo_bar, p_bar] | ||
327 | ) | ||
328 | self.assertEqual( | ||
329 | out, [[p_bar, p_foo], [p_foo_bar], [p_foo_bar_baz_baq]] | ||
330 | ) | ||
331 | |||
332 | |||
307 | class GetPreciousObjectsState(unittest.TestCase): | 333 | class GetPreciousObjectsState(unittest.TestCase): |
308 | """Tests for _GetPreciousObjectsState.""" | 334 | """Tests for _GetPreciousObjectsState.""" |
309 | 335 | ||