summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--subcmds/sync.py64
-rw-r--r--tests/test_subcmds_sync.py26
2 files changed, 81 insertions, 9 deletions
diff --git a/subcmds/sync.py b/subcmds/sync.py
index bf1171dd..c6682a5b 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -21,6 +21,7 @@ import multiprocessing
21import netrc 21import netrc
22import optparse 22import optparse
23import os 23import os
24from pathlib import Path
24import sys 25import sys
25import tempfile 26import tempfile
26import time 27import time
@@ -87,6 +88,45 @@ _REPO_ALLOW_SHALLOW = os.environ.get("REPO_ALLOW_SHALLOW")
87logger = RepoLogger(__file__) 88logger = RepoLogger(__file__)
88 89
89 90
91def _SafeCheckoutOrder(checkouts: List[Project]) -> List[List[Project]]:
92 """Generate a sequence of checkouts that is safe to perform. The client
93 should checkout everything from n-th index before moving to n+1.
94
95 This is only useful if manifest contains nested projects.
96
97 E.g. if foo, foo/bar and foo/bar/baz are project paths, then foo needs to
98 finish before foo/bar can proceed, and foo/bar needs to finish before
99 foo/bar/baz."""
100 res = [[]]
101 current = res[0]
102
103 # depth_stack contains a current stack of parent paths.
104 depth_stack = []
105 # checkouts are iterated in asc order by relpath. That way, it can easily be
106 # determined if the previous checkout is parent of the current checkout.
107 for checkout in sorted(checkouts, key=lambda x: x.relpath):
108 checkout_path = Path(checkout.relpath)
109 while depth_stack:
110 try:
111 checkout_path.relative_to(depth_stack[-1])
112 except ValueError:
113 # Path.relative_to returns ValueError if paths are not relative.
114 # TODO(sokcevic): Switch to is_relative_to once min supported
115 # version is py3.9.
116 depth_stack.pop()
117 else:
118 if len(depth_stack) >= len(res):
119 # Another depth created.
120 res.append([])
121 break
122
123 current = res[len(depth_stack)]
124 current.append(checkout)
125 depth_stack.append(checkout_path)
126
127 return res
128
129
90class _FetchOneResult(NamedTuple): 130class _FetchOneResult(NamedTuple):
91 """_FetchOne return value. 131 """_FetchOne return value.
92 132
@@ -1035,15 +1075,21 @@ later is required to fix a server side protocol bug.
1035 pm.update(msg=project.name) 1075 pm.update(msg=project.name)
1036 return ret 1076 return ret
1037 1077
1038 proc_res = self.ExecuteInParallel( 1078 for projects in _SafeCheckoutOrder(all_projects):
1039 opt.jobs_checkout, 1079 proc_res = self.ExecuteInParallel(
1040 functools.partial( 1080 opt.jobs_checkout,
1041 self._CheckoutOne, opt.detach_head, opt.force_sync, opt.verbose 1081 functools.partial(
1042 ), 1082 self._CheckoutOne,
1043 all_projects, 1083 opt.detach_head,
1044 callback=_ProcessResults, 1084 opt.force_sync,
1045 output=Progress("Checking out", len(all_projects), quiet=opt.quiet), 1085 opt.verbose,
1046 ) 1086 ),
1087 projects,
1088 callback=_ProcessResults,
1089 output=Progress(
1090 "Checking out", len(all_projects), quiet=opt.quiet
1091 ),
1092 )
1047 1093
1048 self._local_sync_state.Save() 1094 self._local_sync_state.Save()
1049 return proc_res and not err_results 1095 return proc_res and not err_results
diff --git a/tests/test_subcmds_sync.py b/tests/test_subcmds_sync.py
index af6bbef7..13e23e34 100644
--- a/tests/test_subcmds_sync.py
+++ b/tests/test_subcmds_sync.py
@@ -304,6 +304,32 @@ class LocalSyncState(unittest.TestCase):
304 self.assertEqual(self.state.GetFetchTime(projA), 5) 304 self.assertEqual(self.state.GetFetchTime(projA), 5)
305 305
306 306
307class SafeCheckoutOrder(unittest.TestCase):
308 def test_no_nested(self):
309 p_f = mock.MagicMock(relpath="f")
310 p_foo = mock.MagicMock(relpath="foo")
311 out = sync._SafeCheckoutOrder([p_f, p_foo])
312 self.assertEqual(out, [[p_f, p_foo]])
313
314 def test_basic_nested(self):
315 p_foo = p_foo = mock.MagicMock(relpath="foo")
316 p_foo_bar = mock.MagicMock(relpath="foo/bar")
317 out = sync._SafeCheckoutOrder([p_foo, p_foo_bar])
318 self.assertEqual(out, [[p_foo], [p_foo_bar]])
319
320 def test_complex_nested(self):
321 p_foo = mock.MagicMock(relpath="foo")
322 p_foo_bar = mock.MagicMock(relpath="foo/bar")
323 p_foo_bar_baz_baq = mock.MagicMock(relpath="foo/bar/baz/baq")
324 p_bar = mock.MagicMock(relpath="bar")
325 out = sync._SafeCheckoutOrder(
326 [p_foo_bar_baz_baq, p_foo, p_foo_bar, p_bar]
327 )
328 self.assertEqual(
329 out, [[p_bar, p_foo], [p_foo_bar], [p_foo_bar_baz_baq]]
330 )
331
332
307class GetPreciousObjectsState(unittest.TestCase): 333class GetPreciousObjectsState(unittest.TestCase):
308 """Tests for _GetPreciousObjectsState.""" 334 """Tests for _GetPreciousObjectsState."""
309 335