summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--subcmds/sync.py83
1 files changed, 34 insertions, 49 deletions
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 27c8c728..a0a68960 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -219,7 +219,7 @@ later is required to fix a server side protocol bug.
219 dest='repo_upgraded', action='store_true', 219 dest='repo_upgraded', action='store_true',
220 help=SUPPRESS_HELP) 220 help=SUPPRESS_HELP)
221 221
222 def _FetchProjectList(self, opt, projects, *args): 222 def _FetchProjectList(self, opt, projects, *args, **kwargs):
223 """Main function of the fetch threads when jobs are > 1. 223 """Main function of the fetch threads when jobs are > 1.
224 224
225 Delegates most of the work to _FetchHelper. 225 Delegates most of the work to _FetchHelper.
@@ -227,11 +227,11 @@ later is required to fix a server side protocol bug.
227 Args: 227 Args:
228 opt: Program options returned from optparse. See _Options(). 228 opt: Program options returned from optparse. See _Options().
229 projects: Projects to fetch. 229 projects: Projects to fetch.
230 *args: Remaining arguments to pass to _FetchHelper. See the 230 *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
231 _FetchHelper docstring for details. 231 _FetchHelper docstring for details.
232 """ 232 """
233 for project in projects: 233 for project in projects:
234 success = self._FetchHelper(opt, project, *args) 234 success = self._FetchHelper(opt, project, *args, **kwargs)
235 if not success and not opt.force_broken: 235 if not success and not opt.force_broken:
236 break 236 break
237 237
@@ -304,62 +304,47 @@ later is required to fix a server side protocol bug.
304 304
305 def _Fetch(self, projects, opt): 305 def _Fetch(self, projects, opt):
306 fetched = set() 306 fetched = set()
307 lock = _threading.Lock()
307 pm = Progress('Fetching projects', len(projects)) 308 pm = Progress('Fetching projects', len(projects))
308 309
309 if self.jobs == 1: 310 objdir_project_map = dict()
310 for project in projects: 311 for project in projects:
311 pm.update() 312 objdir_project_map.setdefault(project.objdir, []).append(project)
312 if not opt.quiet: 313
313 print('Fetching project %s' % project.name) 314 threads = set()
314 if project.Sync_NetworkHalf( 315 sem = _threading.Semaphore(self.jobs)
315 quiet=opt.quiet, 316 err_event = _threading.Event()
316 current_branch_only=opt.current_branch_only, 317 for project_list in objdir_project_map.values():
317 clone_bundle=not opt.no_clone_bundle, 318 # Check for any errors before running any more tasks.
318 no_tags=opt.no_tags, 319 # ...we'll let existing threads finish, though.
319 archive=self.manifest.IsArchive): 320 if err_event.isSet() and not opt.force_broken:
320 fetched.add(project.gitdir) 321 break
321 else:
322 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
323 if opt.force_broken:
324 print('warn: --force-broken, continuing to sync', file=sys.stderr)
325 else:
326 sys.exit(1)
327 else:
328 objdir_project_map = dict()
329 for project in projects:
330 objdir_project_map.setdefault(project.objdir, []).append(project)
331
332 threads = set()
333 lock = _threading.Lock()
334 sem = _threading.Semaphore(self.jobs)
335 err_event = _threading.Event()
336 for project_list in objdir_project_map.values():
337 # Check for any errors before starting any new threads.
338 # ...we'll let existing threads finish, though.
339 if err_event.isSet():
340 break
341 322
342 sem.acquire() 323 sem.acquire()
324 kwargs = dict(opt=opt,
325 projects=project_list,
326 lock=lock,
327 fetched=fetched,
328 pm=pm,
329 sem=sem,
330 err_event=err_event)
331 if self.jobs > 1:
343 t = _threading.Thread(target = self._FetchProjectList, 332 t = _threading.Thread(target = self._FetchProjectList,
344 args = (opt, 333 kwargs = kwargs)
345 project_list,
346 lock,
347 fetched,
348 pm,
349 sem,
350 err_event))
351 # Ensure that Ctrl-C will not freeze the repo process. 334 # Ensure that Ctrl-C will not freeze the repo process.
352 t.daemon = True 335 t.daemon = True
353 threads.add(t) 336 threads.add(t)
354 t.start() 337 t.start()
338 else:
339 self._FetchProjectList(**kwargs)
355 340
356 for t in threads: 341 for t in threads:
357 t.join() 342 t.join()
358 343
359 # If we saw an error, exit with code 1 so that other scripts can check. 344 # If we saw an error, exit with code 1 so that other scripts can check.
360 if err_event.isSet(): 345 if err_event.isSet():
361 print('\nerror: Exited sync due to fetch errors', file=sys.stderr) 346 print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
362 sys.exit(1) 347 sys.exit(1)
363 348
364 pm.end() 349 pm.end()
365 self._fetch_times.Save() 350 self._fetch_times.Save()