summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--project.py27
-rw-r--r--subcmds/selfupdate.py2
-rw-r--r--subcmds/sync.py95
3 files changed, 103 insertions, 21 deletions
diff --git a/project.py b/project.py
index 2b57a5fb..0da2118b 100644
--- a/project.py
+++ b/project.py
@@ -26,6 +26,7 @@ import sys
26import tarfile 26import tarfile
27import tempfile 27import tempfile
28import time 28import time
29from typing import NamedTuple
29import urllib.parse 30import urllib.parse
30 31
31from color import Coloring 32from color import Coloring
@@ -45,6 +46,14 @@ from repo_trace import IsTrace, Trace
45from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M, R_WORKTREE_M 46from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M, R_WORKTREE_M
46 47
47 48
49class SyncNetworkHalfResult(NamedTuple):
50 """Sync_NetworkHalf return value."""
51 # True if successful.
52 success: bool
53 # Did we query the remote? False when optimized_fetch is True and we have the
54 # commit already present.
55 remote_fetched: bool
56
48# Maximum sleep time allowed during retries. 57# Maximum sleep time allowed during retries.
49MAXIMUM_RETRY_SLEEP_SEC = 3600.0 58MAXIMUM_RETRY_SLEEP_SEC = 3600.0
50# +-10% random jitter is added to each Fetches retry sleep duration. 59# +-10% random jitter is added to each Fetches retry sleep duration.
@@ -1133,7 +1142,7 @@ class Project(object):
1133 if archive and not isinstance(self, MetaProject): 1142 if archive and not isinstance(self, MetaProject):
1134 if self.remote.url.startswith(('http://', 'https://')): 1143 if self.remote.url.startswith(('http://', 'https://')):
1135 _error("%s: Cannot fetch archives from http/https remotes.", self.name) 1144 _error("%s: Cannot fetch archives from http/https remotes.", self.name)
1136 return False 1145 return SyncNetworkHalfResult(False, False)
1137 1146
1138 name = self.relpath.replace('\\', '/') 1147 name = self.relpath.replace('\\', '/')
1139 name = name.replace('/', '_') 1148 name = name.replace('/', '_')
@@ -1144,19 +1153,19 @@ class Project(object):
1144 self._FetchArchive(tarpath, cwd=topdir) 1153 self._FetchArchive(tarpath, cwd=topdir)
1145 except GitError as e: 1154 except GitError as e:
1146 _error('%s', e) 1155 _error('%s', e)
1147 return False 1156 return SyncNetworkHalfResult(False, False)
1148 1157
1149 # From now on, we only need absolute tarpath 1158 # From now on, we only need absolute tarpath
1150 tarpath = os.path.join(topdir, tarpath) 1159 tarpath = os.path.join(topdir, tarpath)
1151 1160
1152 if not self._ExtractArchive(tarpath, path=topdir): 1161 if not self._ExtractArchive(tarpath, path=topdir):
1153 return False 1162 return SyncNetworkHalfResult(False, True)
1154 try: 1163 try:
1155 platform_utils.remove(tarpath) 1164 platform_utils.remove(tarpath)
1156 except OSError as e: 1165 except OSError as e:
1157 _warn("Cannot remove archive %s: %s", tarpath, str(e)) 1166 _warn("Cannot remove archive %s: %s", tarpath, str(e))
1158 self._CopyAndLinkFiles() 1167 self._CopyAndLinkFiles()
1159 return True 1168 return SyncNetworkHalfResult(True, True)
1160 1169
1161 # If the shared object dir already exists, don't try to rebootstrap with a 1170 # If the shared object dir already exists, don't try to rebootstrap with a
1162 # clone bundle download. We should have the majority of objects already. 1171 # clone bundle download. We should have the majority of objects already.
@@ -1220,9 +1229,11 @@ class Project(object):
1220 depth = self.manifest.manifestProject.depth 1229 depth = self.manifest.manifestProject.depth
1221 1230
1222 # See if we can skip the network fetch entirely. 1231 # See if we can skip the network fetch entirely.
1232 remote_fetched = False
1223 if not (optimized_fetch and 1233 if not (optimized_fetch and
1224 (ID_RE.match(self.revisionExpr) and 1234 (ID_RE.match(self.revisionExpr) and
1225 self._CheckForImmutableRevision())): 1235 self._CheckForImmutableRevision())):
1236 remote_fetched = True
1226 if not self._RemoteFetch( 1237 if not self._RemoteFetch(
1227 initial=is_new, 1238 initial=is_new,
1228 quiet=quiet, verbose=verbose, output_redir=output_redir, 1239 quiet=quiet, verbose=verbose, output_redir=output_redir,
@@ -1231,7 +1242,7 @@ class Project(object):
1231 submodules=submodules, force_sync=force_sync, 1242 submodules=submodules, force_sync=force_sync,
1232 ssh_proxy=ssh_proxy, 1243 ssh_proxy=ssh_proxy,
1233 clone_filter=clone_filter, retry_fetches=retry_fetches): 1244 clone_filter=clone_filter, retry_fetches=retry_fetches):
1234 return False 1245 return SyncNetworkHalfResult(False, remote_fetched)
1235 1246
1236 mp = self.manifest.manifestProject 1247 mp = self.manifest.manifestProject
1237 dissociate = mp.dissociate 1248 dissociate = mp.dissociate
@@ -1244,7 +1255,7 @@ class Project(object):
1244 if p.stdout and output_redir: 1255 if p.stdout and output_redir:
1245 output_redir.write(p.stdout) 1256 output_redir.write(p.stdout)
1246 if p.Wait() != 0: 1257 if p.Wait() != 0:
1247 return False 1258 return SyncNetworkHalfResult(False, remote_fetched)
1248 platform_utils.remove(alternates_file) 1259 platform_utils.remove(alternates_file)
1249 1260
1250 if self.worktree: 1261 if self.worktree:
@@ -1253,7 +1264,7 @@ class Project(object):
1253 self._InitMirrorHead() 1264 self._InitMirrorHead()
1254 platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'), 1265 platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'),
1255 missing_ok=True) 1266 missing_ok=True)
1256 return True 1267 return SyncNetworkHalfResult(True, remote_fetched)
1257 1268
1258 def PostRepoUpgrade(self): 1269 def PostRepoUpgrade(self):
1259 self._InitHooks() 1270 self._InitHooks()
@@ -3836,7 +3847,7 @@ class ManifestProject(MetaProject):
3836 is_new=is_new, quiet=not verbose, verbose=verbose, 3847 is_new=is_new, quiet=not verbose, verbose=verbose,
3837 clone_bundle=clone_bundle, current_branch_only=current_branch_only, 3848 clone_bundle=clone_bundle, current_branch_only=current_branch_only,
3838 tags=tags, submodules=submodules, clone_filter=clone_filter, 3849 tags=tags, submodules=submodules, clone_filter=clone_filter,
3839 partial_clone_exclude=self.manifest.PartialCloneExclude): 3850 partial_clone_exclude=self.manifest.PartialCloneExclude).success:
3840 r = self.GetRemote() 3851 r = self.GetRemote()
3841 print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr) 3852 print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
3842 3853
diff --git a/subcmds/selfupdate.py b/subcmds/selfupdate.py
index 282f518e..898bc3f2 100644
--- a/subcmds/selfupdate.py
+++ b/subcmds/selfupdate.py
@@ -51,7 +51,7 @@ need to be performed by an end-user.
51 _PostRepoUpgrade(self.manifest) 51 _PostRepoUpgrade(self.manifest)
52 52
53 else: 53 else:
54 if not rp.Sync_NetworkHalf(): 54 if not rp.Sync_NetworkHalf().success:
55 print("error: can't update repo", file=sys.stderr) 55 print("error: can't update repo", file=sys.stderr)
56 sys.exit(1) 56 sys.exit(1)
57 57
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 9e9c8f02..1c49b46e 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -26,6 +26,7 @@ import socket
26import sys 26import sys
27import tempfile 27import tempfile
28import time 28import time
29from typing import NamedTuple
29import urllib.error 30import urllib.error
30import urllib.parse 31import urllib.parse
31import urllib.request 32import urllib.request
@@ -71,6 +72,58 @@ REPO_BACKUP_OBJECTS = 'REPO_BACKUP_OBJECTS'
71_BACKUP_OBJECTS = os.environ.get(REPO_BACKUP_OBJECTS) != '0' 72_BACKUP_OBJECTS = os.environ.get(REPO_BACKUP_OBJECTS) != '0'
72 73
73 74
75class _FetchOneResult(NamedTuple):
76 """_FetchOne return value.
77
78 Attributes:
79 success (bool): True if successful.
80 project (Project): The fetched project.
81 start (float): The starting time.time().
82 finish (float): The ending time.time().
83 remote_fetched (bool): True if the remote was actually queried.
84 """
85 success: bool
86 project: Project
87 start: float
88 finish: float
89 remote_fetched: bool
90
91
92class _FetchResult(NamedTuple):
93 """_Fetch return value.
94
95 Attributes:
96 success (bool): True if successful.
97 projects (set[str]): The names of the git directories of fetched projects.
98 """
99 success: bool
100 projects: set[str]
101
102
103class _FetchMainResult(NamedTuple):
104 """_FetchMain return value.
105
106 Attributes:
107 all_projects (list[Project]): The fetched projects.
108 """
109 all_projects: list[Project]
110
111
112class _CheckoutOneResult(NamedTuple):
113 """_CheckoutOne return value.
114
115 Attributes:
116 success (bool): True if successful.
117 project (Project): The project.
118 start (float): The starting time.time().
119 finish (float): The ending time.time().
120 """
121 success: bool
122 project: Project
123 start: float
124 finish: float
125
126
74class Sync(Command, MirrorSafeCommand): 127class Sync(Command, MirrorSafeCommand):
75 COMMON = True 128 COMMON = True
76 MULTI_MANIFEST_SUPPORT = True 129 MULTI_MANIFEST_SUPPORT = True
@@ -412,7 +465,7 @@ later is required to fix a server side protocol bug.
412 success = False 465 success = False
413 buf = io.StringIO() 466 buf = io.StringIO()
414 try: 467 try:
415 success = project.Sync_NetworkHalf( 468 sync_result = project.Sync_NetworkHalf(
416 quiet=opt.quiet, 469 quiet=opt.quiet,
417 verbose=opt.verbose, 470 verbose=opt.verbose,
418 output_redir=buf, 471 output_redir=buf,
@@ -426,6 +479,7 @@ later is required to fix a server side protocol bug.
426 ssh_proxy=self.ssh_proxy, 479 ssh_proxy=self.ssh_proxy,
427 clone_filter=project.manifest.CloneFilter, 480 clone_filter=project.manifest.CloneFilter,
428 partial_clone_exclude=project.manifest.PartialCloneExclude) 481 partial_clone_exclude=project.manifest.PartialCloneExclude)
482 success = sync_result.success
429 483
430 output = buf.getvalue() 484 output = buf.getvalue()
431 if (opt.verbose or not success) and output: 485 if (opt.verbose or not success) and output:
@@ -443,7 +497,8 @@ later is required to fix a server side protocol bug.
443 raise 497 raise
444 498
445 finish = time.time() 499 finish = time.time()
446 return (success, project, start, finish) 500 return _FetchOneResult(success, project, start, finish,
501 sync_result.remote_fetched)
447 502
448 @classmethod 503 @classmethod
449 def _FetchInitChild(cls, ssh_proxy): 504 def _FetchInitChild(cls, ssh_proxy):
@@ -454,6 +509,7 @@ later is required to fix a server side protocol bug.
454 509
455 jobs = opt.jobs_network 510 jobs = opt.jobs_network
456 fetched = set() 511 fetched = set()
512 remote_fetched = set()
457 pm = Progress('Fetching', len(projects), delay=False, quiet=opt.quiet) 513 pm = Progress('Fetching', len(projects), delay=False, quiet=opt.quiet)
458 514
459 objdir_project_map = dict() 515 objdir_project_map = dict()
@@ -464,10 +520,16 @@ later is required to fix a server side protocol bug.
464 def _ProcessResults(results_sets): 520 def _ProcessResults(results_sets):
465 ret = True 521 ret = True
466 for results in results_sets: 522 for results in results_sets:
467 for (success, project, start, finish) in results: 523 for result in results:
524 success = result.success
525 project = result.project
526 start = result.start
527 finish = result.finish
468 self._fetch_times.Set(project, finish - start) 528 self._fetch_times.Set(project, finish - start)
469 self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK, 529 self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
470 start, finish, success) 530 start, finish, success)
531 if result.remote_fetched:
532 remote_fetched.add(project)
471 # Check for any errors before running any more tasks. 533 # Check for any errors before running any more tasks.
472 # ...we'll let existing jobs finish, though. 534 # ...we'll let existing jobs finish, though.
473 if not success: 535 if not success:
@@ -525,7 +587,7 @@ later is required to fix a server side protocol bug.
525 if not self.outer_client.manifest.IsArchive: 587 if not self.outer_client.manifest.IsArchive:
526 self._GCProjects(projects, opt, err_event) 588 self._GCProjects(projects, opt, err_event)
527 589
528 return (ret, fetched) 590 return _FetchResult(ret, fetched)
529 591
530 def _FetchMain(self, opt, args, all_projects, err_event, 592 def _FetchMain(self, opt, args, all_projects, err_event,
531 ssh_proxy, manifest): 593 ssh_proxy, manifest):
@@ -551,7 +613,9 @@ later is required to fix a server side protocol bug.
551 to_fetch.extend(all_projects) 613 to_fetch.extend(all_projects)
552 to_fetch.sort(key=self._fetch_times.Get, reverse=True) 614 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
553 615
554 success, fetched = self._Fetch(to_fetch, opt, err_event, ssh_proxy) 616 result = self._Fetch(to_fetch, opt, err_event, ssh_proxy)
617 success = result.success
618 fetched = result.projects
555 if not success: 619 if not success:
556 err_event.set() 620 err_event.set()
557 621
@@ -561,7 +625,7 @@ later is required to fix a server side protocol bug.
561 if err_event.is_set(): 625 if err_event.is_set():
562 print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr) 626 print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
563 sys.exit(1) 627 sys.exit(1)
564 return 628 return _FetchMainResult([])
565 629
566 # Iteratively fetch missing and/or nested unregistered submodules 630 # Iteratively fetch missing and/or nested unregistered submodules
567 previously_missing_set = set() 631 previously_missing_set = set()
@@ -584,12 +648,14 @@ later is required to fix a server side protocol bug.
584 if previously_missing_set == missing_set: 648 if previously_missing_set == missing_set:
585 break 649 break
586 previously_missing_set = missing_set 650 previously_missing_set = missing_set
587 success, new_fetched = self._Fetch(missing, opt, err_event, ssh_proxy) 651 result = self._Fetch(missing, opt, err_event, ssh_proxy)
652 success = result.success
653 new_fetched = result.projects
588 if not success: 654 if not success:
589 err_event.set() 655 err_event.set()
590 fetched.update(new_fetched) 656 fetched.update(new_fetched)
591 657
592 return all_projects 658 return _FetchMainResult(all_projects)
593 659
594 def _CheckoutOne(self, detach_head, force_sync, project): 660 def _CheckoutOne(self, detach_head, force_sync, project):
595 """Checkout work tree for one project 661 """Checkout work tree for one project
@@ -621,7 +687,7 @@ later is required to fix a server side protocol bug.
621 if not success: 687 if not success:
622 print('error: Cannot checkout %s' % (project.name), file=sys.stderr) 688 print('error: Cannot checkout %s' % (project.name), file=sys.stderr)
623 finish = time.time() 689 finish = time.time()
624 return (success, project, start, finish) 690 return _CheckoutOneResult(success, project, start, finish)
625 691
626 def _Checkout(self, all_projects, opt, err_results): 692 def _Checkout(self, all_projects, opt, err_results):
627 """Checkout projects listed in all_projects 693 """Checkout projects listed in all_projects
@@ -636,7 +702,11 @@ later is required to fix a server side protocol bug.
636 702
637 def _ProcessResults(pool, pm, results): 703 def _ProcessResults(pool, pm, results):
638 ret = True 704 ret = True
639 for (success, project, start, finish) in results: 705 for result in results:
706 success = result.success
707 project = result.project
708 start = result.start
709 finish = result.finish
640 self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, 710 self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
641 start, finish, success) 711 start, finish, success)
642 # Check for any errors before running any more tasks. 712 # Check for any errors before running any more tasks.
@@ -1208,8 +1278,9 @@ later is required to fix a server side protocol bug.
1208 with ssh.ProxyManager(manager) as ssh_proxy: 1278 with ssh.ProxyManager(manager) as ssh_proxy:
1209 # Initialize the socket dir once in the parent. 1279 # Initialize the socket dir once in the parent.
1210 ssh_proxy.sock() 1280 ssh_proxy.sock()
1211 all_projects = self._FetchMain(opt, args, all_projects, err_event, 1281 result = self._FetchMain(opt, args, all_projects, err_event,
1212 ssh_proxy, manifest) 1282 ssh_proxy, manifest)
1283 all_projects = result.all_projects
1213 1284
1214 if opt.network_only: 1285 if opt.network_only:
1215 return 1286 return