diff options
-rw-r--r-- | progress.py | 49 | ||||
-rw-r--r-- | subcmds/abandon.py | 2 | ||||
-rw-r--r-- | subcmds/checkout.py | 2 | ||||
-rw-r--r-- | subcmds/start.py | 4 | ||||
-rw-r--r-- | subcmds/sync.py | 39 |
5 files changed, 70 insertions, 26 deletions
diff --git a/progress.py b/progress.py index 4844eb88..6686ad4a 100644 --- a/progress.py +++ b/progress.py | |||
@@ -82,10 +82,10 @@ class Progress(object): | |||
82 | title, | 82 | title, |
83 | total=0, | 83 | total=0, |
84 | units="", | 84 | units="", |
85 | print_newline=False, | ||
86 | delay=True, | 85 | delay=True, |
87 | quiet=False, | 86 | quiet=False, |
88 | show_elapsed=False, | 87 | show_elapsed=False, |
88 | elide=False, | ||
89 | ): | 89 | ): |
90 | self._title = title | 90 | self._title = title |
91 | self._total = total | 91 | self._total = total |
@@ -93,7 +93,7 @@ class Progress(object): | |||
93 | self._start = time.time() | 93 | self._start = time.time() |
94 | self._show = not delay | 94 | self._show = not delay |
95 | self._units = units | 95 | self._units = units |
96 | self._print_newline = print_newline | 96 | self._elide = elide |
97 | # Only show the active jobs section if we run more than one in parallel. | 97 | # Only show the active jobs section if we run more than one in parallel. |
98 | self._show_jobs = False | 98 | self._show_jobs = False |
99 | self._active = 0 | 99 | self._active = 0 |
@@ -118,10 +118,18 @@ class Progress(object): | |||
118 | 118 | ||
119 | def _update_loop(self): | 119 | def _update_loop(self): |
120 | while True: | 120 | while True: |
121 | if self._update_event.is_set(): | 121 | self.update(inc=0) |
122 | if self._update_event.wait(timeout=1): | ||
122 | return | 123 | return |
123 | self.update(inc=0, msg=self._last_msg) | 124 | |
124 | time.sleep(1) | 125 | def _write(self, s): |
126 | s = "\r" + s | ||
127 | if self._elide: | ||
128 | col = os.get_terminal_size().columns | ||
129 | if len(s) > col: | ||
130 | s = s[: col - 1] + ".." | ||
131 | sys.stderr.write(s) | ||
132 | sys.stderr.flush() | ||
125 | 133 | ||
126 | def start(self, name): | 134 | def start(self, name): |
127 | self._active += 1 | 135 | self._active += 1 |
@@ -133,8 +141,16 @@ class Progress(object): | |||
133 | self.update(msg="finished " + name) | 141 | self.update(msg="finished " + name) |
134 | self._active -= 1 | 142 | self._active -= 1 |
135 | 143 | ||
136 | def update(self, inc=1, msg=""): | 144 | def update(self, inc=1, msg=None): |
145 | """Updates the progress indicator. | ||
146 | |||
147 | Args: | ||
148 | inc: The number of items completed. | ||
149 | msg: The message to display. If None, use the last message. | ||
150 | """ | ||
137 | self._done += inc | 151 | self._done += inc |
152 | if msg is None: | ||
153 | msg = self._last_msg | ||
138 | self._last_msg = msg | 154 | self._last_msg = msg |
139 | 155 | ||
140 | if _NOT_TTY or IsTraceToStderr(): | 156 | if _NOT_TTY or IsTraceToStderr(): |
@@ -148,10 +164,9 @@ class Progress(object): | |||
148 | return | 164 | return |
149 | 165 | ||
150 | if self._total <= 0: | 166 | if self._total <= 0: |
151 | sys.stderr.write( | 167 | self._write( |
152 | "\r%s: %d,%s" % (self._title, self._done, CSI_ERASE_LINE_AFTER) | 168 | "%s: %d,%s" % (self._title, self._done, CSI_ERASE_LINE_AFTER) |
153 | ) | 169 | ) |
154 | sys.stderr.flush() | ||
155 | else: | 170 | else: |
156 | p = (100 * self._done) / self._total | 171 | p = (100 * self._done) / self._total |
157 | if self._show_jobs: | 172 | if self._show_jobs: |
@@ -165,8 +180,8 @@ class Progress(object): | |||
165 | elapsed = f" {elapsed_str(elapsed_sec)} |" | 180 | elapsed = f" {elapsed_str(elapsed_sec)} |" |
166 | else: | 181 | else: |
167 | elapsed = "" | 182 | elapsed = "" |
168 | sys.stderr.write( | 183 | self._write( |
169 | "\r%s: %2d%% %s(%d%s/%d%s)%s %s%s%s" | 184 | "%s: %2d%% %s(%d%s/%d%s)%s %s%s" |
170 | % ( | 185 | % ( |
171 | self._title, | 186 | self._title, |
172 | p, | 187 | p, |
@@ -178,10 +193,8 @@ class Progress(object): | |||
178 | elapsed, | 193 | elapsed, |
179 | msg, | 194 | msg, |
180 | CSI_ERASE_LINE_AFTER, | 195 | CSI_ERASE_LINE_AFTER, |
181 | "\n" if self._print_newline else "", | ||
182 | ) | 196 | ) |
183 | ) | 197 | ) |
184 | sys.stderr.flush() | ||
185 | 198 | ||
186 | def end(self): | 199 | def end(self): |
187 | self._update_event.set() | 200 | self._update_event.set() |
@@ -190,15 +203,14 @@ class Progress(object): | |||
190 | 203 | ||
191 | duration = duration_str(time.time() - self._start) | 204 | duration = duration_str(time.time() - self._start) |
192 | if self._total <= 0: | 205 | if self._total <= 0: |
193 | sys.stderr.write( | 206 | self._write( |
194 | "\r%s: %d, done in %s%s\n" | 207 | "%s: %d, done in %s%s\n" |
195 | % (self._title, self._done, duration, CSI_ERASE_LINE_AFTER) | 208 | % (self._title, self._done, duration, CSI_ERASE_LINE_AFTER) |
196 | ) | 209 | ) |
197 | sys.stderr.flush() | ||
198 | else: | 210 | else: |
199 | p = (100 * self._done) / self._total | 211 | p = (100 * self._done) / self._total |
200 | sys.stderr.write( | 212 | self._write( |
201 | "\r%s: %3d%% (%d%s/%d%s), done in %s%s\n" | 213 | "%s: %3d%% (%d%s/%d%s), done in %s%s\n" |
202 | % ( | 214 | % ( |
203 | self._title, | 215 | self._title, |
204 | p, | 216 | p, |
@@ -210,4 +222,3 @@ class Progress(object): | |||
210 | CSI_ERASE_LINE_AFTER, | 222 | CSI_ERASE_LINE_AFTER, |
211 | ) | 223 | ) |
212 | ) | 224 | ) |
213 | sys.stderr.flush() | ||
diff --git a/subcmds/abandon.py b/subcmds/abandon.py index 4036f306..ded287f6 100644 --- a/subcmds/abandon.py +++ b/subcmds/abandon.py | |||
@@ -90,7 +90,7 @@ It is equivalent to "git branch -D <branchname>". | |||
90 | success[branch].append(project) | 90 | success[branch].append(project) |
91 | else: | 91 | else: |
92 | err[branch].append(project) | 92 | err[branch].append(project) |
93 | pm.update() | 93 | pm.update(msg="") |
94 | 94 | ||
95 | self.ExecuteInParallel( | 95 | self.ExecuteInParallel( |
96 | opt.jobs, | 96 | opt.jobs, |
diff --git a/subcmds/checkout.py b/subcmds/checkout.py index 08012a82..6448518f 100644 --- a/subcmds/checkout.py +++ b/subcmds/checkout.py | |||
@@ -58,7 +58,7 @@ The command is equivalent to: | |||
58 | success.append(project) | 58 | success.append(project) |
59 | else: | 59 | else: |
60 | err.append(project) | 60 | err.append(project) |
61 | pm.update() | 61 | pm.update(msg="") |
62 | 62 | ||
63 | self.ExecuteInParallel( | 63 | self.ExecuteInParallel( |
64 | opt.jobs, | 64 | opt.jobs, |
diff --git a/subcmds/start.py b/subcmds/start.py index 9baf4256..f6355126 100644 --- a/subcmds/start.py +++ b/subcmds/start.py | |||
@@ -142,14 +142,14 @@ revision specified in the manifest. | |||
142 | sync_buf = SyncBuffer(self.manifest.manifestProject.config) | 142 | sync_buf = SyncBuffer(self.manifest.manifestProject.config) |
143 | project.Sync_LocalHalf(sync_buf) | 143 | project.Sync_LocalHalf(sync_buf) |
144 | project.revisionId = gitc_project.old_revision | 144 | project.revisionId = gitc_project.old_revision |
145 | pm.update() | 145 | pm.update(msg="") |
146 | pm.end() | 146 | pm.end() |
147 | 147 | ||
148 | def _ProcessResults(_pool, pm, results): | 148 | def _ProcessResults(_pool, pm, results): |
149 | for result, project in results: | 149 | for result, project in results: |
150 | if not result: | 150 | if not result: |
151 | err.append(project) | 151 | err.append(project) |
152 | pm.update() | 152 | pm.update(msg="") |
153 | 153 | ||
154 | self.ExecuteInParallel( | 154 | self.ExecuteInParallel( |
155 | opt.jobs, | 155 | opt.jobs, |
diff --git a/subcmds/sync.py b/subcmds/sync.py index 8f73d27f..da9918b9 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
@@ -66,7 +66,7 @@ from command import ( | |||
66 | from error import RepoChangedException, GitError | 66 | from error import RepoChangedException, GitError |
67 | import platform_utils | 67 | import platform_utils |
68 | from project import SyncBuffer | 68 | from project import SyncBuffer |
69 | from progress import Progress | 69 | from progress import Progress, elapsed_str |
70 | from repo_trace import Trace | 70 | from repo_trace import Trace |
71 | import ssh | 71 | import ssh |
72 | from wrapper import Wrapper | 72 | from wrapper import Wrapper |
@@ -596,7 +596,7 @@ later is required to fix a server side protocol bug. | |||
596 | The projects we're given share the same underlying git object store, so | 596 | The projects we're given share the same underlying git object store, so |
597 | we have to fetch them in serial. | 597 | we have to fetch them in serial. |
598 | 598 | ||
599 | Delegates most of the work to _FetchHelper. | 599 | Delegates most of the work to _FetchOne. |
600 | 600 | ||
601 | Args: | 601 | Args: |
602 | opt: Program options returned from optparse. See _Options(). | 602 | opt: Program options returned from optparse. See _Options(). |
@@ -615,6 +615,8 @@ later is required to fix a server side protocol bug. | |||
615 | Whether the fetch was successful. | 615 | Whether the fetch was successful. |
616 | """ | 616 | """ |
617 | start = time.time() | 617 | start = time.time() |
618 | k = f"{project.name} @ {project.relpath}" | ||
619 | self._sync_dict[k] = start | ||
618 | success = False | 620 | success = False |
619 | remote_fetched = False | 621 | remote_fetched = False |
620 | buf = io.StringIO() | 622 | buf = io.StringIO() |
@@ -660,15 +662,31 @@ later is required to fix a server side protocol bug. | |||
660 | % (project.name, type(e).__name__, str(e)), | 662 | % (project.name, type(e).__name__, str(e)), |
661 | file=sys.stderr, | 663 | file=sys.stderr, |
662 | ) | 664 | ) |
665 | del self._sync_dict[k] | ||
663 | raise | 666 | raise |
664 | 667 | ||
665 | finish = time.time() | 668 | finish = time.time() |
669 | del self._sync_dict[k] | ||
666 | return _FetchOneResult(success, project, start, finish, remote_fetched) | 670 | return _FetchOneResult(success, project, start, finish, remote_fetched) |
667 | 671 | ||
668 | @classmethod | 672 | @classmethod |
669 | def _FetchInitChild(cls, ssh_proxy): | 673 | def _FetchInitChild(cls, ssh_proxy): |
670 | cls.ssh_proxy = ssh_proxy | 674 | cls.ssh_proxy = ssh_proxy |
671 | 675 | ||
676 | def _GetLongestSyncMessage(self): | ||
677 | if len(self._sync_dict) == 0: | ||
678 | return None | ||
679 | |||
680 | earliest_time = float("inf") | ||
681 | earliest_proj = None | ||
682 | for project, t in self._sync_dict.items(): | ||
683 | if t < earliest_time: | ||
684 | earliest_time = t | ||
685 | earliest_proj = project | ||
686 | |||
687 | elapsed = time.time() - earliest_time | ||
688 | return f"{elapsed_str(elapsed)} {earliest_proj}" | ||
689 | |||
672 | def _Fetch(self, projects, opt, err_event, ssh_proxy): | 690 | def _Fetch(self, projects, opt, err_event, ssh_proxy): |
673 | ret = True | 691 | ret = True |
674 | 692 | ||
@@ -681,8 +699,22 @@ later is required to fix a server side protocol bug. | |||
681 | delay=False, | 699 | delay=False, |
682 | quiet=opt.quiet, | 700 | quiet=opt.quiet, |
683 | show_elapsed=True, | 701 | show_elapsed=True, |
702 | elide=True, | ||
684 | ) | 703 | ) |
685 | 704 | ||
705 | self._sync_dict = multiprocessing.Manager().dict() | ||
706 | sync_event = _threading.Event() | ||
707 | |||
708 | def _MonitorSyncLoop(): | ||
709 | while True: | ||
710 | pm.update(inc=0, msg=self._GetLongestSyncMessage()) | ||
711 | if sync_event.wait(timeout=1): | ||
712 | return | ||
713 | |||
714 | sync_progress_thread = _threading.Thread(target=_MonitorSyncLoop) | ||
715 | sync_progress_thread.daemon = True | ||
716 | sync_progress_thread.start() | ||
717 | |||
686 | objdir_project_map = dict() | 718 | objdir_project_map = dict() |
687 | for project in projects: | 719 | for project in projects: |
688 | objdir_project_map.setdefault(project.objdir, []).append(project) | 720 | objdir_project_map.setdefault(project.objdir, []).append(project) |
@@ -712,7 +744,7 @@ later is required to fix a server side protocol bug. | |||
712 | ret = False | 744 | ret = False |
713 | else: | 745 | else: |
714 | fetched.add(project.gitdir) | 746 | fetched.add(project.gitdir) |
715 | pm.update(msg=f"Last synced: {project.name}") | 747 | pm.update() |
716 | if not ret and opt.fail_fast: | 748 | if not ret and opt.fail_fast: |
717 | break | 749 | break |
718 | return ret | 750 | return ret |
@@ -764,6 +796,7 @@ later is required to fix a server side protocol bug. | |||
764 | # crash. | 796 | # crash. |
765 | del Sync.ssh_proxy | 797 | del Sync.ssh_proxy |
766 | 798 | ||
799 | sync_event.set() | ||
767 | pm.end() | 800 | pm.end() |
768 | self._fetch_times.Save() | 801 | self._fetch_times.Save() |
769 | 802 | ||