diff options
-rw-r--r-- | git_command.py | 47 | ||||
-rw-r--r-- | git_config.py | 10 | ||||
-rw-r--r-- | git_refs.py | 57 | ||||
-rwxr-xr-x | main.py | 33 | ||||
-rw-r--r-- | project.py | 24 | ||||
-rw-r--r-- | repo_trace.py | 110 | ||||
-rwxr-xr-x | run_tests | 1 | ||||
-rw-r--r-- | ssh.py | 39 | ||||
-rw-r--r-- | subcmds/gitc_init.py | 3 | ||||
-rw-r--r-- | subcmds/sync.py | 109 | ||||
-rw-r--r-- | tests/test_git_config.py | 14 | ||||
-rw-r--r-- | tests/test_git_superproject.py | 2 | ||||
-rw-r--r-- | tests/test_manifest_xml.py | 16 | ||||
-rw-r--r-- | tests/test_project.py | 8 | ||||
-rw-r--r-- | tests/test_subcmds_sync.py | 55 |
15 files changed, 149 insertions, 379 deletions
diff --git a/git_command.py b/git_command.py index 56e18e02..19100fa9 100644 --- a/git_command.py +++ b/git_command.py | |||
@@ -230,11 +230,12 @@ class GitCommand(object): | |||
230 | stderr = (subprocess.STDOUT if merge_output else | 230 | stderr = (subprocess.STDOUT if merge_output else |
231 | (subprocess.PIPE if capture_stderr else None)) | 231 | (subprocess.PIPE if capture_stderr else None)) |
232 | 232 | ||
233 | dbg = '' | ||
234 | if IsTrace(): | 233 | if IsTrace(): |
235 | global LAST_CWD | 234 | global LAST_CWD |
236 | global LAST_GITDIR | 235 | global LAST_GITDIR |
237 | 236 | ||
237 | dbg = '' | ||
238 | |||
238 | if cwd and LAST_CWD != cwd: | 239 | if cwd and LAST_CWD != cwd: |
239 | if LAST_GITDIR or LAST_CWD: | 240 | if LAST_GITDIR or LAST_CWD: |
240 | dbg += '\n' | 241 | dbg += '\n' |
@@ -262,31 +263,31 @@ class GitCommand(object): | |||
262 | dbg += ' 2>|' | 263 | dbg += ' 2>|' |
263 | elif stderr == subprocess.STDOUT: | 264 | elif stderr == subprocess.STDOUT: |
264 | dbg += ' 2>&1' | 265 | dbg += ' 2>&1' |
266 | Trace('%s', dbg) | ||
267 | |||
268 | try: | ||
269 | p = subprocess.Popen(command, | ||
270 | cwd=cwd, | ||
271 | env=env, | ||
272 | encoding='utf-8', | ||
273 | errors='backslashreplace', | ||
274 | stdin=stdin, | ||
275 | stdout=stdout, | ||
276 | stderr=stderr) | ||
277 | except Exception as e: | ||
278 | raise GitError('%s: %s' % (command[1], e)) | ||
265 | 279 | ||
266 | with Trace('git command %s %s with debug: %s', LAST_GITDIR, command, dbg): | 280 | if ssh_proxy: |
267 | try: | 281 | ssh_proxy.add_client(p) |
268 | p = subprocess.Popen(command, | ||
269 | cwd=cwd, | ||
270 | env=env, | ||
271 | encoding='utf-8', | ||
272 | errors='backslashreplace', | ||
273 | stdin=stdin, | ||
274 | stdout=stdout, | ||
275 | stderr=stderr) | ||
276 | except Exception as e: | ||
277 | raise GitError('%s: %s' % (command[1], e)) | ||
278 | |||
279 | if ssh_proxy: | ||
280 | ssh_proxy.add_client(p) | ||
281 | 282 | ||
282 | self.process = p | 283 | self.process = p |
283 | 284 | ||
284 | try: | 285 | try: |
285 | self.stdout, self.stderr = p.communicate(input=input) | 286 | self.stdout, self.stderr = p.communicate(input=input) |
286 | finally: | 287 | finally: |
287 | if ssh_proxy: | 288 | if ssh_proxy: |
288 | ssh_proxy.remove_client(p) | 289 | ssh_proxy.remove_client(p) |
289 | self.rc = p.wait() | 290 | self.rc = p.wait() |
290 | 291 | ||
291 | @staticmethod | 292 | @staticmethod |
292 | def _GetBasicEnv(): | 293 | def _GetBasicEnv(): |
diff --git a/git_config.py b/git_config.py index 94378e9a..6f80ae08 100644 --- a/git_config.py +++ b/git_config.py | |||
@@ -219,8 +219,8 @@ class GitConfig(object): | |||
219 | """Set the value(s) for a key. | 219 | """Set the value(s) for a key. |
220 | Only this configuration file is modified. | 220 | Only this configuration file is modified. |
221 | 221 | ||
222 | The supplied value should be either a string, or a list of strings (to | 222 | The supplied value should be either a string, |
223 | store multiple values), or None (to delete the key). | 223 | or a list of strings (to store multiple values). |
224 | """ | 224 | """ |
225 | key = _key(name) | 225 | key = _key(name) |
226 | 226 | ||
@@ -349,9 +349,9 @@ class GitConfig(object): | |||
349 | except OSError: | 349 | except OSError: |
350 | return None | 350 | return None |
351 | try: | 351 | try: |
352 | with Trace(': parsing %s', self.file): | 352 | Trace(': parsing %s', self.file) |
353 | with open(self._json) as fd: | 353 | with open(self._json) as fd: |
354 | return json.load(fd) | 354 | return json.load(fd) |
355 | except (IOError, ValueError): | 355 | except (IOError, ValueError): |
356 | platform_utils.remove(self._json, missing_ok=True) | 356 | platform_utils.remove(self._json, missing_ok=True) |
357 | return None | 357 | return None |
diff --git a/git_refs.py b/git_refs.py index 300d2b30..2d4a8090 100644 --- a/git_refs.py +++ b/git_refs.py | |||
@@ -67,37 +67,38 @@ class GitRefs(object): | |||
67 | self._LoadAll() | 67 | self._LoadAll() |
68 | 68 | ||
69 | def _NeedUpdate(self): | 69 | def _NeedUpdate(self): |
70 | with Trace(': scan refs %s', self._gitdir): | 70 | Trace(': scan refs %s', self._gitdir) |
71 | for name, mtime in self._mtime.items(): | 71 | |
72 | try: | 72 | for name, mtime in self._mtime.items(): |
73 | if mtime != os.path.getmtime(os.path.join(self._gitdir, name)): | 73 | try: |
74 | return True | 74 | if mtime != os.path.getmtime(os.path.join(self._gitdir, name)): |
75 | except OSError: | ||
76 | return True | 75 | return True |
77 | return False | 76 | except OSError: |
77 | return True | ||
78 | return False | ||
78 | 79 | ||
79 | def _LoadAll(self): | 80 | def _LoadAll(self): |
80 | with Trace(': load refs %s', self._gitdir): | 81 | Trace(': load refs %s', self._gitdir) |
81 | 82 | ||
82 | self._phyref = {} | 83 | self._phyref = {} |
83 | self._symref = {} | 84 | self._symref = {} |
84 | self._mtime = {} | 85 | self._mtime = {} |
85 | 86 | ||
86 | self._ReadPackedRefs() | 87 | self._ReadPackedRefs() |
87 | self._ReadLoose('refs/') | 88 | self._ReadLoose('refs/') |
88 | self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD) | 89 | self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD) |
89 | 90 | ||
90 | scan = self._symref | 91 | scan = self._symref |
91 | attempts = 0 | 92 | attempts = 0 |
92 | while scan and attempts < 5: | 93 | while scan and attempts < 5: |
93 | scan_next = {} | 94 | scan_next = {} |
94 | for name, dest in scan.items(): | 95 | for name, dest in scan.items(): |
95 | if dest in self._phyref: | 96 | if dest in self._phyref: |
96 | self._phyref[name] = self._phyref[dest] | 97 | self._phyref[name] = self._phyref[dest] |
97 | else: | 98 | else: |
98 | scan_next[name] = dest | 99 | scan_next[name] = dest |
99 | scan = scan_next | 100 | scan = scan_next |
100 | attempts += 1 | 101 | attempts += 1 |
101 | 102 | ||
102 | def _ReadPackedRefs(self): | 103 | def _ReadPackedRefs(self): |
103 | path = os.path.join(self._gitdir, 'packed-refs') | 104 | path = os.path.join(self._gitdir, 'packed-refs') |
@@ -37,7 +37,7 @@ except ImportError: | |||
37 | 37 | ||
38 | from color import SetDefaultColoring | 38 | from color import SetDefaultColoring |
39 | import event_log | 39 | import event_log |
40 | from repo_trace import SetTrace, Trace, SetTraceToStderr | 40 | from repo_trace import SetTrace |
41 | from git_command import user_agent | 41 | from git_command import user_agent |
42 | from git_config import RepoConfig | 42 | from git_config import RepoConfig |
43 | from git_trace2_event_log import EventLog | 43 | from git_trace2_event_log import EventLog |
@@ -109,9 +109,6 @@ global_options.add_option('--color', | |||
109 | global_options.add_option('--trace', | 109 | global_options.add_option('--trace', |
110 | dest='trace', action='store_true', | 110 | dest='trace', action='store_true', |
111 | help='trace git command execution (REPO_TRACE=1)') | 111 | help='trace git command execution (REPO_TRACE=1)') |
112 | global_options.add_option('--trace_to_stderr', | ||
113 | dest='trace_to_stderr', action='store_true', | ||
114 | help='trace outputs go to stderr in addition to .repo/TRACE_FILE') | ||
115 | global_options.add_option('--trace-python', | 112 | global_options.add_option('--trace-python', |
116 | dest='trace_python', action='store_true', | 113 | dest='trace_python', action='store_true', |
117 | help='trace python command execution') | 114 | help='trace python command execution') |
@@ -201,6 +198,9 @@ class _Repo(object): | |||
201 | """Execute the requested subcommand.""" | 198 | """Execute the requested subcommand.""" |
202 | result = 0 | 199 | result = 0 |
203 | 200 | ||
201 | if gopts.trace: | ||
202 | SetTrace() | ||
203 | |||
204 | # Handle options that terminate quickly first. | 204 | # Handle options that terminate quickly first. |
205 | if gopts.help or gopts.help_all: | 205 | if gopts.help or gopts.help_all: |
206 | self._PrintHelp(short=False, all_commands=gopts.help_all) | 206 | self._PrintHelp(short=False, all_commands=gopts.help_all) |
@@ -652,26 +652,17 @@ def _Main(argv): | |||
652 | Version.wrapper_path = opt.wrapper_path | 652 | Version.wrapper_path = opt.wrapper_path |
653 | 653 | ||
654 | repo = _Repo(opt.repodir) | 654 | repo = _Repo(opt.repodir) |
655 | |||
656 | try: | 655 | try: |
657 | init_http() | 656 | init_http() |
658 | name, gopts, argv = repo._ParseArgs(argv) | 657 | name, gopts, argv = repo._ParseArgs(argv) |
659 | 658 | run = lambda: repo._Run(name, gopts, argv) or 0 | |
660 | if gopts.trace: | 659 | if gopts.trace_python: |
661 | SetTrace() | 660 | import trace |
662 | 661 | tracer = trace.Trace(count=False, trace=True, timing=True, | |
663 | if gopts.trace_to_stderr: | 662 | ignoredirs=set(sys.path[1:])) |
664 | SetTraceToStderr() | 663 | result = tracer.runfunc(run) |
665 | 664 | else: | |
666 | with Trace('starting new command: %s', ', '.join([name] + argv), first_trace=True): | 665 | result = run() |
667 | run = lambda: repo._Run(name, gopts, argv) or 0 | ||
668 | if gopts.trace_python: | ||
669 | import trace | ||
670 | tracer = trace.Trace(count=False, trace=True, timing=True, | ||
671 | ignoredirs=set(sys.path[1:])) | ||
672 | result = tracer.runfunc(run) | ||
673 | else: | ||
674 | result = run() | ||
675 | except KeyboardInterrupt: | 666 | except KeyboardInterrupt: |
676 | print('aborted by user', file=sys.stderr) | 667 | print('aborted by user', file=sys.stderr) |
677 | result = 1 | 668 | result = 1 |
@@ -41,7 +41,7 @@ from error import ManifestInvalidRevisionError, ManifestInvalidPathError | |||
41 | from error import NoManifestException, ManifestParseError | 41 | from error import NoManifestException, ManifestParseError |
42 | import platform_utils | 42 | import platform_utils |
43 | import progress | 43 | import progress |
44 | from repo_trace import Trace | 44 | from repo_trace import IsTrace, Trace |
45 | 45 | ||
46 | from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M, R_WORKTREE_M | 46 | from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M, R_WORKTREE_M |
47 | 47 | ||
@@ -59,7 +59,7 @@ MAXIMUM_RETRY_SLEEP_SEC = 3600.0 | |||
59 | # +-10% random jitter is added to each Fetches retry sleep duration. | 59 | # +-10% random jitter is added to each Fetches retry sleep duration. |
60 | RETRY_JITTER_PERCENT = 0.1 | 60 | RETRY_JITTER_PERCENT = 0.1 |
61 | 61 | ||
62 | # Whether to use alternates. Switching back and forth is *NOT* supported. | 62 | # Whether to use alternates. |
63 | # TODO(vapier): Remove knob once behavior is verified. | 63 | # TODO(vapier): Remove knob once behavior is verified. |
64 | _ALTERNATES = os.environ.get('REPO_USE_ALTERNATES') == '1' | 64 | _ALTERNATES = os.environ.get('REPO_USE_ALTERNATES') == '1' |
65 | 65 | ||
@@ -2416,16 +2416,16 @@ class Project(object): | |||
2416 | srcUrl = 'http' + srcUrl[len('persistent-http'):] | 2416 | srcUrl = 'http' + srcUrl[len('persistent-http'):] |
2417 | cmd += [srcUrl] | 2417 | cmd += [srcUrl] |
2418 | 2418 | ||
2419 | proc = None | 2419 | if IsTrace(): |
2420 | with Trace('Fetching bundle: %s', ' '.join(cmd)): | 2420 | Trace('%s', ' '.join(cmd)) |
2421 | if verbose: | 2421 | if verbose: |
2422 | print('%s: Downloading bundle: %s' % (self.name, srcUrl)) | 2422 | print('%s: Downloading bundle: %s' % (self.name, srcUrl)) |
2423 | stdout = None if verbose else subprocess.PIPE | 2423 | stdout = None if verbose else subprocess.PIPE |
2424 | stderr = None if verbose else subprocess.STDOUT | 2424 | stderr = None if verbose else subprocess.STDOUT |
2425 | try: | 2425 | try: |
2426 | proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr) | 2426 | proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr) |
2427 | except OSError: | 2427 | except OSError: |
2428 | return False | 2428 | return False |
2429 | 2429 | ||
2430 | (output, _) = proc.communicate() | 2430 | (output, _) = proc.communicate() |
2431 | curlret = proc.returncode | 2431 | curlret = proc.returncode |
diff --git a/repo_trace.py b/repo_trace.py index 0ff3b694..7be0c045 100644 --- a/repo_trace.py +++ b/repo_trace.py | |||
@@ -15,128 +15,26 @@ | |||
15 | """Logic for tracing repo interactions. | 15 | """Logic for tracing repo interactions. |
16 | 16 | ||
17 | Activated via `repo --trace ...` or `REPO_TRACE=1 repo ...`. | 17 | Activated via `repo --trace ...` or `REPO_TRACE=1 repo ...`. |
18 | |||
19 | Temporary: Tracing is always on. Set `REPO_TRACE=0` to turn off. | ||
20 | To also include trace outputs in stderr do `repo --trace_to_stderr ...` | ||
21 | """ | 18 | """ |
22 | 19 | ||
23 | import sys | 20 | import sys |
24 | import os | 21 | import os |
25 | import tempfile | ||
26 | import time | ||
27 | from contextlib import ContextDecorator | ||
28 | 22 | ||
29 | # Env var to implicitly turn on tracing. | 23 | # Env var to implicitly turn on tracing. |
30 | REPO_TRACE = 'REPO_TRACE' | 24 | REPO_TRACE = 'REPO_TRACE' |
31 | 25 | ||
32 | # Temporarily set tracing to always on unless user expicitly sets to 0. | 26 | _TRACE = os.environ.get(REPO_TRACE) == '1' |
33 | _TRACE = os.environ.get(REPO_TRACE) != '0' | ||
34 | |||
35 | _TRACE_TO_STDERR = False | ||
36 | |||
37 | _TRACE_FILE = None | ||
38 | |||
39 | _TRACE_FILE_NAME = 'TRACE_FILE' | ||
40 | |||
41 | _MAX_SIZE = 5 # in mb | ||
42 | |||
43 | _NEW_COMMAND_SEP = '+++++++++++++++NEW COMMAND+++++++++++++++++++' | ||
44 | |||
45 | |||
46 | def IsStraceToStderr(): | ||
47 | return _TRACE_TO_STDERR | ||
48 | 27 | ||
49 | 28 | ||
50 | def IsTrace(): | 29 | def IsTrace(): |
51 | return _TRACE | 30 | return _TRACE |
52 | 31 | ||
53 | 32 | ||
54 | def SetTraceToStderr(): | ||
55 | global _TRACE_TO_STDERR | ||
56 | _TRACE_TO_STDERR = True | ||
57 | |||
58 | |||
59 | def SetTrace(): | 33 | def SetTrace(): |
60 | global _TRACE | 34 | global _TRACE |
61 | _TRACE = True | 35 | _TRACE = True |
62 | 36 | ||
63 | 37 | ||
64 | def _SetTraceFile(): | 38 | def Trace(fmt, *args): |
65 | global _TRACE_FILE | 39 | if IsTrace(): |
66 | _TRACE_FILE = _GetTraceFile() | 40 | print(fmt % args, file=sys.stderr) |
67 | |||
68 | |||
69 | class Trace(ContextDecorator): | ||
70 | |||
71 | def _time(self): | ||
72 | """Generate nanoseconds of time in a py3.6 safe way""" | ||
73 | return int(time.time()*1e+9) | ||
74 | |||
75 | def __init__(self, fmt, *args, first_trace=False): | ||
76 | if not IsTrace(): | ||
77 | return | ||
78 | self._trace_msg = fmt % args | ||
79 | |||
80 | if not _TRACE_FILE: | ||
81 | _SetTraceFile() | ||
82 | |||
83 | if first_trace: | ||
84 | _ClearOldTraces() | ||
85 | self._trace_msg = '%s %s' % (_NEW_COMMAND_SEP, self._trace_msg) | ||
86 | |||
87 | |||
88 | def __enter__(self): | ||
89 | if not IsTrace(): | ||
90 | return self | ||
91 | |||
92 | print_msg = f"PID: {os.getpid()} START: {self._time()} :" + self._trace_msg + '\n' | ||
93 | |||
94 | with open(_TRACE_FILE, 'a') as f: | ||
95 | print(print_msg, file=f) | ||
96 | |||
97 | if _TRACE_TO_STDERR: | ||
98 | print(print_msg, file=sys.stderr) | ||
99 | |||
100 | return self | ||
101 | |||
102 | def __exit__(self, *exc): | ||
103 | if not IsTrace(): | ||
104 | return False | ||
105 | |||
106 | print_msg = f"PID: {os.getpid()} END: {self._time()} :" + self._trace_msg + '\n' | ||
107 | |||
108 | with open(_TRACE_FILE, 'a') as f: | ||
109 | print(print_msg, file=f) | ||
110 | |||
111 | if _TRACE_TO_STDERR: | ||
112 | print(print_msg, file=sys.stderr) | ||
113 | |||
114 | return False | ||
115 | |||
116 | |||
117 | def _GetTraceFile(): | ||
118 | """Get the trace file or create one.""" | ||
119 | # TODO: refactor to pass repodir to Trace. | ||
120 | repo_dir = os.path.dirname(os.path.dirname(__file__)) | ||
121 | trace_file = os.path.join(repo_dir, _TRACE_FILE_NAME) | ||
122 | print('Trace outputs in %s' % trace_file) | ||
123 | return trace_file | ||
124 | |||
125 | def _ClearOldTraces(): | ||
126 | """Clear traces from old commands if trace file is too big. | ||
127 | |||
128 | Note: If the trace file contains output from two `repo` | ||
129 | commands that were running at the same time, this | ||
130 | will not work precisely. | ||
131 | """ | ||
132 | if os.path.isfile(_TRACE_FILE): | ||
133 | while os.path.getsize(_TRACE_FILE)/(1024*1024) > _MAX_SIZE: | ||
134 | temp = tempfile.NamedTemporaryFile(mode='w', delete=False) | ||
135 | with open(_TRACE_FILE, 'r', errors='ignore') as fin: | ||
136 | trace_lines = fin.readlines() | ||
137 | for i , l in enumerate(trace_lines): | ||
138 | if 'END:' in l and _NEW_COMMAND_SEP in l: | ||
139 | temp.writelines(trace_lines[i+1:]) | ||
140 | break | ||
141 | temp.close() | ||
142 | os.replace(temp.name, _TRACE_FILE) | ||
@@ -20,7 +20,6 @@ import os | |||
20 | import shutil | 20 | import shutil |
21 | import subprocess | 21 | import subprocess |
22 | import sys | 22 | import sys |
23 | import repo_trace | ||
24 | 23 | ||
25 | 24 | ||
26 | def find_pytest(): | 25 | def find_pytest(): |
@@ -182,29 +182,28 @@ class ProxyManager: | |||
182 | # be important because we can't tell that that 'git@myhost.com' is the same | 182 | # be important because we can't tell that that 'git@myhost.com' is the same |
183 | # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file. | 183 | # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file. |
184 | check_command = command_base + ['-O', 'check'] | 184 | check_command = command_base + ['-O', 'check'] |
185 | with Trace('Call to ssh (check call): %s', ' '.join(check_command)): | 185 | try: |
186 | try: | 186 | Trace(': %s', ' '.join(check_command)) |
187 | check_process = subprocess.Popen(check_command, | 187 | check_process = subprocess.Popen(check_command, |
188 | stdout=subprocess.PIPE, | 188 | stdout=subprocess.PIPE, |
189 | stderr=subprocess.PIPE) | 189 | stderr=subprocess.PIPE) |
190 | check_process.communicate() # read output, but ignore it... | 190 | check_process.communicate() # read output, but ignore it... |
191 | isnt_running = check_process.wait() | 191 | isnt_running = check_process.wait() |
192 | 192 | ||
193 | if not isnt_running: | 193 | if not isnt_running: |
194 | # Our double-check found that the master _was_ infact running. Add to | 194 | # Our double-check found that the master _was_ infact running. Add to |
195 | # the list of keys. | 195 | # the list of keys. |
196 | self._master_keys[key] = True | 196 | self._master_keys[key] = True |
197 | return True | 197 | return True |
198 | except Exception: | 198 | except Exception: |
199 | # Ignore excpetions. We we will fall back to the normal command and | 199 | # Ignore excpetions. We we will fall back to the normal command and print |
200 | # print to the log there. | 200 | # to the log there. |
201 | pass | 201 | pass |
202 | 202 | ||
203 | command = command_base[:1] + ['-M', '-N'] + command_base[1:] | 203 | command = command_base[:1] + ['-M', '-N'] + command_base[1:] |
204 | p = None | ||
205 | try: | 204 | try: |
206 | with Trace('Call to ssh: %s', ' '.join(command)): | 205 | Trace(': %s', ' '.join(command)) |
207 | p = subprocess.Popen(command) | 206 | p = subprocess.Popen(command) |
208 | except Exception as e: | 207 | except Exception as e: |
209 | self._master_broken.value = True | 208 | self._master_broken.value = True |
210 | print('\nwarn: cannot enable ssh control master for %s:%s\n%s' | 209 | print('\nwarn: cannot enable ssh control master for %s:%s\n%s' |
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py index e3a5813d..1d81baf5 100644 --- a/subcmds/gitc_init.py +++ b/subcmds/gitc_init.py | |||
@@ -68,8 +68,7 @@ use for this GITC client. | |||
68 | sys.exit(1) | 68 | sys.exit(1) |
69 | manifest_file = opt.manifest_file | 69 | manifest_file = opt.manifest_file |
70 | 70 | ||
71 | manifest = GitcManifest(self.repodir, os.path.join(self.client_dir, | 71 | manifest = GitcManifest(self.repodir, gitc_client) |
72 | '.manifest')) | ||
73 | manifest.Override(manifest_file) | 72 | manifest.Override(manifest_file) |
74 | gitc_utils.generate_gitc_manifest(None, manifest) | 73 | gitc_utils.generate_gitc_manifest(None, manifest) |
75 | print('Please run `cd %s` to view your GITC client.' % | 74 | print('Please run `cd %s` to view your GITC client.' % |
diff --git a/subcmds/sync.py b/subcmds/sync.py index 83c9ad36..fe63b484 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
@@ -60,7 +60,7 @@ from error import RepoChangedException, GitError, ManifestParseError | |||
60 | import platform_utils | 60 | import platform_utils |
61 | from project import SyncBuffer | 61 | from project import SyncBuffer |
62 | from progress import Progress | 62 | from progress import Progress |
63 | from repo_trace import Trace | 63 | from repo_trace import IsTrace, Trace |
64 | import ssh | 64 | import ssh |
65 | from wrapper import Wrapper | 65 | from wrapper import Wrapper |
66 | from manifest_xml import GitcManifest | 66 | from manifest_xml import GitcManifest |
@@ -739,6 +739,7 @@ later is required to fix a server side protocol bug. | |||
739 | bak_dir = os.path.join(objdir, '.repo', 'pack.bak') | 739 | bak_dir = os.path.join(objdir, '.repo', 'pack.bak') |
740 | if not _BACKUP_OBJECTS or not platform_utils.isdir(pack_dir): | 740 | if not _BACKUP_OBJECTS or not platform_utils.isdir(pack_dir): |
741 | return | 741 | return |
742 | saved = [] | ||
742 | files = set(platform_utils.listdir(pack_dir)) | 743 | files = set(platform_utils.listdir(pack_dir)) |
743 | to_backup = [] | 744 | to_backup = [] |
744 | for f in files: | 745 | for f in files: |
@@ -750,83 +751,12 @@ later is required to fix a server side protocol bug. | |||
750 | for fname in to_backup: | 751 | for fname in to_backup: |
751 | bak_fname = os.path.join(bak_dir, fname) | 752 | bak_fname = os.path.join(bak_dir, fname) |
752 | if not os.path.exists(bak_fname): | 753 | if not os.path.exists(bak_fname): |
753 | with Trace('%s saved %s', bare_git._project.name, fname): | 754 | saved.append(fname) |
754 | # Use a tmp file so that we are sure of a complete copy. | 755 | # Use a tmp file so that we are sure of a complete copy. |
755 | shutil.copy(os.path.join(pack_dir, fname), bak_fname + '.tmp') | 756 | shutil.copy(os.path.join(pack_dir, fname), bak_fname + '.tmp') |
756 | shutil.move(bak_fname + '.tmp', bak_fname) | 757 | shutil.move(bak_fname + '.tmp', bak_fname) |
757 | 758 | if saved: | |
758 | @staticmethod | 759 | Trace('%s saved %s', bare_git._project.name, ' '.join(saved)) |
759 | def _GetPreciousObjectsState(project: Project, opt): | ||
760 | """Get the preciousObjects state for the project. | ||
761 | |||
762 | Args: | ||
763 | project (Project): the project to examine, and possibly correct. | ||
764 | opt (optparse.Values): options given to sync. | ||
765 | |||
766 | Returns: | ||
767 | Expected state of extensions.preciousObjects: | ||
768 | False: Should be disabled. (not present) | ||
769 | True: Should be enabled. | ||
770 | """ | ||
771 | if project.use_git_worktrees: | ||
772 | return False | ||
773 | projects = project.manifest.GetProjectsWithName(project.name, | ||
774 | all_manifests=True) | ||
775 | if len(projects) == 1: | ||
776 | return False | ||
777 | relpath = project.RelPath(local=opt.this_manifest_only) | ||
778 | if len(projects) > 1: | ||
779 | # Objects are potentially shared with another project. | ||
780 | # See the logic in Project.Sync_NetworkHalf regarding UseAlternates. | ||
781 | # - When False, shared projects share (via symlink) | ||
782 | # .repo/project-objects/{PROJECT_NAME}.git as the one-and-only objects | ||
783 | # directory. All objects are precious, since there is no project with a | ||
784 | # complete set of refs. | ||
785 | # - When True, shared projects share (via info/alternates) | ||
786 | # .repo/project-objects/{PROJECT_NAME}.git as an alternate object store, | ||
787 | # which is written only on the first clone of the project, and is not | ||
788 | # written subsequently. (When Sync_NetworkHalf sees that it exists, it | ||
789 | # makes sure that the alternates file points there, and uses a | ||
790 | # project-local .git/objects directory for all syncs going forward. | ||
791 | # We do not support switching between the options. The environment | ||
792 | # variable is present for testing and migration only. | ||
793 | return not project.UseAlternates | ||
794 | print(f'\r{relpath}: project not found in manifest.', file=sys.stderr) | ||
795 | return False | ||
796 | |||
797 | def _RepairPreciousObjectsState(self, project: Project, opt): | ||
798 | """Correct the preciousObjects state for the project. | ||
799 | |||
800 | Args: | ||
801 | project (Project): the project to examine, and possibly correct. | ||
802 | opt (optparse.Values): options given to sync. | ||
803 | """ | ||
804 | expected = self._GetPreciousObjectsState(project, opt) | ||
805 | actual = project.config.GetBoolean('extensions.preciousObjects') or False | ||
806 | relpath = project.RelPath(local = opt.this_manifest_only) | ||
807 | |||
808 | if (expected != actual and | ||
809 | not project.config.GetBoolean('repo.preservePreciousObjects')): | ||
810 | # If this is unexpected, log it and repair. | ||
811 | Trace(f'{relpath} expected preciousObjects={expected}, got {actual}') | ||
812 | if expected: | ||
813 | if not opt.quiet: | ||
814 | print('\r%s: Shared project %s found, disabling pruning.' % | ||
815 | (relpath, project.name)) | ||
816 | if git_require((2, 7, 0)): | ||
817 | project.EnableRepositoryExtension('preciousObjects') | ||
818 | else: | ||
819 | # This isn't perfect, but it's the best we can do with old git. | ||
820 | print('\r%s: WARNING: shared projects are unreliable when using ' | ||
821 | 'old versions of git; please upgrade to git-2.7.0+.' | ||
822 | % (relpath,), | ||
823 | file=sys.stderr) | ||
824 | project.config.SetString('gc.pruneExpire', 'never') | ||
825 | else: | ||
826 | if not opt.quiet: | ||
827 | print(f'\r{relpath}: not shared, disabling pruning.') | ||
828 | project.config.SetString('extensions.preciousObjects', None) | ||
829 | project.config.SetString('gc.pruneExpire', None) | ||
830 | 760 | ||
831 | def _GCProjects(self, projects, opt, err_event): | 761 | def _GCProjects(self, projects, opt, err_event): |
832 | pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet) | 762 | pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet) |
@@ -834,8 +764,27 @@ later is required to fix a server side protocol bug. | |||
834 | 764 | ||
835 | tidy_dirs = {} | 765 | tidy_dirs = {} |
836 | for project in projects: | 766 | for project in projects: |
837 | self._RepairPreciousObjectsState(project, opt) | 767 | # Make sure pruning never kicks in with shared projects that do not use |
838 | 768 | # alternates to avoid corruption. | |
769 | if (not project.use_git_worktrees and | ||
770 | len(project.manifest.GetProjectsWithName(project.name, all_manifests=True)) > 1): | ||
771 | if project.UseAlternates: | ||
772 | # Undo logic set by previous versions of repo. | ||
773 | project.config.SetString('extensions.preciousObjects', None) | ||
774 | project.config.SetString('gc.pruneExpire', None) | ||
775 | else: | ||
776 | if not opt.quiet: | ||
777 | print('\r%s: Shared project %s found, disabling pruning.' % | ||
778 | (project.relpath, project.name)) | ||
779 | if git_require((2, 7, 0)): | ||
780 | project.EnableRepositoryExtension('preciousObjects') | ||
781 | else: | ||
782 | # This isn't perfect, but it's the best we can do with old git. | ||
783 | print('\r%s: WARNING: shared projects are unreliable when using old ' | ||
784 | 'versions of git; please upgrade to git-2.7.0+.' | ||
785 | % (project.relpath,), | ||
786 | file=sys.stderr) | ||
787 | project.config.SetString('gc.pruneExpire', 'never') | ||
839 | project.config.SetString('gc.autoDetach', 'false') | 788 | project.config.SetString('gc.autoDetach', 'false') |
840 | # Only call git gc once per objdir, but call pack-refs for the remainder. | 789 | # Only call git gc once per objdir, but call pack-refs for the remainder. |
841 | if project.objdir not in tidy_dirs: | 790 | if project.objdir not in tidy_dirs: |
diff --git a/tests/test_git_config.py b/tests/test_git_config.py index 0df38430..a4fad9ef 100644 --- a/tests/test_git_config.py +++ b/tests/test_git_config.py | |||
@@ -19,7 +19,6 @@ import tempfile | |||
19 | import unittest | 19 | import unittest |
20 | 20 | ||
21 | import git_config | 21 | import git_config |
22 | import repo_trace | ||
23 | 22 | ||
24 | 23 | ||
25 | def fixture(*paths): | 24 | def fixture(*paths): |
@@ -34,16 +33,9 @@ class GitConfigReadOnlyTests(unittest.TestCase): | |||
34 | def setUp(self): | 33 | def setUp(self): |
35 | """Create a GitConfig object using the test.gitconfig fixture. | 34 | """Create a GitConfig object using the test.gitconfig fixture. |
36 | """ | 35 | """ |
37 | |||
38 | self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests') | ||
39 | repo_trace._TRACE_FILE = os.path.join(self.tempdirobj.name, 'TRACE_FILE_from_test') | ||
40 | |||
41 | config_fixture = fixture('test.gitconfig') | 36 | config_fixture = fixture('test.gitconfig') |
42 | self.config = git_config.GitConfig(config_fixture) | 37 | self.config = git_config.GitConfig(config_fixture) |
43 | 38 | ||
44 | def tearDown(self): | ||
45 | self.tempdirobj.cleanup() | ||
46 | |||
47 | def test_GetString_with_empty_config_values(self): | 39 | def test_GetString_with_empty_config_values(self): |
48 | """ | 40 | """ |
49 | Test config entries with no value. | 41 | Test config entries with no value. |
@@ -117,15 +109,9 @@ class GitConfigReadWriteTests(unittest.TestCase): | |||
117 | """Read/write tests of the GitConfig class.""" | 109 | """Read/write tests of the GitConfig class.""" |
118 | 110 | ||
119 | def setUp(self): | 111 | def setUp(self): |
120 | self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests') | ||
121 | repo_trace._TRACE_FILE = os.path.join(self.tempdirobj.name, 'TRACE_FILE_from_test') | ||
122 | |||
123 | self.tmpfile = tempfile.NamedTemporaryFile() | 112 | self.tmpfile = tempfile.NamedTemporaryFile() |
124 | self.config = self.get_config() | 113 | self.config = self.get_config() |
125 | 114 | ||
126 | def tearDown(self): | ||
127 | self.tempdirobj.cleanup() | ||
128 | |||
129 | def get_config(self): | 115 | def get_config(self): |
130 | """Get a new GitConfig instance.""" | 116 | """Get a new GitConfig instance.""" |
131 | return git_config.GitConfig(self.tmpfile.name) | 117 | return git_config.GitConfig(self.tmpfile.name) |
diff --git a/tests/test_git_superproject.py b/tests/test_git_superproject.py index 0bb77185..0ad9b01d 100644 --- a/tests/test_git_superproject.py +++ b/tests/test_git_superproject.py | |||
@@ -24,7 +24,6 @@ from unittest import mock | |||
24 | import git_superproject | 24 | import git_superproject |
25 | import git_trace2_event_log | 25 | import git_trace2_event_log |
26 | import manifest_xml | 26 | import manifest_xml |
27 | import repo_trace | ||
28 | from test_manifest_xml import sort_attributes | 27 | from test_manifest_xml import sort_attributes |
29 | 28 | ||
30 | 29 | ||
@@ -40,7 +39,6 @@ class SuperprojectTestCase(unittest.TestCase): | |||
40 | """Set up superproject every time.""" | 39 | """Set up superproject every time.""" |
41 | self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests') | 40 | self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests') |
42 | self.tempdir = self.tempdirobj.name | 41 | self.tempdir = self.tempdirobj.name |
43 | repo_trace._TRACE_FILE = os.path.join(self.tempdir, 'TRACE_FILE_from_test') | ||
44 | self.repodir = os.path.join(self.tempdir, '.repo') | 42 | self.repodir = os.path.join(self.tempdir, '.repo') |
45 | self.manifest_file = os.path.join( | 43 | self.manifest_file = os.path.join( |
46 | self.repodir, manifest_xml.MANIFEST_FILE_NAME) | 44 | self.repodir, manifest_xml.MANIFEST_FILE_NAME) |
diff --git a/tests/test_manifest_xml.py b/tests/test_manifest_xml.py index f92108e1..e181b642 100644 --- a/tests/test_manifest_xml.py +++ b/tests/test_manifest_xml.py | |||
@@ -23,7 +23,6 @@ import xml.dom.minidom | |||
23 | 23 | ||
24 | import error | 24 | import error |
25 | import manifest_xml | 25 | import manifest_xml |
26 | import repo_trace | ||
27 | 26 | ||
28 | 27 | ||
29 | # Invalid paths that we don't want in the filesystem. | 28 | # Invalid paths that we don't want in the filesystem. |
@@ -94,7 +93,6 @@ class ManifestParseTestCase(unittest.TestCase): | |||
94 | def setUp(self): | 93 | def setUp(self): |
95 | self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests') | 94 | self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests') |
96 | self.tempdir = self.tempdirobj.name | 95 | self.tempdir = self.tempdirobj.name |
97 | repo_trace._TRACE_FILE = os.path.join(self.tempdir, 'TRACE_FILE_from_test') | ||
98 | self.repodir = os.path.join(self.tempdir, '.repo') | 96 | self.repodir = os.path.join(self.tempdir, '.repo') |
99 | self.manifest_dir = os.path.join(self.repodir, 'manifests') | 97 | self.manifest_dir = os.path.join(self.repodir, 'manifests') |
100 | self.manifest_file = os.path.join( | 98 | self.manifest_file = os.path.join( |
@@ -264,10 +262,10 @@ class XmlManifestTests(ManifestParseTestCase): | |||
264 | '<project name="r" groups="keep"/>' | 262 | '<project name="r" groups="keep"/>' |
265 | '</manifest>') | 263 | '</manifest>') |
266 | self.assertEqual( | 264 | self.assertEqual( |
267 | sort_attributes(manifest.ToXml(omit_local=True).toxml()), | 265 | manifest.ToXml(omit_local=True).toxml(), |
268 | '<?xml version="1.0" ?><manifest>' | 266 | '<?xml version="1.0" ?><manifest>' |
269 | '<remote fetch=".." name="a"/><default remote="a" revision="r"/>' | 267 | '<remote name="a" fetch=".."/><default remote="a" revision="r"/>' |
270 | '<project name="q"/><project groups="keep" name="r"/></manifest>') | 268 | '<project name="q"/><project name="r" groups="keep"/></manifest>') |
271 | 269 | ||
272 | def test_toxml_with_local(self): | 270 | def test_toxml_with_local(self): |
273 | """Does include local_manifests projects when omit_local=False.""" | 271 | """Does include local_manifests projects when omit_local=False.""" |
@@ -279,11 +277,11 @@ class XmlManifestTests(ManifestParseTestCase): | |||
279 | '<project name="r" groups="keep"/>' | 277 | '<project name="r" groups="keep"/>' |
280 | '</manifest>') | 278 | '</manifest>') |
281 | self.assertEqual( | 279 | self.assertEqual( |
282 | sort_attributes(manifest.ToXml(omit_local=False).toxml()), | 280 | manifest.ToXml(omit_local=False).toxml(), |
283 | '<?xml version="1.0" ?><manifest>' | 281 | '<?xml version="1.0" ?><manifest>' |
284 | '<remote fetch=".." name="a"/><default remote="a" revision="r"/>' | 282 | '<remote name="a" fetch=".."/><default remote="a" revision="r"/>' |
285 | '<project groups="local::me" name="p"/>' | 283 | '<project name="p" groups="local::me"/>' |
286 | '<project name="q"/><project groups="keep" name="r"/></manifest>') | 284 | '<project name="q"/><project name="r" groups="keep"/></manifest>') |
287 | 285 | ||
288 | def test_repo_hooks(self): | 286 | def test_repo_hooks(self): |
289 | """Check repo-hooks settings.""" | 287 | """Check repo-hooks settings.""" |
diff --git a/tests/test_project.py b/tests/test_project.py index 5c600be7..acd44ccc 100644 --- a/tests/test_project.py +++ b/tests/test_project.py | |||
@@ -26,7 +26,6 @@ import git_command | |||
26 | import git_config | 26 | import git_config |
27 | import platform_utils | 27 | import platform_utils |
28 | import project | 28 | import project |
29 | import repo_trace | ||
30 | 29 | ||
31 | 30 | ||
32 | @contextlib.contextmanager | 31 | @contextlib.contextmanager |
@@ -65,13 +64,6 @@ class FakeProject(object): | |||
65 | class ReviewableBranchTests(unittest.TestCase): | 64 | class ReviewableBranchTests(unittest.TestCase): |
66 | """Check ReviewableBranch behavior.""" | 65 | """Check ReviewableBranch behavior.""" |
67 | 66 | ||
68 | def setUp(self): | ||
69 | self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests') | ||
70 | repo_trace._TRACE_FILE = os.path.join(self.tempdirobj.name, 'TRACE_FILE_from_test') | ||
71 | |||
72 | def tearDown(self): | ||
73 | self.tempdirobj.cleanup() | ||
74 | |||
75 | def test_smoke(self): | 67 | def test_smoke(self): |
76 | """A quick run through everything.""" | 68 | """A quick run through everything.""" |
77 | with TempGitTree() as tempdir: | 69 | with TempGitTree() as tempdir: |
diff --git a/tests/test_subcmds_sync.py b/tests/test_subcmds_sync.py index 13f3f873..aad713f2 100644 --- a/tests/test_subcmds_sync.py +++ b/tests/test_subcmds_sync.py | |||
@@ -11,9 +11,9 @@ | |||
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | # See the License for the specific language governing permissions and | 12 | # See the License for the specific language governing permissions and |
13 | # limitations under the License. | 13 | # limitations under the License. |
14 | |||
14 | """Unittests for the subcmds/sync.py module.""" | 15 | """Unittests for the subcmds/sync.py module.""" |
15 | 16 | ||
16 | import unittest | ||
17 | from unittest import mock | 17 | from unittest import mock |
18 | 18 | ||
19 | import pytest | 19 | import pytest |
@@ -21,14 +21,17 @@ import pytest | |||
21 | from subcmds import sync | 21 | from subcmds import sync |
22 | 22 | ||
23 | 23 | ||
24 | @pytest.mark.parametrize('use_superproject, cli_args, result', [ | 24 | @pytest.mark.parametrize( |
25 | 'use_superproject, cli_args, result', | ||
26 | [ | ||
25 | (True, ['--current-branch'], True), | 27 | (True, ['--current-branch'], True), |
26 | (True, ['--no-current-branch'], True), | 28 | (True, ['--no-current-branch'], True), |
27 | (True, [], True), | 29 | (True, [], True), |
28 | (False, ['--current-branch'], True), | 30 | (False, ['--current-branch'], True), |
29 | (False, ['--no-current-branch'], False), | 31 | (False, ['--no-current-branch'], False), |
30 | (False, [], None), | 32 | (False, [], None), |
31 | ]) | 33 | ] |
34 | ) | ||
32 | def test_get_current_branch_only(use_superproject, cli_args, result): | 35 | def test_get_current_branch_only(use_superproject, cli_args, result): |
33 | """Test Sync._GetCurrentBranchOnly logic. | 36 | """Test Sync._GetCurrentBranchOnly logic. |
34 | 37 | ||
@@ -38,49 +41,5 @@ def test_get_current_branch_only(use_superproject, cli_args, result): | |||
38 | cmd = sync.Sync() | 41 | cmd = sync.Sync() |
39 | opts, _ = cmd.OptionParser.parse_args(cli_args) | 42 | opts, _ = cmd.OptionParser.parse_args(cli_args) |
40 | 43 | ||
41 | with mock.patch('git_superproject.UseSuperproject', | 44 | with mock.patch('git_superproject.UseSuperproject', return_value=use_superproject): |
42 | return_value=use_superproject): | ||
43 | assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result | 45 | assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result |
44 | |||
45 | |||
46 | class GetPreciousObjectsState(unittest.TestCase): | ||
47 | """Tests for _GetPreciousObjectsState.""" | ||
48 | |||
49 | def setUp(self): | ||
50 | """Common setup.""" | ||
51 | self.cmd = sync.Sync() | ||
52 | self.project = p = mock.MagicMock(use_git_worktrees=False, | ||
53 | UseAlternates=False) | ||
54 | p.manifest.GetProjectsWithName.return_value = [p] | ||
55 | |||
56 | self.opt = mock.Mock(spec_set=['this_manifest_only']) | ||
57 | self.opt.this_manifest_only = False | ||
58 | |||
59 | def test_worktrees(self): | ||
60 | """False for worktrees.""" | ||
61 | self.project.use_git_worktrees = True | ||
62 | self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt)) | ||
63 | |||
64 | def test_not_shared(self): | ||
65 | """Singleton project.""" | ||
66 | self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt)) | ||
67 | |||
68 | def test_shared(self): | ||
69 | """Shared project.""" | ||
70 | self.project.manifest.GetProjectsWithName.return_value = [ | ||
71 | self.project, self.project | ||
72 | ] | ||
73 | self.assertTrue(self.cmd._GetPreciousObjectsState(self.project, self.opt)) | ||
74 | |||
75 | def test_shared_with_alternates(self): | ||
76 | """Shared project, with alternates.""" | ||
77 | self.project.manifest.GetProjectsWithName.return_value = [ | ||
78 | self.project, self.project | ||
79 | ] | ||
80 | self.project.UseAlternates = True | ||
81 | self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt)) | ||
82 | |||
83 | def test_not_found(self): | ||
84 | """Project not found in manifest.""" | ||
85 | self.project.manifest.GetProjectsWithName.return_value = [] | ||
86 | self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt)) | ||