summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--git_command.py47
-rw-r--r--git_config.py10
-rw-r--r--git_refs.py57
-rwxr-xr-xmain.py33
-rw-r--r--project.py24
-rw-r--r--repo_trace.py110
-rwxr-xr-xrun_tests1
-rw-r--r--ssh.py39
-rw-r--r--subcmds/gitc_init.py3
-rw-r--r--subcmds/sync.py109
-rw-r--r--tests/test_git_config.py14
-rw-r--r--tests/test_git_superproject.py2
-rw-r--r--tests/test_manifest_xml.py16
-rw-r--r--tests/test_project.py8
-rw-r--r--tests/test_subcmds_sync.py55
15 files changed, 149 insertions, 379 deletions
diff --git a/git_command.py b/git_command.py
index 56e18e02..19100fa9 100644
--- a/git_command.py
+++ b/git_command.py
@@ -230,11 +230,12 @@ class GitCommand(object):
230 stderr = (subprocess.STDOUT if merge_output else 230 stderr = (subprocess.STDOUT if merge_output else
231 (subprocess.PIPE if capture_stderr else None)) 231 (subprocess.PIPE if capture_stderr else None))
232 232
233 dbg = ''
234 if IsTrace(): 233 if IsTrace():
235 global LAST_CWD 234 global LAST_CWD
236 global LAST_GITDIR 235 global LAST_GITDIR
237 236
237 dbg = ''
238
238 if cwd and LAST_CWD != cwd: 239 if cwd and LAST_CWD != cwd:
239 if LAST_GITDIR or LAST_CWD: 240 if LAST_GITDIR or LAST_CWD:
240 dbg += '\n' 241 dbg += '\n'
@@ -262,31 +263,31 @@ class GitCommand(object):
262 dbg += ' 2>|' 263 dbg += ' 2>|'
263 elif stderr == subprocess.STDOUT: 264 elif stderr == subprocess.STDOUT:
264 dbg += ' 2>&1' 265 dbg += ' 2>&1'
266 Trace('%s', dbg)
267
268 try:
269 p = subprocess.Popen(command,
270 cwd=cwd,
271 env=env,
272 encoding='utf-8',
273 errors='backslashreplace',
274 stdin=stdin,
275 stdout=stdout,
276 stderr=stderr)
277 except Exception as e:
278 raise GitError('%s: %s' % (command[1], e))
265 279
266 with Trace('git command %s %s with debug: %s', LAST_GITDIR, command, dbg): 280 if ssh_proxy:
267 try: 281 ssh_proxy.add_client(p)
268 p = subprocess.Popen(command,
269 cwd=cwd,
270 env=env,
271 encoding='utf-8',
272 errors='backslashreplace',
273 stdin=stdin,
274 stdout=stdout,
275 stderr=stderr)
276 except Exception as e:
277 raise GitError('%s: %s' % (command[1], e))
278
279 if ssh_proxy:
280 ssh_proxy.add_client(p)
281 282
282 self.process = p 283 self.process = p
283 284
284 try: 285 try:
285 self.stdout, self.stderr = p.communicate(input=input) 286 self.stdout, self.stderr = p.communicate(input=input)
286 finally: 287 finally:
287 if ssh_proxy: 288 if ssh_proxy:
288 ssh_proxy.remove_client(p) 289 ssh_proxy.remove_client(p)
289 self.rc = p.wait() 290 self.rc = p.wait()
290 291
291 @staticmethod 292 @staticmethod
292 def _GetBasicEnv(): 293 def _GetBasicEnv():
diff --git a/git_config.py b/git_config.py
index 94378e9a..6f80ae08 100644
--- a/git_config.py
+++ b/git_config.py
@@ -219,8 +219,8 @@ class GitConfig(object):
219 """Set the value(s) for a key. 219 """Set the value(s) for a key.
220 Only this configuration file is modified. 220 Only this configuration file is modified.
221 221
222 The supplied value should be either a string, or a list of strings (to 222 The supplied value should be either a string,
223 store multiple values), or None (to delete the key). 223 or a list of strings (to store multiple values).
224 """ 224 """
225 key = _key(name) 225 key = _key(name)
226 226
@@ -349,9 +349,9 @@ class GitConfig(object):
349 except OSError: 349 except OSError:
350 return None 350 return None
351 try: 351 try:
352 with Trace(': parsing %s', self.file): 352 Trace(': parsing %s', self.file)
353 with open(self._json) as fd: 353 with open(self._json) as fd:
354 return json.load(fd) 354 return json.load(fd)
355 except (IOError, ValueError): 355 except (IOError, ValueError):
356 platform_utils.remove(self._json, missing_ok=True) 356 platform_utils.remove(self._json, missing_ok=True)
357 return None 357 return None
diff --git a/git_refs.py b/git_refs.py
index 300d2b30..2d4a8090 100644
--- a/git_refs.py
+++ b/git_refs.py
@@ -67,37 +67,38 @@ class GitRefs(object):
67 self._LoadAll() 67 self._LoadAll()
68 68
69 def _NeedUpdate(self): 69 def _NeedUpdate(self):
70 with Trace(': scan refs %s', self._gitdir): 70 Trace(': scan refs %s', self._gitdir)
71 for name, mtime in self._mtime.items(): 71
72 try: 72 for name, mtime in self._mtime.items():
73 if mtime != os.path.getmtime(os.path.join(self._gitdir, name)): 73 try:
74 return True 74 if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
75 except OSError:
76 return True 75 return True
77 return False 76 except OSError:
77 return True
78 return False
78 79
79 def _LoadAll(self): 80 def _LoadAll(self):
80 with Trace(': load refs %s', self._gitdir): 81 Trace(': load refs %s', self._gitdir)
81 82
82 self._phyref = {} 83 self._phyref = {}
83 self._symref = {} 84 self._symref = {}
84 self._mtime = {} 85 self._mtime = {}
85 86
86 self._ReadPackedRefs() 87 self._ReadPackedRefs()
87 self._ReadLoose('refs/') 88 self._ReadLoose('refs/')
88 self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD) 89 self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
89 90
90 scan = self._symref 91 scan = self._symref
91 attempts = 0 92 attempts = 0
92 while scan and attempts < 5: 93 while scan and attempts < 5:
93 scan_next = {} 94 scan_next = {}
94 for name, dest in scan.items(): 95 for name, dest in scan.items():
95 if dest in self._phyref: 96 if dest in self._phyref:
96 self._phyref[name] = self._phyref[dest] 97 self._phyref[name] = self._phyref[dest]
97 else: 98 else:
98 scan_next[name] = dest 99 scan_next[name] = dest
99 scan = scan_next 100 scan = scan_next
100 attempts += 1 101 attempts += 1
101 102
102 def _ReadPackedRefs(self): 103 def _ReadPackedRefs(self):
103 path = os.path.join(self._gitdir, 'packed-refs') 104 path = os.path.join(self._gitdir, 'packed-refs')
diff --git a/main.py b/main.py
index e629b30f..c54f9281 100755
--- a/main.py
+++ b/main.py
@@ -37,7 +37,7 @@ except ImportError:
37 37
38from color import SetDefaultColoring 38from color import SetDefaultColoring
39import event_log 39import event_log
40from repo_trace import SetTrace, Trace, SetTraceToStderr 40from repo_trace import SetTrace
41from git_command import user_agent 41from git_command import user_agent
42from git_config import RepoConfig 42from git_config import RepoConfig
43from git_trace2_event_log import EventLog 43from git_trace2_event_log import EventLog
@@ -109,9 +109,6 @@ global_options.add_option('--color',
109global_options.add_option('--trace', 109global_options.add_option('--trace',
110 dest='trace', action='store_true', 110 dest='trace', action='store_true',
111 help='trace git command execution (REPO_TRACE=1)') 111 help='trace git command execution (REPO_TRACE=1)')
112global_options.add_option('--trace_to_stderr',
113 dest='trace_to_stderr', action='store_true',
114 help='trace outputs go to stderr in addition to .repo/TRACE_FILE')
115global_options.add_option('--trace-python', 112global_options.add_option('--trace-python',
116 dest='trace_python', action='store_true', 113 dest='trace_python', action='store_true',
117 help='trace python command execution') 114 help='trace python command execution')
@@ -201,6 +198,9 @@ class _Repo(object):
201 """Execute the requested subcommand.""" 198 """Execute the requested subcommand."""
202 result = 0 199 result = 0
203 200
201 if gopts.trace:
202 SetTrace()
203
204 # Handle options that terminate quickly first. 204 # Handle options that terminate quickly first.
205 if gopts.help or gopts.help_all: 205 if gopts.help or gopts.help_all:
206 self._PrintHelp(short=False, all_commands=gopts.help_all) 206 self._PrintHelp(short=False, all_commands=gopts.help_all)
@@ -652,26 +652,17 @@ def _Main(argv):
652 Version.wrapper_path = opt.wrapper_path 652 Version.wrapper_path = opt.wrapper_path
653 653
654 repo = _Repo(opt.repodir) 654 repo = _Repo(opt.repodir)
655
656 try: 655 try:
657 init_http() 656 init_http()
658 name, gopts, argv = repo._ParseArgs(argv) 657 name, gopts, argv = repo._ParseArgs(argv)
659 658 run = lambda: repo._Run(name, gopts, argv) or 0
660 if gopts.trace: 659 if gopts.trace_python:
661 SetTrace() 660 import trace
662 661 tracer = trace.Trace(count=False, trace=True, timing=True,
663 if gopts.trace_to_stderr: 662 ignoredirs=set(sys.path[1:]))
664 SetTraceToStderr() 663 result = tracer.runfunc(run)
665 664 else:
666 with Trace('starting new command: %s', ', '.join([name] + argv), first_trace=True): 665 result = run()
667 run = lambda: repo._Run(name, gopts, argv) or 0
668 if gopts.trace_python:
669 import trace
670 tracer = trace.Trace(count=False, trace=True, timing=True,
671 ignoredirs=set(sys.path[1:]))
672 result = tracer.runfunc(run)
673 else:
674 result = run()
675 except KeyboardInterrupt: 666 except KeyboardInterrupt:
676 print('aborted by user', file=sys.stderr) 667 print('aborted by user', file=sys.stderr)
677 result = 1 668 result = 1
diff --git a/project.py b/project.py
index 9d7476b4..1c85b044 100644
--- a/project.py
+++ b/project.py
@@ -41,7 +41,7 @@ from error import ManifestInvalidRevisionError, ManifestInvalidPathError
41from error import NoManifestException, ManifestParseError 41from error import NoManifestException, ManifestParseError
42import platform_utils 42import platform_utils
43import progress 43import progress
44from repo_trace import Trace 44from repo_trace import IsTrace, Trace
45 45
46from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M, R_WORKTREE_M 46from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M, R_WORKTREE_M
47 47
@@ -59,7 +59,7 @@ MAXIMUM_RETRY_SLEEP_SEC = 3600.0
59# +-10% random jitter is added to each Fetches retry sleep duration. 59# +-10% random jitter is added to each Fetches retry sleep duration.
60RETRY_JITTER_PERCENT = 0.1 60RETRY_JITTER_PERCENT = 0.1
61 61
62# Whether to use alternates. Switching back and forth is *NOT* supported. 62# Whether to use alternates.
63# TODO(vapier): Remove knob once behavior is verified. 63# TODO(vapier): Remove knob once behavior is verified.
64_ALTERNATES = os.environ.get('REPO_USE_ALTERNATES') == '1' 64_ALTERNATES = os.environ.get('REPO_USE_ALTERNATES') == '1'
65 65
@@ -2416,16 +2416,16 @@ class Project(object):
2416 srcUrl = 'http' + srcUrl[len('persistent-http'):] 2416 srcUrl = 'http' + srcUrl[len('persistent-http'):]
2417 cmd += [srcUrl] 2417 cmd += [srcUrl]
2418 2418
2419 proc = None 2419 if IsTrace():
2420 with Trace('Fetching bundle: %s', ' '.join(cmd)): 2420 Trace('%s', ' '.join(cmd))
2421 if verbose: 2421 if verbose:
2422 print('%s: Downloading bundle: %s' % (self.name, srcUrl)) 2422 print('%s: Downloading bundle: %s' % (self.name, srcUrl))
2423 stdout = None if verbose else subprocess.PIPE 2423 stdout = None if verbose else subprocess.PIPE
2424 stderr = None if verbose else subprocess.STDOUT 2424 stderr = None if verbose else subprocess.STDOUT
2425 try: 2425 try:
2426 proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr) 2426 proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)
2427 except OSError: 2427 except OSError:
2428 return False 2428 return False
2429 2429
2430 (output, _) = proc.communicate() 2430 (output, _) = proc.communicate()
2431 curlret = proc.returncode 2431 curlret = proc.returncode
diff --git a/repo_trace.py b/repo_trace.py
index 0ff3b694..7be0c045 100644
--- a/repo_trace.py
+++ b/repo_trace.py
@@ -15,128 +15,26 @@
15"""Logic for tracing repo interactions. 15"""Logic for tracing repo interactions.
16 16
17Activated via `repo --trace ...` or `REPO_TRACE=1 repo ...`. 17Activated via `repo --trace ...` or `REPO_TRACE=1 repo ...`.
18
19Temporary: Tracing is always on. Set `REPO_TRACE=0` to turn off.
20To also include trace outputs in stderr do `repo --trace_to_stderr ...`
21""" 18"""
22 19
23import sys 20import sys
24import os 21import os
25import tempfile
26import time
27from contextlib import ContextDecorator
28 22
29# Env var to implicitly turn on tracing. 23# Env var to implicitly turn on tracing.
30REPO_TRACE = 'REPO_TRACE' 24REPO_TRACE = 'REPO_TRACE'
31 25
32# Temporarily set tracing to always on unless user expicitly sets to 0. 26_TRACE = os.environ.get(REPO_TRACE) == '1'
33_TRACE = os.environ.get(REPO_TRACE) != '0'
34
35_TRACE_TO_STDERR = False
36
37_TRACE_FILE = None
38
39_TRACE_FILE_NAME = 'TRACE_FILE'
40
41_MAX_SIZE = 5 # in mb
42
43_NEW_COMMAND_SEP = '+++++++++++++++NEW COMMAND+++++++++++++++++++'
44
45
46def IsStraceToStderr():
47 return _TRACE_TO_STDERR
48 27
49 28
50def IsTrace(): 29def IsTrace():
51 return _TRACE 30 return _TRACE
52 31
53 32
54def SetTraceToStderr():
55 global _TRACE_TO_STDERR
56 _TRACE_TO_STDERR = True
57
58
59def SetTrace(): 33def SetTrace():
60 global _TRACE 34 global _TRACE
61 _TRACE = True 35 _TRACE = True
62 36
63 37
64def _SetTraceFile(): 38def Trace(fmt, *args):
65 global _TRACE_FILE 39 if IsTrace():
66 _TRACE_FILE = _GetTraceFile() 40 print(fmt % args, file=sys.stderr)
67
68
69class Trace(ContextDecorator):
70
71 def _time(self):
72 """Generate nanoseconds of time in a py3.6 safe way"""
73 return int(time.time()*1e+9)
74
75 def __init__(self, fmt, *args, first_trace=False):
76 if not IsTrace():
77 return
78 self._trace_msg = fmt % args
79
80 if not _TRACE_FILE:
81 _SetTraceFile()
82
83 if first_trace:
84 _ClearOldTraces()
85 self._trace_msg = '%s %s' % (_NEW_COMMAND_SEP, self._trace_msg)
86
87
88 def __enter__(self):
89 if not IsTrace():
90 return self
91
92 print_msg = f"PID: {os.getpid()} START: {self._time()} :" + self._trace_msg + '\n'
93
94 with open(_TRACE_FILE, 'a') as f:
95 print(print_msg, file=f)
96
97 if _TRACE_TO_STDERR:
98 print(print_msg, file=sys.stderr)
99
100 return self
101
102 def __exit__(self, *exc):
103 if not IsTrace():
104 return False
105
106 print_msg = f"PID: {os.getpid()} END: {self._time()} :" + self._trace_msg + '\n'
107
108 with open(_TRACE_FILE, 'a') as f:
109 print(print_msg, file=f)
110
111 if _TRACE_TO_STDERR:
112 print(print_msg, file=sys.stderr)
113
114 return False
115
116
117def _GetTraceFile():
118 """Get the trace file or create one."""
119 # TODO: refactor to pass repodir to Trace.
120 repo_dir = os.path.dirname(os.path.dirname(__file__))
121 trace_file = os.path.join(repo_dir, _TRACE_FILE_NAME)
122 print('Trace outputs in %s' % trace_file)
123 return trace_file
124
125def _ClearOldTraces():
126 """Clear traces from old commands if trace file is too big.
127
128 Note: If the trace file contains output from two `repo`
129 commands that were running at the same time, this
130 will not work precisely.
131 """
132 if os.path.isfile(_TRACE_FILE):
133 while os.path.getsize(_TRACE_FILE)/(1024*1024) > _MAX_SIZE:
134 temp = tempfile.NamedTemporaryFile(mode='w', delete=False)
135 with open(_TRACE_FILE, 'r', errors='ignore') as fin:
136 trace_lines = fin.readlines()
137 for i , l in enumerate(trace_lines):
138 if 'END:' in l and _NEW_COMMAND_SEP in l:
139 temp.writelines(trace_lines[i+1:])
140 break
141 temp.close()
142 os.replace(temp.name, _TRACE_FILE)
diff --git a/run_tests b/run_tests
index 7c9ff41d..573dd446 100755
--- a/run_tests
+++ b/run_tests
@@ -20,7 +20,6 @@ import os
20import shutil 20import shutil
21import subprocess 21import subprocess
22import sys 22import sys
23import repo_trace
24 23
25 24
26def find_pytest(): 25def find_pytest():
diff --git a/ssh.py b/ssh.py
index 004fdbad..450383dc 100644
--- a/ssh.py
+++ b/ssh.py
@@ -182,29 +182,28 @@ class ProxyManager:
182 # be important because we can't tell that that 'git@myhost.com' is the same 182 # be important because we can't tell that that 'git@myhost.com' is the same
183 # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file. 183 # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
184 check_command = command_base + ['-O', 'check'] 184 check_command = command_base + ['-O', 'check']
185 with Trace('Call to ssh (check call): %s', ' '.join(check_command)): 185 try:
186 try: 186 Trace(': %s', ' '.join(check_command))
187 check_process = subprocess.Popen(check_command, 187 check_process = subprocess.Popen(check_command,
188 stdout=subprocess.PIPE, 188 stdout=subprocess.PIPE,
189 stderr=subprocess.PIPE) 189 stderr=subprocess.PIPE)
190 check_process.communicate() # read output, but ignore it... 190 check_process.communicate() # read output, but ignore it...
191 isnt_running = check_process.wait() 191 isnt_running = check_process.wait()
192 192
193 if not isnt_running: 193 if not isnt_running:
194 # Our double-check found that the master _was_ infact running. Add to 194 # Our double-check found that the master _was_ infact running. Add to
195 # the list of keys. 195 # the list of keys.
196 self._master_keys[key] = True 196 self._master_keys[key] = True
197 return True 197 return True
198 except Exception: 198 except Exception:
199 # Ignore excpetions. We we will fall back to the normal command and 199 # Ignore excpetions. We we will fall back to the normal command and print
200 # print to the log there. 200 # to the log there.
201 pass 201 pass
202 202
203 command = command_base[:1] + ['-M', '-N'] + command_base[1:] 203 command = command_base[:1] + ['-M', '-N'] + command_base[1:]
204 p = None
205 try: 204 try:
206 with Trace('Call to ssh: %s', ' '.join(command)): 205 Trace(': %s', ' '.join(command))
207 p = subprocess.Popen(command) 206 p = subprocess.Popen(command)
208 except Exception as e: 207 except Exception as e:
209 self._master_broken.value = True 208 self._master_broken.value = True
210 print('\nwarn: cannot enable ssh control master for %s:%s\n%s' 209 print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py
index e3a5813d..1d81baf5 100644
--- a/subcmds/gitc_init.py
+++ b/subcmds/gitc_init.py
@@ -68,8 +68,7 @@ use for this GITC client.
68 sys.exit(1) 68 sys.exit(1)
69 manifest_file = opt.manifest_file 69 manifest_file = opt.manifest_file
70 70
71 manifest = GitcManifest(self.repodir, os.path.join(self.client_dir, 71 manifest = GitcManifest(self.repodir, gitc_client)
72 '.manifest'))
73 manifest.Override(manifest_file) 72 manifest.Override(manifest_file)
74 gitc_utils.generate_gitc_manifest(None, manifest) 73 gitc_utils.generate_gitc_manifest(None, manifest)
75 print('Please run `cd %s` to view your GITC client.' % 74 print('Please run `cd %s` to view your GITC client.' %
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 83c9ad36..fe63b484 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -60,7 +60,7 @@ from error import RepoChangedException, GitError, ManifestParseError
60import platform_utils 60import platform_utils
61from project import SyncBuffer 61from project import SyncBuffer
62from progress import Progress 62from progress import Progress
63from repo_trace import Trace 63from repo_trace import IsTrace, Trace
64import ssh 64import ssh
65from wrapper import Wrapper 65from wrapper import Wrapper
66from manifest_xml import GitcManifest 66from manifest_xml import GitcManifest
@@ -739,6 +739,7 @@ later is required to fix a server side protocol bug.
739 bak_dir = os.path.join(objdir, '.repo', 'pack.bak') 739 bak_dir = os.path.join(objdir, '.repo', 'pack.bak')
740 if not _BACKUP_OBJECTS or not platform_utils.isdir(pack_dir): 740 if not _BACKUP_OBJECTS or not platform_utils.isdir(pack_dir):
741 return 741 return
742 saved = []
742 files = set(platform_utils.listdir(pack_dir)) 743 files = set(platform_utils.listdir(pack_dir))
743 to_backup = [] 744 to_backup = []
744 for f in files: 745 for f in files:
@@ -750,83 +751,12 @@ later is required to fix a server side protocol bug.
750 for fname in to_backup: 751 for fname in to_backup:
751 bak_fname = os.path.join(bak_dir, fname) 752 bak_fname = os.path.join(bak_dir, fname)
752 if not os.path.exists(bak_fname): 753 if not os.path.exists(bak_fname):
753 with Trace('%s saved %s', bare_git._project.name, fname): 754 saved.append(fname)
754 # Use a tmp file so that we are sure of a complete copy. 755 # Use a tmp file so that we are sure of a complete copy.
755 shutil.copy(os.path.join(pack_dir, fname), bak_fname + '.tmp') 756 shutil.copy(os.path.join(pack_dir, fname), bak_fname + '.tmp')
756 shutil.move(bak_fname + '.tmp', bak_fname) 757 shutil.move(bak_fname + '.tmp', bak_fname)
757 758 if saved:
758 @staticmethod 759 Trace('%s saved %s', bare_git._project.name, ' '.join(saved))
759 def _GetPreciousObjectsState(project: Project, opt):
760 """Get the preciousObjects state for the project.
761
762 Args:
763 project (Project): the project to examine, and possibly correct.
764 opt (optparse.Values): options given to sync.
765
766 Returns:
767 Expected state of extensions.preciousObjects:
768 False: Should be disabled. (not present)
769 True: Should be enabled.
770 """
771 if project.use_git_worktrees:
772 return False
773 projects = project.manifest.GetProjectsWithName(project.name,
774 all_manifests=True)
775 if len(projects) == 1:
776 return False
777 relpath = project.RelPath(local=opt.this_manifest_only)
778 if len(projects) > 1:
779 # Objects are potentially shared with another project.
780 # See the logic in Project.Sync_NetworkHalf regarding UseAlternates.
781 # - When False, shared projects share (via symlink)
782 # .repo/project-objects/{PROJECT_NAME}.git as the one-and-only objects
783 # directory. All objects are precious, since there is no project with a
784 # complete set of refs.
785 # - When True, shared projects share (via info/alternates)
786 # .repo/project-objects/{PROJECT_NAME}.git as an alternate object store,
787 # which is written only on the first clone of the project, and is not
788 # written subsequently. (When Sync_NetworkHalf sees that it exists, it
789 # makes sure that the alternates file points there, and uses a
790 # project-local .git/objects directory for all syncs going forward.
791 # We do not support switching between the options. The environment
792 # variable is present for testing and migration only.
793 return not project.UseAlternates
794 print(f'\r{relpath}: project not found in manifest.', file=sys.stderr)
795 return False
796
797 def _RepairPreciousObjectsState(self, project: Project, opt):
798 """Correct the preciousObjects state for the project.
799
800 Args:
801 project (Project): the project to examine, and possibly correct.
802 opt (optparse.Values): options given to sync.
803 """
804 expected = self._GetPreciousObjectsState(project, opt)
805 actual = project.config.GetBoolean('extensions.preciousObjects') or False
806 relpath = project.RelPath(local = opt.this_manifest_only)
807
808 if (expected != actual and
809 not project.config.GetBoolean('repo.preservePreciousObjects')):
810 # If this is unexpected, log it and repair.
811 Trace(f'{relpath} expected preciousObjects={expected}, got {actual}')
812 if expected:
813 if not opt.quiet:
814 print('\r%s: Shared project %s found, disabling pruning.' %
815 (relpath, project.name))
816 if git_require((2, 7, 0)):
817 project.EnableRepositoryExtension('preciousObjects')
818 else:
819 # This isn't perfect, but it's the best we can do with old git.
820 print('\r%s: WARNING: shared projects are unreliable when using '
821 'old versions of git; please upgrade to git-2.7.0+.'
822 % (relpath,),
823 file=sys.stderr)
824 project.config.SetString('gc.pruneExpire', 'never')
825 else:
826 if not opt.quiet:
827 print(f'\r{relpath}: not shared, disabling pruning.')
828 project.config.SetString('extensions.preciousObjects', None)
829 project.config.SetString('gc.pruneExpire', None)
830 760
831 def _GCProjects(self, projects, opt, err_event): 761 def _GCProjects(self, projects, opt, err_event):
832 pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet) 762 pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
@@ -834,8 +764,27 @@ later is required to fix a server side protocol bug.
834 764
835 tidy_dirs = {} 765 tidy_dirs = {}
836 for project in projects: 766 for project in projects:
837 self._RepairPreciousObjectsState(project, opt) 767 # Make sure pruning never kicks in with shared projects that do not use
838 768 # alternates to avoid corruption.
769 if (not project.use_git_worktrees and
770 len(project.manifest.GetProjectsWithName(project.name, all_manifests=True)) > 1):
771 if project.UseAlternates:
772 # Undo logic set by previous versions of repo.
773 project.config.SetString('extensions.preciousObjects', None)
774 project.config.SetString('gc.pruneExpire', None)
775 else:
776 if not opt.quiet:
777 print('\r%s: Shared project %s found, disabling pruning.' %
778 (project.relpath, project.name))
779 if git_require((2, 7, 0)):
780 project.EnableRepositoryExtension('preciousObjects')
781 else:
782 # This isn't perfect, but it's the best we can do with old git.
783 print('\r%s: WARNING: shared projects are unreliable when using old '
784 'versions of git; please upgrade to git-2.7.0+.'
785 % (project.relpath,),
786 file=sys.stderr)
787 project.config.SetString('gc.pruneExpire', 'never')
839 project.config.SetString('gc.autoDetach', 'false') 788 project.config.SetString('gc.autoDetach', 'false')
840 # Only call git gc once per objdir, but call pack-refs for the remainder. 789 # Only call git gc once per objdir, but call pack-refs for the remainder.
841 if project.objdir not in tidy_dirs: 790 if project.objdir not in tidy_dirs:
diff --git a/tests/test_git_config.py b/tests/test_git_config.py
index 0df38430..a4fad9ef 100644
--- a/tests/test_git_config.py
+++ b/tests/test_git_config.py
@@ -19,7 +19,6 @@ import tempfile
19import unittest 19import unittest
20 20
21import git_config 21import git_config
22import repo_trace
23 22
24 23
25def fixture(*paths): 24def fixture(*paths):
@@ -34,16 +33,9 @@ class GitConfigReadOnlyTests(unittest.TestCase):
34 def setUp(self): 33 def setUp(self):
35 """Create a GitConfig object using the test.gitconfig fixture. 34 """Create a GitConfig object using the test.gitconfig fixture.
36 """ 35 """
37
38 self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
39 repo_trace._TRACE_FILE = os.path.join(self.tempdirobj.name, 'TRACE_FILE_from_test')
40
41 config_fixture = fixture('test.gitconfig') 36 config_fixture = fixture('test.gitconfig')
42 self.config = git_config.GitConfig(config_fixture) 37 self.config = git_config.GitConfig(config_fixture)
43 38
44 def tearDown(self):
45 self.tempdirobj.cleanup()
46
47 def test_GetString_with_empty_config_values(self): 39 def test_GetString_with_empty_config_values(self):
48 """ 40 """
49 Test config entries with no value. 41 Test config entries with no value.
@@ -117,15 +109,9 @@ class GitConfigReadWriteTests(unittest.TestCase):
117 """Read/write tests of the GitConfig class.""" 109 """Read/write tests of the GitConfig class."""
118 110
119 def setUp(self): 111 def setUp(self):
120 self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
121 repo_trace._TRACE_FILE = os.path.join(self.tempdirobj.name, 'TRACE_FILE_from_test')
122
123 self.tmpfile = tempfile.NamedTemporaryFile() 112 self.tmpfile = tempfile.NamedTemporaryFile()
124 self.config = self.get_config() 113 self.config = self.get_config()
125 114
126 def tearDown(self):
127 self.tempdirobj.cleanup()
128
129 def get_config(self): 115 def get_config(self):
130 """Get a new GitConfig instance.""" 116 """Get a new GitConfig instance."""
131 return git_config.GitConfig(self.tmpfile.name) 117 return git_config.GitConfig(self.tmpfile.name)
diff --git a/tests/test_git_superproject.py b/tests/test_git_superproject.py
index 0bb77185..0ad9b01d 100644
--- a/tests/test_git_superproject.py
+++ b/tests/test_git_superproject.py
@@ -24,7 +24,6 @@ from unittest import mock
24import git_superproject 24import git_superproject
25import git_trace2_event_log 25import git_trace2_event_log
26import manifest_xml 26import manifest_xml
27import repo_trace
28from test_manifest_xml import sort_attributes 27from test_manifest_xml import sort_attributes
29 28
30 29
@@ -40,7 +39,6 @@ class SuperprojectTestCase(unittest.TestCase):
40 """Set up superproject every time.""" 39 """Set up superproject every time."""
41 self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests') 40 self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
42 self.tempdir = self.tempdirobj.name 41 self.tempdir = self.tempdirobj.name
43 repo_trace._TRACE_FILE = os.path.join(self.tempdir, 'TRACE_FILE_from_test')
44 self.repodir = os.path.join(self.tempdir, '.repo') 42 self.repodir = os.path.join(self.tempdir, '.repo')
45 self.manifest_file = os.path.join( 43 self.manifest_file = os.path.join(
46 self.repodir, manifest_xml.MANIFEST_FILE_NAME) 44 self.repodir, manifest_xml.MANIFEST_FILE_NAME)
diff --git a/tests/test_manifest_xml.py b/tests/test_manifest_xml.py
index f92108e1..e181b642 100644
--- a/tests/test_manifest_xml.py
+++ b/tests/test_manifest_xml.py
@@ -23,7 +23,6 @@ import xml.dom.minidom
23 23
24import error 24import error
25import manifest_xml 25import manifest_xml
26import repo_trace
27 26
28 27
29# Invalid paths that we don't want in the filesystem. 28# Invalid paths that we don't want in the filesystem.
@@ -94,7 +93,6 @@ class ManifestParseTestCase(unittest.TestCase):
94 def setUp(self): 93 def setUp(self):
95 self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests') 94 self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
96 self.tempdir = self.tempdirobj.name 95 self.tempdir = self.tempdirobj.name
97 repo_trace._TRACE_FILE = os.path.join(self.tempdir, 'TRACE_FILE_from_test')
98 self.repodir = os.path.join(self.tempdir, '.repo') 96 self.repodir = os.path.join(self.tempdir, '.repo')
99 self.manifest_dir = os.path.join(self.repodir, 'manifests') 97 self.manifest_dir = os.path.join(self.repodir, 'manifests')
100 self.manifest_file = os.path.join( 98 self.manifest_file = os.path.join(
@@ -264,10 +262,10 @@ class XmlManifestTests(ManifestParseTestCase):
264 '<project name="r" groups="keep"/>' 262 '<project name="r" groups="keep"/>'
265 '</manifest>') 263 '</manifest>')
266 self.assertEqual( 264 self.assertEqual(
267 sort_attributes(manifest.ToXml(omit_local=True).toxml()), 265 manifest.ToXml(omit_local=True).toxml(),
268 '<?xml version="1.0" ?><manifest>' 266 '<?xml version="1.0" ?><manifest>'
269 '<remote fetch=".." name="a"/><default remote="a" revision="r"/>' 267 '<remote name="a" fetch=".."/><default remote="a" revision="r"/>'
270 '<project name="q"/><project groups="keep" name="r"/></manifest>') 268 '<project name="q"/><project name="r" groups="keep"/></manifest>')
271 269
272 def test_toxml_with_local(self): 270 def test_toxml_with_local(self):
273 """Does include local_manifests projects when omit_local=False.""" 271 """Does include local_manifests projects when omit_local=False."""
@@ -279,11 +277,11 @@ class XmlManifestTests(ManifestParseTestCase):
279 '<project name="r" groups="keep"/>' 277 '<project name="r" groups="keep"/>'
280 '</manifest>') 278 '</manifest>')
281 self.assertEqual( 279 self.assertEqual(
282 sort_attributes(manifest.ToXml(omit_local=False).toxml()), 280 manifest.ToXml(omit_local=False).toxml(),
283 '<?xml version="1.0" ?><manifest>' 281 '<?xml version="1.0" ?><manifest>'
284 '<remote fetch=".." name="a"/><default remote="a" revision="r"/>' 282 '<remote name="a" fetch=".."/><default remote="a" revision="r"/>'
285 '<project groups="local::me" name="p"/>' 283 '<project name="p" groups="local::me"/>'
286 '<project name="q"/><project groups="keep" name="r"/></manifest>') 284 '<project name="q"/><project name="r" groups="keep"/></manifest>')
287 285
288 def test_repo_hooks(self): 286 def test_repo_hooks(self):
289 """Check repo-hooks settings.""" 287 """Check repo-hooks settings."""
diff --git a/tests/test_project.py b/tests/test_project.py
index 5c600be7..acd44ccc 100644
--- a/tests/test_project.py
+++ b/tests/test_project.py
@@ -26,7 +26,6 @@ import git_command
26import git_config 26import git_config
27import platform_utils 27import platform_utils
28import project 28import project
29import repo_trace
30 29
31 30
32@contextlib.contextmanager 31@contextlib.contextmanager
@@ -65,13 +64,6 @@ class FakeProject(object):
65class ReviewableBranchTests(unittest.TestCase): 64class ReviewableBranchTests(unittest.TestCase):
66 """Check ReviewableBranch behavior.""" 65 """Check ReviewableBranch behavior."""
67 66
68 def setUp(self):
69 self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
70 repo_trace._TRACE_FILE = os.path.join(self.tempdirobj.name, 'TRACE_FILE_from_test')
71
72 def tearDown(self):
73 self.tempdirobj.cleanup()
74
75 def test_smoke(self): 67 def test_smoke(self):
76 """A quick run through everything.""" 68 """A quick run through everything."""
77 with TempGitTree() as tempdir: 69 with TempGitTree() as tempdir:
diff --git a/tests/test_subcmds_sync.py b/tests/test_subcmds_sync.py
index 13f3f873..aad713f2 100644
--- a/tests/test_subcmds_sync.py
+++ b/tests/test_subcmds_sync.py
@@ -11,9 +11,9 @@
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14
14"""Unittests for the subcmds/sync.py module.""" 15"""Unittests for the subcmds/sync.py module."""
15 16
16import unittest
17from unittest import mock 17from unittest import mock
18 18
19import pytest 19import pytest
@@ -21,14 +21,17 @@ import pytest
21from subcmds import sync 21from subcmds import sync
22 22
23 23
24@pytest.mark.parametrize('use_superproject, cli_args, result', [ 24@pytest.mark.parametrize(
25 'use_superproject, cli_args, result',
26 [
25 (True, ['--current-branch'], True), 27 (True, ['--current-branch'], True),
26 (True, ['--no-current-branch'], True), 28 (True, ['--no-current-branch'], True),
27 (True, [], True), 29 (True, [], True),
28 (False, ['--current-branch'], True), 30 (False, ['--current-branch'], True),
29 (False, ['--no-current-branch'], False), 31 (False, ['--no-current-branch'], False),
30 (False, [], None), 32 (False, [], None),
31]) 33 ]
34)
32def test_get_current_branch_only(use_superproject, cli_args, result): 35def test_get_current_branch_only(use_superproject, cli_args, result):
33 """Test Sync._GetCurrentBranchOnly logic. 36 """Test Sync._GetCurrentBranchOnly logic.
34 37
@@ -38,49 +41,5 @@ def test_get_current_branch_only(use_superproject, cli_args, result):
38 cmd = sync.Sync() 41 cmd = sync.Sync()
39 opts, _ = cmd.OptionParser.parse_args(cli_args) 42 opts, _ = cmd.OptionParser.parse_args(cli_args)
40 43
41 with mock.patch('git_superproject.UseSuperproject', 44 with mock.patch('git_superproject.UseSuperproject', return_value=use_superproject):
42 return_value=use_superproject):
43 assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result 45 assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result
44
45
46class GetPreciousObjectsState(unittest.TestCase):
47 """Tests for _GetPreciousObjectsState."""
48
49 def setUp(self):
50 """Common setup."""
51 self.cmd = sync.Sync()
52 self.project = p = mock.MagicMock(use_git_worktrees=False,
53 UseAlternates=False)
54 p.manifest.GetProjectsWithName.return_value = [p]
55
56 self.opt = mock.Mock(spec_set=['this_manifest_only'])
57 self.opt.this_manifest_only = False
58
59 def test_worktrees(self):
60 """False for worktrees."""
61 self.project.use_git_worktrees = True
62 self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))
63
64 def test_not_shared(self):
65 """Singleton project."""
66 self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))
67
68 def test_shared(self):
69 """Shared project."""
70 self.project.manifest.GetProjectsWithName.return_value = [
71 self.project, self.project
72 ]
73 self.assertTrue(self.cmd._GetPreciousObjectsState(self.project, self.opt))
74
75 def test_shared_with_alternates(self):
76 """Shared project, with alternates."""
77 self.project.manifest.GetProjectsWithName.return_value = [
78 self.project, self.project
79 ]
80 self.project.UseAlternates = True
81 self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))
82
83 def test_not_found(self):
84 """Project not found in manifest."""
85 self.project.manifest.GetProjectsWithName.return_value = []
86 self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))