summaryrefslogtreecommitdiffstats
path: root/subcmds
diff options
context:
space:
mode:
Diffstat (limited to 'subcmds')
-rw-r--r--subcmds/__init__.py11
-rw-r--r--subcmds/abandon.py75
-rw-r--r--subcmds/branches.py49
-rw-r--r--subcmds/checkout.py39
-rw-r--r--subcmds/cherry_pick.py27
-rw-r--r--subcmds/diff.py53
-rw-r--r--subcmds/diffmanifests.py32
-rw-r--r--subcmds/download.py80
-rw-r--r--subcmds/forall.py289
-rw-r--r--subcmds/gitc_delete.py10
-rw-r--r--subcmds/gitc_init.py19
-rw-r--r--subcmds/grep.py202
-rw-r--r--subcmds/help.py61
-rw-r--r--subcmds/info.py55
-rw-r--r--subcmds/init.py362
-rw-r--r--subcmds/list.py49
-rw-r--r--subcmds/manifest.py70
-rw-r--r--subcmds/overview.py20
-rw-r--r--subcmds/prune.py31
-rw-r--r--subcmds/rebase.py39
-rw-r--r--subcmds/selfupdate.py12
-rw-r--r--subcmds/smartsync.py5
-rw-r--r--subcmds/stage.py11
-rw-r--r--subcmds/start.py68
-rw-r--r--subcmds/status.py96
-rw-r--r--subcmds/sync.py1054
-rw-r--r--subcmds/upload.py315
-rw-r--r--subcmds/version.py27
28 files changed, 1766 insertions, 1395 deletions
diff --git a/subcmds/__init__.py b/subcmds/__init__.py
index 27341038..051dda06 100644
--- a/subcmds/__init__.py
+++ b/subcmds/__init__.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,6 +14,7 @@
16 14
17import os 15import os
18 16
17# A mapping of the subcommand name to the class that implements it.
19all_commands = {} 18all_commands = {}
20 19
21my_dir = os.path.dirname(__file__) 20my_dir = os.path.dirname(__file__)
@@ -37,14 +36,14 @@ for py in os.listdir(my_dir):
37 ['%s' % name]) 36 ['%s' % name])
38 mod = getattr(mod, name) 37 mod = getattr(mod, name)
39 try: 38 try:
40 cmd = getattr(mod, clsn)() 39 cmd = getattr(mod, clsn)
41 except AttributeError: 40 except AttributeError:
42 raise SyntaxError('%s/%s does not define class %s' % ( 41 raise SyntaxError('%s/%s does not define class %s' % (
43 __name__, py, clsn)) 42 __name__, py, clsn))
44 43
45 name = name.replace('_', '-') 44 name = name.replace('_', '-')
46 cmd.NAME = name 45 cmd.NAME = name
47 all_commands[name] = cmd 46 all_commands[name] = cmd
48 47
49if 'help' in all_commands: 48# Add 'branch' as an alias for 'branches'.
50 all_commands['help'].commands = all_commands 49all_commands['branch'] = all_commands['branches']
diff --git a/subcmds/abandon.py b/subcmds/abandon.py
index cd1d0c40..85d85f5a 100644
--- a/subcmds/abandon.py
+++ b/subcmds/abandon.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,15 +12,18 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function
18import sys
19from command import Command
20from collections import defaultdict 15from collections import defaultdict
16import functools
17import itertools
18import sys
19
20from command import Command, DEFAULT_LOCAL_JOBS
21from git_command import git 21from git_command import git
22from progress import Progress 22from progress import Progress
23 23
24
24class Abandon(Command): 25class Abandon(Command):
25 common = True 26 COMMON = True
26 helpSummary = "Permanently abandon a development branch" 27 helpSummary = "Permanently abandon a development branch"
27 helpUsage = """ 28 helpUsage = """
28%prog [--all | <branchname>] [<project>...] 29%prog [--all | <branchname>] [<project>...]
@@ -32,6 +33,8 @@ deleting it (and all its history) from your local repository.
32 33
33It is equivalent to "git branch -D <branchname>". 34It is equivalent to "git branch -D <branchname>".
34""" 35"""
36 PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
37
35 def _Options(self, p): 38 def _Options(self, p):
36 p.add_option('--all', 39 p.add_option('--all',
37 dest='all', action='store_true', 40 dest='all', action='store_true',
@@ -48,52 +51,64 @@ It is equivalent to "git branch -D <branchname>".
48 else: 51 else:
49 args.insert(0, "'All local branches'") 52 args.insert(0, "'All local branches'")
50 53
54 def _ExecuteOne(self, all_branches, nb, project):
55 """Abandon one project."""
56 if all_branches:
57 branches = project.GetBranches()
58 else:
59 branches = [nb]
60
61 ret = {}
62 for name in branches:
63 status = project.AbandonBranch(name)
64 if status is not None:
65 ret[name] = status
66 return (ret, project)
67
51 def Execute(self, opt, args): 68 def Execute(self, opt, args):
52 nb = args[0] 69 nb = args[0]
53 err = defaultdict(list) 70 err = defaultdict(list)
54 success = defaultdict(list) 71 success = defaultdict(list)
55 all_projects = self.GetProjects(args[1:]) 72 all_projects = self.GetProjects(args[1:])
56 73
57 pm = Progress('Abandon %s' % nb, len(all_projects)) 74 def _ProcessResults(_pool, pm, states):
58 for project in all_projects: 75 for (results, project) in states:
59 pm.update() 76 for branch, status in results.items():
60
61 if opt.all:
62 branches = list(project.GetBranches().keys())
63 else:
64 branches = [nb]
65
66 for name in branches:
67 status = project.AbandonBranch(name)
68 if status is not None:
69 if status: 77 if status:
70 success[name].append(project) 78 success[branch].append(project)
71 else: 79 else:
72 err[name].append(project) 80 err[branch].append(project)
73 pm.end() 81 pm.update()
74 82
75 width = 25 83 self.ExecuteInParallel(
76 for name in branches: 84 opt.jobs,
77 if width < len(name): 85 functools.partial(self._ExecuteOne, opt.all, nb),
78 width = len(name) 86 all_projects,
87 callback=_ProcessResults,
88 output=Progress('Abandon %s' % (nb,), len(all_projects), quiet=opt.quiet))
79 89
90 width = max(itertools.chain(
91 [25], (len(x) for x in itertools.chain(success, err))))
80 if err: 92 if err:
81 for br in err.keys(): 93 for br in err.keys():
82 err_msg = "error: cannot abandon %s" %br 94 err_msg = "error: cannot abandon %s" % br
83 print(err_msg, file=sys.stderr) 95 print(err_msg, file=sys.stderr)
84 for proj in err[br]: 96 for proj in err[br]:
85 print(' '*len(err_msg) + " | %s" % proj.relpath, file=sys.stderr) 97 print(' ' * len(err_msg) + " | %s" % proj.relpath, file=sys.stderr)
86 sys.exit(1) 98 sys.exit(1)
87 elif not success: 99 elif not success:
88 print('error: no project has local branch(es) : %s' % nb, 100 print('error: no project has local branch(es) : %s' % nb,
89 file=sys.stderr) 101 file=sys.stderr)
90 sys.exit(1) 102 sys.exit(1)
91 else: 103 else:
92 print('Abandoned branches:', file=sys.stderr) 104 # Everything below here is displaying status.
105 if opt.quiet:
106 return
107 print('Abandoned branches:')
93 for br in success.keys(): 108 for br in success.keys():
94 if len(all_projects) > 1 and len(all_projects) == len(success[br]): 109 if len(all_projects) > 1 and len(all_projects) == len(success[br]):
95 result = "all project" 110 result = "all project"
96 else: 111 else:
97 result = "%s" % ( 112 result = "%s" % (
98 ('\n'+' '*width + '| ').join(p.relpath for p in success[br])) 113 ('\n' + ' ' * width + '| ').join(p.relpath for p in success[br]))
99 print("%s%s| %s\n" % (br,' '*(width-len(br)), result),file=sys.stderr) 114 print("%s%s| %s\n" % (br, ' ' * (width - len(br)), result))
diff --git a/subcmds/branches.py b/subcmds/branches.py
index fb60d7de..6d975ed4 100644
--- a/subcmds/branches.py
+++ b/subcmds/branches.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2009 The Android Open Source Project 1# Copyright (C) 2009 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,18 +12,21 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function 15import itertools
18import sys 16import sys
17
19from color import Coloring 18from color import Coloring
20from command import Command 19from command import Command, DEFAULT_LOCAL_JOBS
20
21 21
22class BranchColoring(Coloring): 22class BranchColoring(Coloring):
23 def __init__(self, config): 23 def __init__(self, config):
24 Coloring.__init__(self, config, 'branch') 24 Coloring.__init__(self, config, 'branch')
25 self.current = self.printer('current', fg='green') 25 self.current = self.printer('current', fg='green')
26 self.local = self.printer('local') 26 self.local = self.printer('local')
27 self.notinproject = self.printer('notinproject', fg='red') 27 self.notinproject = self.printer('notinproject', fg='red')
28 28
29
29class BranchInfo(object): 30class BranchInfo(object):
30 def __init__(self, name): 31 def __init__(self, name):
31 self.name = name 32 self.name = name
@@ -61,7 +62,7 @@ class BranchInfo(object):
61 62
62 63
63class Branches(Command): 64class Branches(Command):
64 common = True 65 COMMON = True
65 helpSummary = "View current topic branches" 66 helpSummary = "View current topic branches"
66 helpUsage = """ 67 helpUsage = """
67%prog [<project>...] 68%prog [<project>...]
@@ -94,6 +95,7 @@ the branch appears in, or does not appear in. If no project list
94is shown, then the branch appears in all projects. 95is shown, then the branch appears in all projects.
95 96
96""" 97"""
98 PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
97 99
98 def Execute(self, opt, args): 100 def Execute(self, opt, args):
99 projects = self.GetProjects(args) 101 projects = self.GetProjects(args)
@@ -101,14 +103,19 @@ is shown, then the branch appears in all projects.
101 all_branches = {} 103 all_branches = {}
102 project_cnt = len(projects) 104 project_cnt = len(projects)
103 105
104 for project in projects: 106 def _ProcessResults(_pool, _output, results):
105 for name, b in project.GetBranches().items(): 107 for name, b in itertools.chain.from_iterable(results):
106 b.project = project
107 if name not in all_branches: 108 if name not in all_branches:
108 all_branches[name] = BranchInfo(name) 109 all_branches[name] = BranchInfo(name)
109 all_branches[name].add(b) 110 all_branches[name].add(b)
110 111
111 names = list(sorted(all_branches)) 112 self.ExecuteInParallel(
113 opt.jobs,
114 expand_project_to_branches,
115 projects,
116 callback=_ProcessResults)
117
118 names = sorted(all_branches)
112 119
113 if not names: 120 if not names:
114 print(' (no branches)', file=sys.stderr) 121 print(' (no branches)', file=sys.stderr)
@@ -158,7 +165,7 @@ is shown, then the branch appears in all projects.
158 for b in i.projects: 165 for b in i.projects:
159 have.add(b.project) 166 have.add(b.project)
160 for p in projects: 167 for p in projects:
161 if not p in have: 168 if p not in have:
162 paths.append(p.relpath) 169 paths.append(p.relpath)
163 170
164 s = ' %s %s' % (in_type, ', '.join(paths)) 171 s = ' %s %s' % (in_type, ', '.join(paths))
@@ -170,11 +177,27 @@ is shown, then the branch appears in all projects.
170 fmt = out.current if i.IsCurrent else out.write 177 fmt = out.current if i.IsCurrent else out.write
171 for p in paths: 178 for p in paths:
172 out.nl() 179 out.nl()
173 fmt(width*' ' + ' %s' % p) 180 fmt(width * ' ' + ' %s' % p)
174 fmt = out.write 181 fmt = out.write
175 for p in non_cur_paths: 182 for p in non_cur_paths:
176 out.nl() 183 out.nl()
177 fmt(width*' ' + ' %s' % p) 184 fmt(width * ' ' + ' %s' % p)
178 else: 185 else:
179 out.write(' in all projects') 186 out.write(' in all projects')
180 out.nl() 187 out.nl()
188
189
190def expand_project_to_branches(project):
191 """Expands a project into a list of branch names & associated information.
192
193 Args:
194 project: project.Project
195
196 Returns:
197 List[Tuple[str, git_config.Branch]]
198 """
199 branches = []
200 for name, b in project.GetBranches().items():
201 b.project = project
202 branches.append((name, b))
203 return branches
diff --git a/subcmds/checkout.py b/subcmds/checkout.py
index c8a09a8e..9b429489 100644
--- a/subcmds/checkout.py
+++ b/subcmds/checkout.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2009 The Android Open Source Project 1# Copyright (C) 2009 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,13 +12,15 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function 15import functools
18import sys 16import sys
19from command import Command 17
18from command import Command, DEFAULT_LOCAL_JOBS
20from progress import Progress 19from progress import Progress
21 20
21
22class Checkout(Command): 22class Checkout(Command):
23 common = True 23 COMMON = True
24 helpSummary = "Checkout a branch for development" 24 helpSummary = "Checkout a branch for development"
25 helpUsage = """ 25 helpUsage = """
26%prog <branchname> [<project>...] 26%prog <branchname> [<project>...]
@@ -33,28 +33,37 @@ The command is equivalent to:
33 33
34 repo forall [<project>...] -c git checkout <branchname> 34 repo forall [<project>...] -c git checkout <branchname>
35""" 35"""
36 PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
36 37
37 def ValidateOptions(self, opt, args): 38 def ValidateOptions(self, opt, args):
38 if not args: 39 if not args:
39 self.Usage() 40 self.Usage()
40 41
42 def _ExecuteOne(self, nb, project):
43 """Checkout one project."""
44 return (project.CheckoutBranch(nb), project)
45
41 def Execute(self, opt, args): 46 def Execute(self, opt, args):
42 nb = args[0] 47 nb = args[0]
43 err = [] 48 err = []
44 success = [] 49 success = []
45 all_projects = self.GetProjects(args[1:]) 50 all_projects = self.GetProjects(args[1:])
46 51
47 pm = Progress('Checkout %s' % nb, len(all_projects)) 52 def _ProcessResults(_pool, pm, results):
48 for project in all_projects: 53 for status, project in results:
49 pm.update() 54 if status is not None:
55 if status:
56 success.append(project)
57 else:
58 err.append(project)
59 pm.update()
50 60
51 status = project.CheckoutBranch(nb) 61 self.ExecuteInParallel(
52 if status is not None: 62 opt.jobs,
53 if status: 63 functools.partial(self._ExecuteOne, nb),
54 success.append(project) 64 all_projects,
55 else: 65 callback=_ProcessResults,
56 err.append(project) 66 output=Progress('Checkout %s' % (nb,), len(all_projects), quiet=opt.quiet))
57 pm.end()
58 67
59 if err: 68 if err:
60 for p in err: 69 for p in err:
diff --git a/subcmds/cherry_pick.py b/subcmds/cherry_pick.py
index a541a040..7bd858bf 100644
--- a/subcmds/cherry_pick.py
+++ b/subcmds/cherry_pick.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2010 The Android Open Source Project 1# Copyright (C) 2010 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +12,6 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function
18import re 15import re
19import sys 16import sys
20from command import Command 17from command import Command
@@ -22,8 +19,9 @@ from git_command import GitCommand
22 19
23CHANGE_ID_RE = re.compile(r'^\s*Change-Id: I([0-9a-f]{40})\s*$') 20CHANGE_ID_RE = re.compile(r'^\s*Change-Id: I([0-9a-f]{40})\s*$')
24 21
22
25class CherryPick(Command): 23class CherryPick(Command):
26 common = True 24 COMMON = True
27 helpSummary = "Cherry-pick a change." 25 helpSummary = "Cherry-pick a change."
28 helpUsage = """ 26 helpUsage = """
29%prog <sha1> 27%prog <sha1>
@@ -34,9 +32,6 @@ The change id will be updated, and a reference to the old
34change id will be added. 32change id will be added.
35""" 33"""
36 34
37 def _Options(self, p):
38 pass
39
40 def ValidateOptions(self, opt, args): 35 def ValidateOptions(self, opt, args):
41 if len(args) != 1: 36 if len(args) != 1:
42 self.Usage() 37 self.Usage()
@@ -46,8 +41,8 @@ change id will be added.
46 41
47 p = GitCommand(None, 42 p = GitCommand(None,
48 ['rev-parse', '--verify', reference], 43 ['rev-parse', '--verify', reference],
49 capture_stdout = True, 44 capture_stdout=True,
50 capture_stderr = True) 45 capture_stderr=True)
51 if p.Wait() != 0: 46 if p.Wait() != 0:
52 print(p.stderr, file=sys.stderr) 47 print(p.stderr, file=sys.stderr)
53 sys.exit(1) 48 sys.exit(1)
@@ -61,8 +56,8 @@ change id will be added.
61 56
62 p = GitCommand(None, 57 p = GitCommand(None,
63 ['cherry-pick', sha1], 58 ['cherry-pick', sha1],
64 capture_stdout = True, 59 capture_stdout=True,
65 capture_stderr = True) 60 capture_stderr=True)
66 status = p.Wait() 61 status = p.Wait()
67 62
68 print(p.stdout, file=sys.stdout) 63 print(p.stdout, file=sys.stdout)
@@ -74,11 +69,9 @@ change id will be added.
74 new_msg = self._Reformat(old_msg, sha1) 69 new_msg = self._Reformat(old_msg, sha1)
75 70
76 p = GitCommand(None, ['commit', '--amend', '-F', '-'], 71 p = GitCommand(None, ['commit', '--amend', '-F', '-'],
77 provide_stdin = True, 72 input=new_msg,
78 capture_stdout = True, 73 capture_stdout=True,
79 capture_stderr = True) 74 capture_stderr=True)
80 p.stdin.write(new_msg)
81 p.stdin.close()
82 if p.Wait() != 0: 75 if p.Wait() != 0:
83 print("error: Failed to update commit message", file=sys.stderr) 76 print("error: Failed to update commit message", file=sys.stderr)
84 sys.exit(1) 77 sys.exit(1)
@@ -97,7 +90,7 @@ change id will be added.
97 90
98 def _StripHeader(self, commit_msg): 91 def _StripHeader(self, commit_msg):
99 lines = commit_msg.splitlines() 92 lines = commit_msg.splitlines()
100 return "\n".join(lines[lines.index("")+1:]) 93 return "\n".join(lines[lines.index("") + 1:])
101 94
102 def _Reformat(self, old_msg, sha1): 95 def _Reformat(self, old_msg, sha1):
103 new_msg = [] 96 new_msg = []
diff --git a/subcmds/diff.py b/subcmds/diff.py
index fa41e70e..00a7ec29 100644
--- a/subcmds/diff.py
+++ b/subcmds/diff.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,10 +12,14 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from command import PagedCommand 15import functools
16import io
17
18from command import DEFAULT_LOCAL_JOBS, PagedCommand
19
18 20
19class Diff(PagedCommand): 21class Diff(PagedCommand):
20 common = True 22 COMMON = True
21 helpSummary = "Show changes between commit and working tree" 23 helpSummary = "Show changes between commit and working tree"
22 helpUsage = """ 24 helpUsage = """
23%prog [<project>...] 25%prog [<project>...]
@@ -26,19 +28,42 @@ The -u option causes '%prog' to generate diff output with file paths
26relative to the repository root, so the output can be applied 28relative to the repository root, so the output can be applied
27to the Unix 'patch' command. 29to the Unix 'patch' command.
28""" 30"""
31 PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
29 32
30 def _Options(self, p): 33 def _Options(self, p):
31 def cmd(option, opt_str, value, parser):
32 setattr(parser.values, option.dest, list(parser.rargs))
33 while parser.rargs:
34 del parser.rargs[0]
35 p.add_option('-u', '--absolute', 34 p.add_option('-u', '--absolute',
36 dest='absolute', action='store_true', 35 dest='absolute', action='store_true',
37 help='Paths are relative to the repository root') 36 help='paths are relative to the repository root')
37
38 def _ExecuteOne(self, absolute, project):
39 """Obtains the diff for a specific project.
40
41 Args:
42 absolute: Paths are relative to the root.
43 project: Project to get status of.
44
45 Returns:
46 The status of the project.
47 """
48 buf = io.StringIO()
49 ret = project.PrintWorkTreeDiff(absolute, output_redir=buf)
50 return (ret, buf.getvalue())
38 51
39 def Execute(self, opt, args): 52 def Execute(self, opt, args):
40 ret = 0 53 all_projects = self.GetProjects(args)
41 for project in self.GetProjects(args): 54
42 if not project.PrintWorkTreeDiff(opt.absolute): 55 def _ProcessResults(_pool, _output, results):
43 ret = 1 56 ret = 0
44 return ret 57 for (state, output) in results:
58 if output:
59 print(output, end='')
60 if not state:
61 ret = 1
62 return ret
63
64 return self.ExecuteInParallel(
65 opt.jobs,
66 functools.partial(self._ExecuteOne, opt.absolute),
67 all_projects,
68 callback=_ProcessResults,
69 ordered=True)
diff --git a/subcmds/diffmanifests.py b/subcmds/diffmanifests.py
index b999699e..f6cc30a2 100644
--- a/subcmds/diffmanifests.py
+++ b/subcmds/diffmanifests.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2014 The Android Open Source Project 1# Copyright (C) 2014 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,12 +14,14 @@
16 14
17from color import Coloring 15from color import Coloring
18from command import PagedCommand 16from command import PagedCommand
19from manifest_xml import XmlManifest 17from manifest_xml import RepoClient
18
20 19
21class _Coloring(Coloring): 20class _Coloring(Coloring):
22 def __init__(self, config): 21 def __init__(self, config):
23 Coloring.__init__(self, config, "status") 22 Coloring.__init__(self, config, "status")
24 23
24
25class Diffmanifests(PagedCommand): 25class Diffmanifests(PagedCommand):
26 """ A command to see logs in projects represented by manifests 26 """ A command to see logs in projects represented by manifests
27 27
@@ -31,7 +31,7 @@ class Diffmanifests(PagedCommand):
31 deeper level. 31 deeper level.
32 """ 32 """
33 33
34 common = True 34 COMMON = True
35 helpSummary = "Manifest diff utility" 35 helpSummary = "Manifest diff utility"
36 helpUsage = """%prog manifest1.xml [manifest2.xml] [options]""" 36 helpUsage = """%prog manifest1.xml [manifest2.xml] [options]"""
37 37
@@ -68,16 +68,16 @@ synced and their revisions won't be found.
68 def _Options(self, p): 68 def _Options(self, p):
69 p.add_option('--raw', 69 p.add_option('--raw',
70 dest='raw', action='store_true', 70 dest='raw', action='store_true',
71 help='Display raw diff.') 71 help='display raw diff')
72 p.add_option('--no-color', 72 p.add_option('--no-color',
73 dest='color', action='store_false', default=True, 73 dest='color', action='store_false', default=True,
74 help='does not display the diff in color.') 74 help='does not display the diff in color')
75 p.add_option('--pretty-format', 75 p.add_option('--pretty-format',
76 dest='pretty_format', action='store', 76 dest='pretty_format', action='store',
77 metavar='<FORMAT>', 77 metavar='<FORMAT>',
78 help='print the log using a custom git pretty format string') 78 help='print the log using a custom git pretty format string')
79 79
80 def _printRawDiff(self, diff): 80 def _printRawDiff(self, diff, pretty_format=None):
81 for project in diff['added']: 81 for project in diff['added']:
82 self.printText("A %s %s" % (project.relpath, project.revisionExpr)) 82 self.printText("A %s %s" % (project.relpath, project.revisionExpr))
83 self.out.nl() 83 self.out.nl()
@@ -90,7 +90,7 @@ synced and their revisions won't be found.
90 self.printText("C %s %s %s" % (project.relpath, project.revisionExpr, 90 self.printText("C %s %s %s" % (project.relpath, project.revisionExpr,
91 otherProject.revisionExpr)) 91 otherProject.revisionExpr))
92 self.out.nl() 92 self.out.nl()
93 self._printLogs(project, otherProject, raw=True, color=False) 93 self._printLogs(project, otherProject, raw=True, color=False, pretty_format=pretty_format)
94 94
95 for project, otherProject in diff['unreachable']: 95 for project, otherProject in diff['unreachable']:
96 self.printText("U %s %s %s" % (project.relpath, project.revisionExpr, 96 self.printText("U %s %s %s" % (project.relpath, project.revisionExpr,
@@ -181,26 +181,26 @@ synced and their revisions won't be found.
181 self.OptionParser.error('missing manifests to diff') 181 self.OptionParser.error('missing manifests to diff')
182 182
183 def Execute(self, opt, args): 183 def Execute(self, opt, args):
184 self.out = _Coloring(self.manifest.globalConfig) 184 self.out = _Coloring(self.client.globalConfig)
185 self.printText = self.out.nofmt_printer('text') 185 self.printText = self.out.nofmt_printer('text')
186 if opt.color: 186 if opt.color:
187 self.printProject = self.out.nofmt_printer('project', attr = 'bold') 187 self.printProject = self.out.nofmt_printer('project', attr='bold')
188 self.printAdded = self.out.nofmt_printer('green', fg = 'green', attr = 'bold') 188 self.printAdded = self.out.nofmt_printer('green', fg='green', attr='bold')
189 self.printRemoved = self.out.nofmt_printer('red', fg = 'red', attr = 'bold') 189 self.printRemoved = self.out.nofmt_printer('red', fg='red', attr='bold')
190 self.printRevision = self.out.nofmt_printer('revision', fg = 'yellow') 190 self.printRevision = self.out.nofmt_printer('revision', fg='yellow')
191 else: 191 else:
192 self.printProject = self.printAdded = self.printRemoved = self.printRevision = self.printText 192 self.printProject = self.printAdded = self.printRemoved = self.printRevision = self.printText
193 193
194 manifest1 = XmlManifest(self.manifest.repodir) 194 manifest1 = RepoClient(self.repodir)
195 manifest1.Override(args[0], load_local_manifests=False) 195 manifest1.Override(args[0], load_local_manifests=False)
196 if len(args) == 1: 196 if len(args) == 1:
197 manifest2 = self.manifest 197 manifest2 = self.manifest
198 else: 198 else:
199 manifest2 = XmlManifest(self.manifest.repodir) 199 manifest2 = RepoClient(self.repodir)
200 manifest2.Override(args[1], load_local_manifests=False) 200 manifest2.Override(args[1], load_local_manifests=False)
201 201
202 diff = manifest1.projectsDiff(manifest2) 202 diff = manifest1.projectsDiff(manifest2)
203 if opt.raw: 203 if opt.raw:
204 self._printRawDiff(diff) 204 self._printRawDiff(diff, pretty_format=opt.pretty_format)
205 else: 205 else:
206 self._printDiff(diff, color=opt.color, pretty_format=opt.pretty_format) 206 self._printDiff(diff, color=opt.color, pretty_format=opt.pretty_format)
diff --git a/subcmds/download.py b/subcmds/download.py
index f746bc23..523f25e0 100644
--- a/subcmds/download.py
+++ b/subcmds/download.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,17 +12,17 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function
18import re 15import re
19import sys 16import sys
20 17
21from command import Command 18from command import Command
22from error import GitError 19from error import GitError, NoSuchProjectError
23 20
24CHANGE_RE = re.compile(r'^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$') 21CHANGE_RE = re.compile(r'^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$')
25 22
23
26class Download(Command): 24class Download(Command):
27 common = True 25 COMMON = True
28 helpSummary = "Download and checkout a change" 26 helpSummary = "Download and checkout a change"
29 helpUsage = """ 27 helpUsage = """
30%prog {[project] change[/patchset]}... 28%prog {[project] change[/patchset]}...
@@ -36,9 +34,13 @@ If no project is specified try to use current directory as a project.
36""" 34"""
37 35
38 def _Options(self, p): 36 def _Options(self, p):
37 p.add_option('-b', '--branch',
38 help='create a new branch first')
39 p.add_option('-c', '--cherry-pick', 39 p.add_option('-c', '--cherry-pick',
40 dest='cherrypick', action='store_true', 40 dest='cherrypick', action='store_true',
41 help="cherry-pick instead of checkout") 41 help="cherry-pick instead of checkout")
42 p.add_option('-x', '--record-origin', action='store_true',
43 help='pass -x when cherry-picking')
42 p.add_option('-r', '--revert', 44 p.add_option('-r', '--revert',
43 dest='revert', action='store_true', 45 dest='revert', action='store_true',
44 help="revert instead of checkout") 46 help="revert instead of checkout")
@@ -58,6 +60,7 @@ If no project is specified try to use current directory as a project.
58 if m: 60 if m:
59 if not project: 61 if not project:
60 project = self.GetProjects(".")[0] 62 project = self.GetProjects(".")[0]
63 print('Defaulting to cwd project', project.name)
61 chg_id = int(m.group(1)) 64 chg_id = int(m.group(1))
62 if m.group(2): 65 if m.group(2):
63 ps_id = int(m.group(2)) 66 ps_id = int(m.group(2))
@@ -74,9 +77,33 @@ If no project is specified try to use current directory as a project.
74 ps_id = max(int(match.group(1)), ps_id) 77 ps_id = max(int(match.group(1)), ps_id)
75 to_get.append((project, chg_id, ps_id)) 78 to_get.append((project, chg_id, ps_id))
76 else: 79 else:
77 project = self.GetProjects([a])[0] 80 projects = self.GetProjects([a])
81 if len(projects) > 1:
82 # If the cwd is one of the projects, assume they want that.
83 try:
84 project = self.GetProjects('.')[0]
85 except NoSuchProjectError:
86 project = None
87 if project not in projects:
88 print('error: %s matches too many projects; please re-run inside '
89 'the project checkout.' % (a,), file=sys.stderr)
90 for project in projects:
91 print(' %s/ @ %s' % (project.relpath, project.revisionExpr),
92 file=sys.stderr)
93 sys.exit(1)
94 else:
95 project = projects[0]
96 print('Defaulting to cwd project', project.name)
78 return to_get 97 return to_get
79 98
99 def ValidateOptions(self, opt, args):
100 if opt.record_origin:
101 if not opt.cherrypick:
102 self.OptionParser.error('-x only makes sense with --cherry-pick')
103
104 if opt.ffonly:
105 self.OptionParser.error('-x and --ff are mutually exclusive options')
106
80 def Execute(self, opt, args): 107 def Execute(self, opt, args):
81 for project, change_id, ps_id in self._ParseChangeIds(args): 108 for project, change_id, ps_id in self._ParseChangeIds(args):
82 dl = project.DownloadPatchSet(change_id, ps_id) 109 dl = project.DownloadPatchSet(change_id, ps_id)
@@ -93,22 +120,41 @@ If no project is specified try to use current directory as a project.
93 continue 120 continue
94 121
95 if len(dl.commits) > 1: 122 if len(dl.commits) > 1:
96 print('[%s] %d/%d depends on %d unmerged changes:' \ 123 print('[%s] %d/%d depends on %d unmerged changes:'
97 % (project.name, change_id, ps_id, len(dl.commits)), 124 % (project.name, change_id, ps_id, len(dl.commits)),
98 file=sys.stderr) 125 file=sys.stderr)
99 for c in dl.commits: 126 for c in dl.commits:
100 print(' %s' % (c), file=sys.stderr) 127 print(' %s' % (c), file=sys.stderr)
101 if opt.cherrypick:
102 try:
103 project._CherryPick(dl.commit)
104 except GitError:
105 print('[%s] Could not complete the cherry-pick of %s' \
106 % (project.name, dl.commit), file=sys.stderr)
107 sys.exit(1)
108 128
129 if opt.cherrypick:
130 mode = 'cherry-pick'
109 elif opt.revert: 131 elif opt.revert:
110 project._Revert(dl.commit) 132 mode = 'revert'
111 elif opt.ffonly: 133 elif opt.ffonly:
112 project._FastForward(dl.commit, ffonly=True) 134 mode = 'fast-forward merge'
113 else: 135 else:
114 project._Checkout(dl.commit) 136 mode = 'checkout'
137
138 # We'll combine the branch+checkout operation, but all the rest need a
139 # dedicated branch start.
140 if opt.branch and mode != 'checkout':
141 project.StartBranch(opt.branch)
142
143 try:
144 if opt.cherrypick:
145 project._CherryPick(dl.commit, ffonly=opt.ffonly,
146 record_origin=opt.record_origin)
147 elif opt.revert:
148 project._Revert(dl.commit)
149 elif opt.ffonly:
150 project._FastForward(dl.commit, ffonly=True)
151 else:
152 if opt.branch:
153 project.StartBranch(opt.branch, revision=dl.commit)
154 else:
155 project._Checkout(dl.commit)
156
157 except GitError:
158 print('[%s] Could not complete the %s of %s'
159 % (project.name, mode, dl.commit), file=sys.stderr)
160 sys.exit(1)
diff --git a/subcmds/forall.py b/subcmds/forall.py
index 131ba676..7c1dea9e 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,8 +12,9 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function
18import errno 15import errno
16import functools
17import io
19import multiprocessing 18import multiprocessing
20import re 19import re
21import os 20import os
@@ -24,14 +23,14 @@ import sys
24import subprocess 23import subprocess
25 24
26from color import Coloring 25from color import Coloring
27from command import Command, MirrorSafeCommand 26from command import DEFAULT_LOCAL_JOBS, Command, MirrorSafeCommand, WORKER_BATCH_SIZE
28import platform_utils 27from error import ManifestInvalidRevisionError
29 28
30_CAN_COLOR = [ 29_CAN_COLOR = [
31 'branch', 30 'branch',
32 'diff', 31 'diff',
33 'grep', 32 'grep',
34 'log', 33 'log',
35] 34]
36 35
37 36
@@ -42,11 +41,11 @@ class ForallColoring(Coloring):
42 41
43 42
44class Forall(Command, MirrorSafeCommand): 43class Forall(Command, MirrorSafeCommand):
45 common = False 44 COMMON = False
46 helpSummary = "Run a shell command in each project" 45 helpSummary = "Run a shell command in each project"
47 helpUsage = """ 46 helpUsage = """
48%prog [<project>...] -c <command> [<arg>...] 47%prog [<project>...] -c <command> [<arg>...]
49%prog -r str1 [str2] ... -c <command> [<arg>...]" 48%prog -r str1 [str2] ... -c <command> [<arg>...]
50""" 49"""
51 helpDescription = """ 50 helpDescription = """
52Executes the same shell command in each project. 51Executes the same shell command in each project.
@@ -54,6 +53,11 @@ Executes the same shell command in each project.
54The -r option allows running the command only on projects matching 53The -r option allows running the command only on projects matching
55regex or wildcard expression. 54regex or wildcard expression.
56 55
56By default, projects are processed non-interactively in parallel. If you want
57to run interactive commands, make sure to pass --interactive to force --jobs 1.
58While the processing order of projects is not guaranteed, the order of project
59output is stable.
60
57# Output Formatting 61# Output Formatting
58 62
59The -p option causes '%prog' to bind pipes to the command's stdin, 63The -p option causes '%prog' to bind pipes to the command's stdin,
@@ -116,70 +120,48 @@ terminal and are not redirected.
116If -e is used, when a command exits unsuccessfully, '%prog' will abort 120If -e is used, when a command exits unsuccessfully, '%prog' will abort
117without iterating through the remaining projects. 121without iterating through the remaining projects.
118""" 122"""
123 PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
124
125 @staticmethod
126 def _cmd_option(option, _opt_str, _value, parser):
127 setattr(parser.values, option.dest, list(parser.rargs))
128 while parser.rargs:
129 del parser.rargs[0]
119 130
120 def _Options(self, p): 131 def _Options(self, p):
121 def cmd(option, opt_str, value, parser):
122 setattr(parser.values, option.dest, list(parser.rargs))
123 while parser.rargs:
124 del parser.rargs[0]
125 p.add_option('-r', '--regex', 132 p.add_option('-r', '--regex',
126 dest='regex', action='store_true', 133 dest='regex', action='store_true',
127 help="Execute the command only on projects matching regex or wildcard expression") 134 help='execute the command only on projects matching regex or wildcard expression')
128 p.add_option('-i', '--inverse-regex', 135 p.add_option('-i', '--inverse-regex',
129 dest='inverse_regex', action='store_true', 136 dest='inverse_regex', action='store_true',
130 help="Execute the command only on projects not matching regex or wildcard expression") 137 help='execute the command only on projects not matching regex or '
138 'wildcard expression')
131 p.add_option('-g', '--groups', 139 p.add_option('-g', '--groups',
132 dest='groups', 140 dest='groups',
133 help="Execute the command only on projects matching the specified groups") 141 help='execute the command only on projects matching the specified groups')
134 p.add_option('-c', '--command', 142 p.add_option('-c', '--command',
135 help='Command (and arguments) to execute', 143 help='command (and arguments) to execute',
136 dest='command', 144 dest='command',
137 action='callback', 145 action='callback',
138 callback=cmd) 146 callback=self._cmd_option)
139 p.add_option('-e', '--abort-on-errors', 147 p.add_option('-e', '--abort-on-errors',
140 dest='abort_on_errors', action='store_true', 148 dest='abort_on_errors', action='store_true',
141 help='Abort if a command exits unsuccessfully') 149 help='abort if a command exits unsuccessfully')
142 p.add_option('--ignore-missing', action='store_true', 150 p.add_option('--ignore-missing', action='store_true',
143 help='Silently skip & do not exit non-zero due missing ' 151 help='silently skip & do not exit non-zero due missing '
144 'checkouts') 152 'checkouts')
145 153
146 g = p.add_option_group('Output') 154 g = p.get_option_group('--quiet')
147 g.add_option('-p', 155 g.add_option('-p',
148 dest='project_header', action='store_true', 156 dest='project_header', action='store_true',
149 help='Show project headers before output') 157 help='show project headers before output')
150 g.add_option('-v', '--verbose', 158 p.add_option('--interactive',
151 dest='verbose', action='store_true', 159 action='store_true',
152 help='Show command error messages') 160 help='force interactive usage')
153 g.add_option('-j', '--jobs',
154 dest='jobs', action='store', type='int', default=1,
155 help='number of commands to execute simultaneously')
156 161
157 def WantPager(self, opt): 162 def WantPager(self, opt):
158 return opt.project_header and opt.jobs == 1 163 return opt.project_header and opt.jobs == 1
159 164
160 def _SerializeProject(self, project):
161 """ Serialize a project._GitGetByExec instance.
162
163 project._GitGetByExec is not pickle-able. Instead of trying to pass it
164 around between processes, make a dict ourselves containing only the
165 attributes that we need.
166
167 """
168 if not self.manifest.IsMirror:
169 lrev = project.GetRevisionId()
170 else:
171 lrev = None
172 return {
173 'name': project.name,
174 'relpath': project.relpath,
175 'remote_name': project.remote.name,
176 'lrev': lrev,
177 'rrev': project.revisionExpr,
178 'annotations': dict((a.name, a.value) for a in project.annotations),
179 'gitdir': project.gitdir,
180 'worktree': project.worktree,
181 }
182
183 def ValidateOptions(self, opt, args): 165 def ValidateOptions(self, opt, args):
184 if not opt.command: 166 if not opt.command:
185 self.Usage() 167 self.Usage()
@@ -195,9 +177,14 @@ without iterating through the remaining projects.
195 cmd.append(cmd[0]) 177 cmd.append(cmd[0])
196 cmd.extend(opt.command[1:]) 178 cmd.extend(opt.command[1:])
197 179
198 if opt.project_header \ 180 # Historically, forall operated interactively, and in serial. If the user
199 and not shell \ 181 # has selected 1 job, then default to interacive mode.
200 and cmd[0] == 'git': 182 if opt.jobs == 1:
183 opt.interactive = True
184
185 if opt.project_header \
186 and not shell \
187 and cmd[0] == 'git':
201 # If this is a direct git command that can enable colorized 188 # If this is a direct git command that can enable colorized
202 # output and the user prefers coloring, add --color into the 189 # output and the user prefers coloring, add --color into the
203 # command line because we are going to wrap the command into 190 # command line because we are going to wrap the command into
@@ -220,7 +207,7 @@ without iterating through the remaining projects.
220 207
221 smart_sync_manifest_name = "smart_sync_override.xml" 208 smart_sync_manifest_name = "smart_sync_override.xml"
222 smart_sync_manifest_path = os.path.join( 209 smart_sync_manifest_path = os.path.join(
223 self.manifest.manifestProject.worktree, smart_sync_manifest_name) 210 self.manifest.manifestProject.worktree, smart_sync_manifest_name)
224 211
225 if os.path.isfile(smart_sync_manifest_path): 212 if os.path.isfile(smart_sync_manifest_path):
226 self.manifest.Override(smart_sync_manifest_path) 213 self.manifest.Override(smart_sync_manifest_path)
@@ -234,58 +221,50 @@ without iterating through the remaining projects.
234 221
235 os.environ['REPO_COUNT'] = str(len(projects)) 222 os.environ['REPO_COUNT'] = str(len(projects))
236 223
237 pool = multiprocessing.Pool(opt.jobs, InitWorker)
238 try: 224 try:
239 config = self.manifest.manifestProject.config 225 config = self.manifest.manifestProject.config
240 results_it = pool.imap( 226 with multiprocessing.Pool(opt.jobs, InitWorker) as pool:
241 DoWorkWrapper, 227 results_it = pool.imap(
242 self.ProjectArgs(projects, mirror, opt, cmd, shell, config)) 228 functools.partial(DoWorkWrapper, mirror, opt, cmd, shell, config),
243 pool.close() 229 enumerate(projects),
244 for r in results_it: 230 chunksize=WORKER_BATCH_SIZE)
245 rc = rc or r 231 first = True
246 if r != 0 and opt.abort_on_errors: 232 for (r, output) in results_it:
247 raise Exception('Aborting due to previous error') 233 if output:
234 if first:
235 first = False
236 elif opt.project_header:
237 print()
238 # To simplify the DoWorkWrapper, take care of automatic newlines.
239 end = '\n'
240 if output[-1] == '\n':
241 end = ''
242 print(output, end=end)
243 rc = rc or r
244 if r != 0 and opt.abort_on_errors:
245 raise Exception('Aborting due to previous error')
248 except (KeyboardInterrupt, WorkerKeyboardInterrupt): 246 except (KeyboardInterrupt, WorkerKeyboardInterrupt):
249 # Catch KeyboardInterrupt raised inside and outside of workers 247 # Catch KeyboardInterrupt raised inside and outside of workers
250 print('Interrupted - terminating the pool')
251 pool.terminate()
252 rc = rc or errno.EINTR 248 rc = rc or errno.EINTR
253 except Exception as e: 249 except Exception as e:
254 # Catch any other exceptions raised 250 # Catch any other exceptions raised
255 print('Got an error, terminating the pool: %s: %s' % 251 print('forall: unhandled error, terminating the pool: %s: %s' %
256 (type(e).__name__, e), 252 (type(e).__name__, e),
257 file=sys.stderr) 253 file=sys.stderr)
258 pool.terminate()
259 rc = rc or getattr(e, 'errno', 1) 254 rc = rc or getattr(e, 'errno', 1)
260 finally:
261 pool.join()
262 if rc != 0: 255 if rc != 0:
263 sys.exit(rc) 256 sys.exit(rc)
264 257
265 def ProjectArgs(self, projects, mirror, opt, cmd, shell, config):
266 for cnt, p in enumerate(projects):
267 try:
268 project = self._SerializeProject(p)
269 except Exception as e:
270 print('Project list error on project %s: %s: %s' %
271 (p.name, type(e).__name__, e),
272 file=sys.stderr)
273 return
274 except KeyboardInterrupt:
275 print('Project list interrupted',
276 file=sys.stderr)
277 return
278 yield [mirror, opt, cmd, shell, cnt, config, project]
279 258
280class WorkerKeyboardInterrupt(Exception): 259class WorkerKeyboardInterrupt(Exception):
281 """ Keyboard interrupt exception for worker processes. """ 260 """ Keyboard interrupt exception for worker processes. """
282 pass
283 261
284 262
285def InitWorker(): 263def InitWorker():
286 signal.signal(signal.SIGINT, signal.SIG_IGN) 264 signal.signal(signal.SIGINT, signal.SIG_IGN)
287 265
288def DoWorkWrapper(args): 266
267def DoWorkWrapper(mirror, opt, cmd, shell, config, args):
289 """ A wrapper around the DoWork() method. 268 """ A wrapper around the DoWork() method.
290 269
291 Catch the KeyboardInterrupt exceptions here and re-raise them as a different, 270 Catch the KeyboardInterrupt exceptions here and re-raise them as a different,
@@ -293,109 +272,81 @@ def DoWorkWrapper(args):
293 and making the parent hang indefinitely. 272 and making the parent hang indefinitely.
294 273
295 """ 274 """
296 project = args.pop() 275 cnt, project = args
297 try: 276 try:
298 return DoWork(project, *args) 277 return DoWork(project, mirror, opt, cmd, shell, cnt, config)
299 except KeyboardInterrupt: 278 except KeyboardInterrupt:
300 print('%s: Worker interrupted' % project['name']) 279 print('%s: Worker interrupted' % project.name)
301 raise WorkerKeyboardInterrupt() 280 raise WorkerKeyboardInterrupt()
302 281
303 282
304def DoWork(project, mirror, opt, cmd, shell, cnt, config): 283def DoWork(project, mirror, opt, cmd, shell, cnt, config):
305 env = os.environ.copy() 284 env = os.environ.copy()
285
306 def setenv(name, val): 286 def setenv(name, val):
307 if val is None: 287 if val is None:
308 val = '' 288 val = ''
309 if hasattr(val, 'encode'):
310 val = val.encode()
311 env[name] = val 289 env[name] = val
312 290
313 setenv('REPO_PROJECT', project['name']) 291 setenv('REPO_PROJECT', project.name)
314 setenv('REPO_PATH', project['relpath']) 292 setenv('REPO_PATH', project.relpath)
315 setenv('REPO_REMOTE', project['remote_name']) 293 setenv('REPO_REMOTE', project.remote.name)
316 setenv('REPO_LREV', project['lrev']) 294 try:
317 setenv('REPO_RREV', project['rrev']) 295 # If we aren't in a fully synced state and we don't have the ref the manifest
296 # wants, then this will fail. Ignore it for the purposes of this code.
297 lrev = '' if mirror else project.GetRevisionId()
298 except ManifestInvalidRevisionError:
299 lrev = ''
300 setenv('REPO_LREV', lrev)
301 setenv('REPO_RREV', project.revisionExpr)
302 setenv('REPO_UPSTREAM', project.upstream)
303 setenv('REPO_DEST_BRANCH', project.dest_branch)
318 setenv('REPO_I', str(cnt + 1)) 304 setenv('REPO_I', str(cnt + 1))
319 for name in project['annotations']: 305 for annotation in project.annotations:
320 setenv("REPO__%s" % (name), project['annotations'][name]) 306 setenv("REPO__%s" % (annotation.name), annotation.value)
321 307
322 if mirror: 308 if mirror:
323 setenv('GIT_DIR', project['gitdir']) 309 setenv('GIT_DIR', project.gitdir)
324 cwd = project['gitdir'] 310 cwd = project.gitdir
325 else: 311 else:
326 cwd = project['worktree'] 312 cwd = project.worktree
327 313
328 if not os.path.exists(cwd): 314 if not os.path.exists(cwd):
329 # Allow the user to silently ignore missing checkouts so they can run on 315 # Allow the user to silently ignore missing checkouts so they can run on
330 # partial checkouts (good for infra recovery tools). 316 # partial checkouts (good for infra recovery tools).
331 if opt.ignore_missing: 317 if opt.ignore_missing:
332 return 0 318 return (0, '')
319
320 output = ''
333 if ((opt.project_header and opt.verbose) 321 if ((opt.project_header and opt.verbose)
334 or not opt.project_header): 322 or not opt.project_header):
335 print('skipping %s/' % project['relpath'], file=sys.stderr) 323 output = 'skipping %s/' % project.relpath
336 return 1 324 return (1, output)
337 325
338 if opt.project_header: 326 if opt.verbose:
339 stdin = subprocess.PIPE 327 stderr = subprocess.STDOUT
340 stdout = subprocess.PIPE
341 stderr = subprocess.PIPE
342 else: 328 else:
343 stdin = None 329 stderr = subprocess.DEVNULL
344 stdout = None 330
345 stderr = None 331 stdin = None if opt.interactive else subprocess.DEVNULL
346
347 p = subprocess.Popen(cmd,
348 cwd=cwd,
349 shell=shell,
350 env=env,
351 stdin=stdin,
352 stdout=stdout,
353 stderr=stderr)
354 332
333 result = subprocess.run(
334 cmd, cwd=cwd, shell=shell, env=env, check=False,
335 encoding='utf-8', errors='replace',
336 stdin=stdin, stdout=subprocess.PIPE, stderr=stderr)
337
338 output = result.stdout
355 if opt.project_header: 339 if opt.project_header:
356 out = ForallColoring(config) 340 if output:
357 out.redirect(sys.stdout) 341 buf = io.StringIO()
358 empty = True 342 out = ForallColoring(config)
359 errbuf = '' 343 out.redirect(buf)
360 344 if mirror:
361 p.stdin.close() 345 project_header_path = project.name
362 s_in = platform_utils.FileDescriptorStreams.create() 346 else:
363 s_in.add(p.stdout, sys.stdout, 'stdout') 347 project_header_path = project.relpath
364 s_in.add(p.stderr, sys.stderr, 'stderr') 348 out.project('project %s/' % project_header_path)
365 349 out.nl()
366 while not s_in.is_done: 350 buf.write(output)
367 in_ready = s_in.select() 351 output = buf.getvalue()
368 for s in in_ready: 352 return (result.returncode, output)
369 buf = s.read().decode()
370 if not buf:
371 s.close()
372 s_in.remove(s)
373 continue
374
375 if not opt.verbose:
376 if s.std_name == 'stderr':
377 errbuf += buf
378 continue
379
380 if empty and out:
381 if not cnt == 0:
382 out.nl()
383
384 if mirror:
385 project_header_path = project['name']
386 else:
387 project_header_path = project['relpath']
388 out.project('project %s/', project_header_path)
389 out.nl()
390 out.flush()
391 if errbuf:
392 sys.stderr.write(errbuf)
393 sys.stderr.flush()
394 errbuf = ''
395 empty = False
396
397 s.dest.write(buf)
398 s.dest.flush()
399
400 r = p.wait()
401 return r
diff --git a/subcmds/gitc_delete.py b/subcmds/gitc_delete.py
index e5214b8e..df749469 100644
--- a/subcmds/gitc_delete.py
+++ b/subcmds/gitc_delete.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2015 The Android Open Source Project 1# Copyright (C) 2015 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,18 +12,14 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function
18import sys 15import sys
19 16
20from command import Command, GitcClientCommand 17from command import Command, GitcClientCommand
21import platform_utils 18import platform_utils
22 19
23from pyversion import is_python3
24if not is_python3():
25 input = raw_input
26 20
27class GitcDelete(Command, GitcClientCommand): 21class GitcDelete(Command, GitcClientCommand):
28 common = True 22 COMMON = True
29 visible_everywhere = False 23 visible_everywhere = False
30 helpSummary = "Delete a GITC Client." 24 helpSummary = "Delete a GITC Client."
31 helpUsage = """ 25 helpUsage = """
@@ -39,7 +33,7 @@ and all locally downloaded sources.
39 def _Options(self, p): 33 def _Options(self, p):
40 p.add_option('-f', '--force', 34 p.add_option('-f', '--force',
41 dest='force', action='store_true', 35 dest='force', action='store_true',
42 help='Force the deletion (no prompt).') 36 help='force the deletion (no prompt)')
43 37
44 def Execute(self, opt, args): 38 def Execute(self, opt, args):
45 if not opt.force: 39 if not opt.force:
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py
index 378f9236..e705b613 100644
--- a/subcmds/gitc_init.py
+++ b/subcmds/gitc_init.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2015 The Android Open Source Project 1# Copyright (C) 2015 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +12,6 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function
18import os 15import os
19import sys 16import sys
20 17
@@ -26,7 +23,7 @@ import wrapper
26 23
27 24
28class GitcInit(init.Init, GitcAvailableCommand): 25class GitcInit(init.Init, GitcAvailableCommand):
29 common = True 26 COMMON = True
30 helpSummary = "Initialize a GITC Client." 27 helpSummary = "Initialize a GITC Client."
31 helpUsage = """ 28 helpUsage = """
32%prog [options] [client name] 29%prog [options] [client name]
@@ -50,23 +47,17 @@ use for this GITC client.
50""" 47"""
51 48
52 def _Options(self, p): 49 def _Options(self, p):
53 super(GitcInit, self)._Options(p, gitc_init=True) 50 super()._Options(p, gitc_init=True)
54 g = p.add_option_group('GITC options')
55 g.add_option('-f', '--manifest-file',
56 dest='manifest_file',
57 help='Optional manifest file to use for this GITC client.')
58 g.add_option('-c', '--gitc-client',
59 dest='gitc_client',
60 help='The name of the gitc_client instance to create or modify.')
61 51
62 def Execute(self, opt, args): 52 def Execute(self, opt, args):
63 gitc_client = gitc_utils.parse_clientdir(os.getcwd()) 53 gitc_client = gitc_utils.parse_clientdir(os.getcwd())
64 if not gitc_client or (opt.gitc_client and gitc_client != opt.gitc_client): 54 if not gitc_client or (opt.gitc_client and gitc_client != opt.gitc_client):
65 print('fatal: Please update your repo command. See go/gitc for instructions.', file=sys.stderr) 55 print('fatal: Please update your repo command. See go/gitc for instructions.',
56 file=sys.stderr)
66 sys.exit(1) 57 sys.exit(1)
67 self.client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(), 58 self.client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
68 gitc_client) 59 gitc_client)
69 super(GitcInit, self).Execute(opt, args) 60 super().Execute(opt, args)
70 61
71 manifest_file = self.manifest.manifestFile 62 manifest_file = self.manifest.manifestFile
72 if opt.manifest_file: 63 if opt.manifest_file:
diff --git a/subcmds/grep.py b/subcmds/grep.py
index 4dd85d57..8ac4ba14 100644
--- a/subcmds/grep.py
+++ b/subcmds/grep.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2009 The Android Open Source Project 1# Copyright (C) 2009 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,14 +12,14 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function 15import functools
18
19import sys 16import sys
20 17
21from color import Coloring 18from color import Coloring
22from command import PagedCommand 19from command import DEFAULT_LOCAL_JOBS, PagedCommand
23from error import GitError 20from error import GitError
24from git_command import git_require, GitCommand 21from git_command import GitCommand
22
25 23
26class GrepColoring(Coloring): 24class GrepColoring(Coloring):
27 def __init__(self, config): 25 def __init__(self, config):
@@ -29,8 +27,9 @@ class GrepColoring(Coloring):
29 self.project = self.printer('project', attr='bold') 27 self.project = self.printer('project', attr='bold')
30 self.fail = self.printer('fail', fg='red') 28 self.fail = self.printer('fail', fg='red')
31 29
30
32class Grep(PagedCommand): 31class Grep(PagedCommand):
33 common = True 32 COMMON = True
34 helpSummary = "Print lines matching a pattern" 33 helpSummary = "Print lines matching a pattern"
35 helpUsage = """ 34 helpUsage = """
36%prog {pattern | -e pattern} [<project>...] 35%prog {pattern | -e pattern} [<project>...]
@@ -63,30 +62,33 @@ contain a line that matches both expressions:
63 repo grep --all-match -e NODE -e Unexpected 62 repo grep --all-match -e NODE -e Unexpected
64 63
65""" 64"""
65 PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
66
67 @staticmethod
68 def _carry_option(_option, opt_str, value, parser):
69 pt = getattr(parser.values, 'cmd_argv', None)
70 if pt is None:
71 pt = []
72 setattr(parser.values, 'cmd_argv', pt)
73
74 if opt_str == '-(':
75 pt.append('(')
76 elif opt_str == '-)':
77 pt.append(')')
78 else:
79 pt.append(opt_str)
66 80
67 def _Options(self, p): 81 if value is not None:
68 def carry(option, 82 pt.append(value)
69 opt_str,
70 value,
71 parser):
72 pt = getattr(parser.values, 'cmd_argv', None)
73 if pt is None:
74 pt = []
75 setattr(parser.values, 'cmd_argv', pt)
76
77 if opt_str == '-(':
78 pt.append('(')
79 elif opt_str == '-)':
80 pt.append(')')
81 else:
82 pt.append(opt_str)
83 83
84 if value is not None: 84 def _CommonOptions(self, p):
85 pt.append(value) 85 """Override common options slightly."""
86 super()._CommonOptions(p, opt_v=False)
86 87
88 def _Options(self, p):
87 g = p.add_option_group('Sources') 89 g = p.add_option_group('Sources')
88 g.add_option('--cached', 90 g.add_option('--cached',
89 action='callback', callback=carry, 91 action='callback', callback=self._carry_option,
90 help='Search the index, instead of the work tree') 92 help='Search the index, instead of the work tree')
91 g.add_option('-r', '--revision', 93 g.add_option('-r', '--revision',
92 dest='revision', action='append', metavar='TREEish', 94 dest='revision', action='append', metavar='TREEish',
@@ -94,136 +96,111 @@ contain a line that matches both expressions:
94 96
95 g = p.add_option_group('Pattern') 97 g = p.add_option_group('Pattern')
96 g.add_option('-e', 98 g.add_option('-e',
97 action='callback', callback=carry, 99 action='callback', callback=self._carry_option,
98 metavar='PATTERN', type='str', 100 metavar='PATTERN', type='str',
99 help='Pattern to search for') 101 help='Pattern to search for')
100 g.add_option('-i', '--ignore-case', 102 g.add_option('-i', '--ignore-case',
101 action='callback', callback=carry, 103 action='callback', callback=self._carry_option,
102 help='Ignore case differences') 104 help='Ignore case differences')
103 g.add_option('-a', '--text', 105 g.add_option('-a', '--text',
104 action='callback', callback=carry, 106 action='callback', callback=self._carry_option,
105 help="Process binary files as if they were text") 107 help="Process binary files as if they were text")
106 g.add_option('-I', 108 g.add_option('-I',
107 action='callback', callback=carry, 109 action='callback', callback=self._carry_option,
108 help="Don't match the pattern in binary files") 110 help="Don't match the pattern in binary files")
109 g.add_option('-w', '--word-regexp', 111 g.add_option('-w', '--word-regexp',
110 action='callback', callback=carry, 112 action='callback', callback=self._carry_option,
111 help='Match the pattern only at word boundaries') 113 help='Match the pattern only at word boundaries')
112 g.add_option('-v', '--invert-match', 114 g.add_option('-v', '--invert-match',
113 action='callback', callback=carry, 115 action='callback', callback=self._carry_option,
114 help='Select non-matching lines') 116 help='Select non-matching lines')
115 g.add_option('-G', '--basic-regexp', 117 g.add_option('-G', '--basic-regexp',
116 action='callback', callback=carry, 118 action='callback', callback=self._carry_option,
117 help='Use POSIX basic regexp for patterns (default)') 119 help='Use POSIX basic regexp for patterns (default)')
118 g.add_option('-E', '--extended-regexp', 120 g.add_option('-E', '--extended-regexp',
119 action='callback', callback=carry, 121 action='callback', callback=self._carry_option,
120 help='Use POSIX extended regexp for patterns') 122 help='Use POSIX extended regexp for patterns')
121 g.add_option('-F', '--fixed-strings', 123 g.add_option('-F', '--fixed-strings',
122 action='callback', callback=carry, 124 action='callback', callback=self._carry_option,
123 help='Use fixed strings (not regexp) for pattern') 125 help='Use fixed strings (not regexp) for pattern')
124 126
125 g = p.add_option_group('Pattern Grouping') 127 g = p.add_option_group('Pattern Grouping')
126 g.add_option('--all-match', 128 g.add_option('--all-match',
127 action='callback', callback=carry, 129 action='callback', callback=self._carry_option,
128 help='Limit match to lines that have all patterns') 130 help='Limit match to lines that have all patterns')
129 g.add_option('--and', '--or', '--not', 131 g.add_option('--and', '--or', '--not',
130 action='callback', callback=carry, 132 action='callback', callback=self._carry_option,
131 help='Boolean operators to combine patterns') 133 help='Boolean operators to combine patterns')
132 g.add_option('-(', '-)', 134 g.add_option('-(', '-)',
133 action='callback', callback=carry, 135 action='callback', callback=self._carry_option,
134 help='Boolean operator grouping') 136 help='Boolean operator grouping')
135 137
136 g = p.add_option_group('Output') 138 g = p.add_option_group('Output')
137 g.add_option('-n', 139 g.add_option('-n',
138 action='callback', callback=carry, 140 action='callback', callback=self._carry_option,
139 help='Prefix the line number to matching lines') 141 help='Prefix the line number to matching lines')
140 g.add_option('-C', 142 g.add_option('-C',
141 action='callback', callback=carry, 143 action='callback', callback=self._carry_option,
142 metavar='CONTEXT', type='str', 144 metavar='CONTEXT', type='str',
143 help='Show CONTEXT lines around match') 145 help='Show CONTEXT lines around match')
144 g.add_option('-B', 146 g.add_option('-B',
145 action='callback', callback=carry, 147 action='callback', callback=self._carry_option,
146 metavar='CONTEXT', type='str', 148 metavar='CONTEXT', type='str',
147 help='Show CONTEXT lines before match') 149 help='Show CONTEXT lines before match')
148 g.add_option('-A', 150 g.add_option('-A',
149 action='callback', callback=carry, 151 action='callback', callback=self._carry_option,
150 metavar='CONTEXT', type='str', 152 metavar='CONTEXT', type='str',
151 help='Show CONTEXT lines after match') 153 help='Show CONTEXT lines after match')
152 g.add_option('-l', '--name-only', '--files-with-matches', 154 g.add_option('-l', '--name-only', '--files-with-matches',
153 action='callback', callback=carry, 155 action='callback', callback=self._carry_option,
154 help='Show only file names containing matching lines') 156 help='Show only file names containing matching lines')
155 g.add_option('-L', '--files-without-match', 157 g.add_option('-L', '--files-without-match',
156 action='callback', callback=carry, 158 action='callback', callback=self._carry_option,
157 help='Show only file names not containing matching lines') 159 help='Show only file names not containing matching lines')
158 160
159 161 def _ExecuteOne(self, cmd_argv, project):
160 def Execute(self, opt, args): 162 """Process one project."""
161 out = GrepColoring(self.manifest.manifestProject.config) 163 try:
162 164 p = GitCommand(project,
163 cmd_argv = ['grep'] 165 cmd_argv,
164 if out.is_on and git_require((1, 6, 3)): 166 bare=False,
165 cmd_argv.append('--color') 167 capture_stdout=True,
166 cmd_argv.extend(getattr(opt, 'cmd_argv', [])) 168 capture_stderr=True)
167 169 except GitError as e:
168 if '-e' not in cmd_argv: 170 return (project, -1, None, str(e))
169 if not args: 171
170 self.Usage() 172 return (project, p.Wait(), p.stdout, p.stderr)
171 cmd_argv.append('-e') 173
172 cmd_argv.append(args[0]) 174 @staticmethod
173 args = args[1:] 175 def _ProcessResults(full_name, have_rev, _pool, out, results):
174
175 projects = self.GetProjects(args)
176
177 full_name = False
178 if len(projects) > 1:
179 cmd_argv.append('--full-name')
180 full_name = True
181
182 have_rev = False
183 if opt.revision:
184 if '--cached' in cmd_argv:
185 print('fatal: cannot combine --cached and --revision', file=sys.stderr)
186 sys.exit(1)
187 have_rev = True
188 cmd_argv.extend(opt.revision)
189 cmd_argv.append('--')
190
191 git_failed = False 176 git_failed = False
192 bad_rev = False 177 bad_rev = False
193 have_match = False 178 have_match = False
194 179
195 for project in projects: 180 for project, rc, stdout, stderr in results:
196 try: 181 if rc < 0:
197 p = GitCommand(project,
198 cmd_argv,
199 bare=False,
200 capture_stdout=True,
201 capture_stderr=True)
202 except GitError as e:
203 git_failed = True 182 git_failed = True
204 out.project('--- project %s ---' % project.relpath) 183 out.project('--- project %s ---' % project.relpath)
205 out.nl() 184 out.nl()
206 out.fail('%s', str(e)) 185 out.fail('%s', stderr)
207 out.nl() 186 out.nl()
208 continue 187 continue
209 188
210 if p.Wait() != 0: 189 if rc:
211 # no results 190 # no results
212 # 191 if stderr:
213 if p.stderr: 192 if have_rev and 'fatal: ambiguous argument' in stderr:
214 if have_rev and 'fatal: ambiguous argument' in p.stderr:
215 bad_rev = True 193 bad_rev = True
216 else: 194 else:
217 out.project('--- project %s ---' % project.relpath) 195 out.project('--- project %s ---' % project.relpath)
218 out.nl() 196 out.nl()
219 out.fail('%s', p.stderr.strip()) 197 out.fail('%s', stderr.strip())
220 out.nl() 198 out.nl()
221 continue 199 continue
222 have_match = True 200 have_match = True
223 201
224 # We cut the last element, to avoid a blank line. 202 # We cut the last element, to avoid a blank line.
225 # 203 r = stdout.split('\n')
226 r = p.stdout.split('\n')
227 r = r[0:-1] 204 r = r[0:-1]
228 205
229 if have_rev and full_name: 206 if have_rev and full_name:
@@ -245,6 +222,47 @@ contain a line that matches both expressions:
245 for line in r: 222 for line in r:
246 print(line) 223 print(line)
247 224
225 return (git_failed, bad_rev, have_match)
226
227 def Execute(self, opt, args):
228 out = GrepColoring(self.manifest.manifestProject.config)
229
230 cmd_argv = ['grep']
231 if out.is_on:
232 cmd_argv.append('--color')
233 cmd_argv.extend(getattr(opt, 'cmd_argv', []))
234
235 if '-e' not in cmd_argv:
236 if not args:
237 self.Usage()
238 cmd_argv.append('-e')
239 cmd_argv.append(args[0])
240 args = args[1:]
241
242 projects = self.GetProjects(args)
243
244 full_name = False
245 if len(projects) > 1:
246 cmd_argv.append('--full-name')
247 full_name = True
248
249 have_rev = False
250 if opt.revision:
251 if '--cached' in cmd_argv:
252 print('fatal: cannot combine --cached and --revision', file=sys.stderr)
253 sys.exit(1)
254 have_rev = True
255 cmd_argv.extend(opt.revision)
256 cmd_argv.append('--')
257
258 git_failed, bad_rev, have_match = self.ExecuteInParallel(
259 opt.jobs,
260 functools.partial(self._ExecuteOne, cmd_argv),
261 projects,
262 callback=functools.partial(self._ProcessResults, full_name, have_rev),
263 output=out,
264 ordered=True)
265
248 if git_failed: 266 if git_failed:
249 sys.exit(1) 267 sys.exit(1)
250 elif have_match: 268 elif have_match:
diff --git a/subcmds/help.py b/subcmds/help.py
index 78930502..1a60ef45 100644
--- a/subcmds/help.py
+++ b/subcmds/help.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,17 +12,19 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function
18import re 15import re
19import sys 16import sys
20from formatter import AbstractFormatter, DumbWriter 17import textwrap
21 18
19from subcmds import all_commands
22from color import Coloring 20from color import Coloring
23from command import PagedCommand, MirrorSafeCommand, GitcAvailableCommand, GitcClientCommand 21from command import PagedCommand, MirrorSafeCommand, GitcAvailableCommand, GitcClientCommand
24import gitc_utils 22import gitc_utils
23from wrapper import Wrapper
24
25 25
26class Help(PagedCommand, MirrorSafeCommand): 26class Help(PagedCommand, MirrorSafeCommand):
27 common = False 27 COMMON = False
28 helpSummary = "Display detailed help on a command" 28 helpSummary = "Display detailed help on a command"
29 helpUsage = """ 29 helpUsage = """
30%prog [--all|command] 30%prog [--all|command]
@@ -41,7 +41,7 @@ Displays detailed usage information about a command.
41 fmt = ' %%-%ds %%s' % maxlen 41 fmt = ' %%-%ds %%s' % maxlen
42 42
43 for name in commandNames: 43 for name in commandNames:
44 command = self.commands[name] 44 command = all_commands[name]()
45 try: 45 try:
46 summary = command.helpSummary.strip() 46 summary = command.helpSummary.strip()
47 except AttributeError: 47 except AttributeError:
@@ -50,20 +50,27 @@ Displays detailed usage information about a command.
50 50
51 def _PrintAllCommands(self): 51 def _PrintAllCommands(self):
52 print('usage: repo COMMAND [ARGS]') 52 print('usage: repo COMMAND [ARGS]')
53 self.PrintAllCommandsBody()
54
55 def PrintAllCommandsBody(self):
53 print('The complete list of recognized repo commands are:') 56 print('The complete list of recognized repo commands are:')
54 commandNames = list(sorted(self.commands)) 57 commandNames = list(sorted(all_commands))
55 self._PrintCommands(commandNames) 58 self._PrintCommands(commandNames)
56 print("See 'repo help <command>' for more information on a " 59 print("See 'repo help <command>' for more information on a "
57 'specific command.') 60 'specific command.')
61 print('Bug reports:', Wrapper().BUG_URL)
58 62
59 def _PrintCommonCommands(self): 63 def _PrintCommonCommands(self):
60 print('usage: repo COMMAND [ARGS]') 64 print('usage: repo COMMAND [ARGS]')
65 self.PrintCommonCommandsBody()
66
67 def PrintCommonCommandsBody(self):
61 print('The most commonly used repo commands are:') 68 print('The most commonly used repo commands are:')
62 69
63 def gitc_supported(cmd): 70 def gitc_supported(cmd):
64 if not isinstance(cmd, GitcAvailableCommand) and not isinstance(cmd, GitcClientCommand): 71 if not isinstance(cmd, GitcAvailableCommand) and not isinstance(cmd, GitcClientCommand):
65 return True 72 return True
66 if self.manifest.isGitcClient: 73 if self.client.isGitcClient:
67 return True 74 return True
68 if isinstance(cmd, GitcClientCommand): 75 if isinstance(cmd, GitcClientCommand):
69 return False 76 return False
@@ -72,21 +79,21 @@ Displays detailed usage information about a command.
72 return False 79 return False
73 80
74 commandNames = list(sorted([name 81 commandNames = list(sorted([name
75 for name, command in self.commands.items() 82 for name, command in all_commands.items()
76 if command.common and gitc_supported(command)])) 83 if command.COMMON and gitc_supported(command)]))
77 self._PrintCommands(commandNames) 84 self._PrintCommands(commandNames)
78 85
79 print( 86 print(
80"See 'repo help <command>' for more information on a specific command.\n" 87 "See 'repo help <command>' for more information on a specific command.\n"
81"See 'repo help --all' for a complete list of recognized commands.") 88 "See 'repo help --all' for a complete list of recognized commands.")
89 print('Bug reports:', Wrapper().BUG_URL)
82 90
83 def _PrintCommandHelp(self, cmd, header_prefix=''): 91 def _PrintCommandHelp(self, cmd, header_prefix=''):
84 class _Out(Coloring): 92 class _Out(Coloring):
85 def __init__(self, gc): 93 def __init__(self, gc):
86 Coloring.__init__(self, gc, 'help') 94 Coloring.__init__(self, gc, 'help')
87 self.heading = self.printer('heading', attr='bold') 95 self.heading = self.printer('heading', attr='bold')
88 96 self._first = True
89 self.wrap = AbstractFormatter(DumbWriter())
90 97
91 def _PrintSection(self, heading, bodyAttr): 98 def _PrintSection(self, heading, bodyAttr):
92 try: 99 try:
@@ -96,7 +103,9 @@ Displays detailed usage information about a command.
96 if body == '' or body is None: 103 if body == '' or body is None:
97 return 104 return
98 105
99 self.nl() 106 if not self._first:
107 self.nl()
108 self._first = False
100 109
101 self.heading('%s%s', header_prefix, heading) 110 self.heading('%s%s', header_prefix, heading)
102 self.nl() 111 self.nl()
@@ -106,7 +115,8 @@ Displays detailed usage information about a command.
106 body = body.strip() 115 body = body.strip()
107 body = body.replace('%prog', me) 116 body = body.replace('%prog', me)
108 117
109 asciidoc_hdr = re.compile(r'^\n?#+ (.+)$') 118 # Extract the title, but skip any trailing {#anchors}.
119 asciidoc_hdr = re.compile(r'^\n?#+ ([^{]+)(\{#.+\})?$')
110 for para in body.split("\n\n"): 120 for para in body.split("\n\n"):
111 if para.startswith(' '): 121 if para.startswith(' '):
112 self.write('%s', para) 122 self.write('%s', para)
@@ -121,19 +131,21 @@ Displays detailed usage information about a command.
121 self.nl() 131 self.nl()
122 continue 132 continue
123 133
124 self.wrap.add_flowing_data(para) 134 lines = textwrap.wrap(para.replace(' ', ' '), width=80,
125 self.wrap.end_paragraph(1) 135 break_long_words=False, break_on_hyphens=False)
126 self.wrap.end_paragraph(0) 136 for line in lines:
137 self.write('%s', line)
138 self.nl()
139 self.nl()
127 140
128 out = _Out(self.manifest.globalConfig) 141 out = _Out(self.client.globalConfig)
129 out._PrintSection('Summary', 'helpSummary') 142 out._PrintSection('Summary', 'helpSummary')
130 cmd.OptionParser.print_help() 143 cmd.OptionParser.print_help()
131 out._PrintSection('Description', 'helpDescription') 144 out._PrintSection('Description', 'helpDescription')
132 145
133 def _PrintAllCommandHelp(self): 146 def _PrintAllCommandHelp(self):
134 for name in sorted(self.commands): 147 for name in sorted(all_commands):
135 cmd = self.commands[name] 148 cmd = all_commands[name](manifest=self.manifest)
136 cmd.manifest = self.manifest
137 self._PrintCommandHelp(cmd, header_prefix='[%s] ' % (name,)) 149 self._PrintCommandHelp(cmd, header_prefix='[%s] ' % (name,))
138 150
139 def _Options(self, p): 151 def _Options(self, p):
@@ -157,12 +169,11 @@ Displays detailed usage information about a command.
157 name = args[0] 169 name = args[0]
158 170
159 try: 171 try:
160 cmd = self.commands[name] 172 cmd = all_commands[name](manifest=self.manifest)
161 except KeyError: 173 except KeyError:
162 print("repo: '%s' is not a repo command." % name, file=sys.stderr) 174 print("repo: '%s' is not a repo command." % name, file=sys.stderr)
163 sys.exit(1) 175 sys.exit(1)
164 176
165 cmd.manifest = self.manifest
166 self._PrintCommandHelp(cmd) 177 self._PrintCommandHelp(cmd)
167 178
168 else: 179 else:
diff --git a/subcmds/info.py b/subcmds/info.py
index d62e1e64..6c1246ef 100644
--- a/subcmds/info.py
+++ b/subcmds/info.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2012 The Android Open Source Project 1# Copyright (C) 2012 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,18 +12,22 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
15import optparse
16
17from command import PagedCommand 17from command import PagedCommand
18from color import Coloring 18from color import Coloring
19from git_refs import R_M 19from git_refs import R_M, R_HEADS
20
20 21
21class _Coloring(Coloring): 22class _Coloring(Coloring):
22 def __init__(self, config): 23 def __init__(self, config):
23 Coloring.__init__(self, config, "status") 24 Coloring.__init__(self, config, "status")
24 25
26
25class Info(PagedCommand): 27class Info(PagedCommand):
26 common = True 28 COMMON = True
27 helpSummary = "Get info on the manifest branch, current branch or unmerged branches" 29 helpSummary = "Get info on the manifest branch, current branch or unmerged branches"
28 helpUsage = "%prog [-dl] [-o [-b]] [<project>...]" 30 helpUsage = "%prog [-dl] [-o [-c]] [<project>...]"
29 31
30 def _Options(self, p): 32 def _Options(self, p):
31 p.add_option('-d', '--diff', 33 p.add_option('-d', '--diff',
@@ -34,22 +36,28 @@ class Info(PagedCommand):
34 p.add_option('-o', '--overview', 36 p.add_option('-o', '--overview',
35 dest='overview', action='store_true', 37 dest='overview', action='store_true',
36 help='show overview of all local commits') 38 help='show overview of all local commits')
37 p.add_option('-b', '--current-branch', 39 p.add_option('-c', '--current-branch',
38 dest="current_branch", action="store_true", 40 dest="current_branch", action="store_true",
39 help="consider only checked out branches") 41 help="consider only checked out branches")
42 p.add_option('--no-current-branch',
43 dest='current_branch', action='store_false',
44 help='consider all local branches')
45 # Turn this into a warning & remove this someday.
46 p.add_option('-b',
47 dest='current_branch', action='store_true',
48 help=optparse.SUPPRESS_HELP)
40 p.add_option('-l', '--local-only', 49 p.add_option('-l', '--local-only',
41 dest="local", action="store_true", 50 dest="local", action="store_true",
42 help="Disable all remote operations") 51 help="disable all remote operations")
43
44 52
45 def Execute(self, opt, args): 53 def Execute(self, opt, args):
46 self.out = _Coloring(self.manifest.globalConfig) 54 self.out = _Coloring(self.client.globalConfig)
47 self.heading = self.out.printer('heading', attr = 'bold') 55 self.heading = self.out.printer('heading', attr='bold')
48 self.headtext = self.out.nofmt_printer('headtext', fg = 'yellow') 56 self.headtext = self.out.nofmt_printer('headtext', fg='yellow')
49 self.redtext = self.out.printer('redtext', fg = 'red') 57 self.redtext = self.out.printer('redtext', fg='red')
50 self.sha = self.out.printer("sha", fg = 'yellow') 58 self.sha = self.out.printer("sha", fg='yellow')
51 self.text = self.out.nofmt_printer('text') 59 self.text = self.out.nofmt_printer('text')
52 self.dimtext = self.out.printer('dimtext', attr = 'dim') 60 self.dimtext = self.out.printer('dimtext', attr='dim')
53 61
54 self.opt = opt 62 self.opt = opt
55 63
@@ -122,11 +130,14 @@ class Info(PagedCommand):
122 self.printSeparator() 130 self.printSeparator()
123 131
124 def findRemoteLocalDiff(self, project): 132 def findRemoteLocalDiff(self, project):
125 #Fetch all the latest commits 133 # Fetch all the latest commits.
126 if not self.opt.local: 134 if not self.opt.local:
127 project.Sync_NetworkHalf(quiet=True, current_branch_only=True) 135 project.Sync_NetworkHalf(quiet=True, current_branch_only=True)
128 136
129 logTarget = R_M + self.manifest.manifestProject.config.GetBranch("default").merge 137 branch = self.manifest.manifestProject.config.GetBranch('default').merge
138 if branch.startswith(R_HEADS):
139 branch = branch[len(R_HEADS):]
140 logTarget = R_M + branch
130 141
131 bareTmp = project.bare_git._bare 142 bareTmp = project.bare_git._bare
132 project.bare_git._bare = False 143 project.bare_git._bare = False
@@ -195,16 +206,16 @@ class Info(PagedCommand):
195 commits = branch.commits 206 commits = branch.commits
196 date = branch.date 207 date = branch.date
197 self.text('%s %-33s (%2d commit%s, %s)' % ( 208 self.text('%s %-33s (%2d commit%s, %s)' % (
198 branch.name == project.CurrentBranch and '*' or ' ', 209 branch.name == project.CurrentBranch and '*' or ' ',
199 branch.name, 210 branch.name,
200 len(commits), 211 len(commits),
201 len(commits) != 1 and 's' or '', 212 len(commits) != 1 and 's' or '',
202 date)) 213 date))
203 self.out.nl() 214 self.out.nl()
204 215
205 for commit in commits: 216 for commit in commits:
206 split = commit.split() 217 split = commit.split()
207 self.text('{0:38}{1} '.format('','-')) 218 self.text('{0:38}{1} '.format('', '-'))
208 self.sha(split[0] + " ") 219 self.sha(split[0] + " ")
209 self.text(" ".join(split[1:])) 220 self.text(" ".join(split[1:]))
210 self.out.nl() 221 self.out.nl()
diff --git a/subcmds/init.py b/subcmds/init.py
index 6594a602..9c6b2ad9 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,34 +12,30 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function
18import os 15import os
19import platform 16import platform
20import re 17import re
18import subprocess
21import sys 19import sys
22 20import urllib.parse
23from pyversion import is_python3
24if is_python3():
25 import urllib.parse
26else:
27 import imp
28 import urlparse
29 urllib = imp.new_module('urllib')
30 urllib.parse = urlparse
31 21
32from color import Coloring 22from color import Coloring
33from command import InteractiveCommand, MirrorSafeCommand 23from command import InteractiveCommand, MirrorSafeCommand
34from error import ManifestParseError 24from error import ManifestParseError
35from project import SyncBuffer 25from project import SyncBuffer
36from git_config import GitConfig 26from git_config import GitConfig
37from git_command import git_require, MIN_GIT_VERSION 27from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
28import fetch
29import git_superproject
38import platform_utils 30import platform_utils
31from wrapper import Wrapper
32
39 33
40class Init(InteractiveCommand, MirrorSafeCommand): 34class Init(InteractiveCommand, MirrorSafeCommand):
41 common = True 35 COMMON = True
42 helpSummary = "Initialize repo in the current directory" 36 helpSummary = "Initialize a repo client checkout in the current directory"
43 helpUsage = """ 37 helpUsage = """
44%prog [options] 38%prog [options] [manifest url]
45""" 39"""
46 helpDescription = """ 40 helpDescription = """
47The '%prog' command is run once to install and initialize repo. 41The '%prog' command is run once to install and initialize repo.
@@ -49,13 +43,24 @@ The latest repo source code and manifest collection is downloaded
49from the server and is installed in the .repo/ directory in the 43from the server and is installed in the .repo/ directory in the
50current working directory. 44current working directory.
51 45
46When creating a new checkout, the manifest URL is the only required setting.
47It may be specified using the --manifest-url option, or as the first optional
48argument.
49
52The optional -b argument can be used to select the manifest branch 50The optional -b argument can be used to select the manifest branch
53to checkout and use. If no branch is specified, master is assumed. 51to checkout and use. If no branch is specified, the remote's default
52branch is used. This is equivalent to using -b HEAD.
54 53
55The optional -m argument can be used to specify an alternate manifest 54The optional -m argument can be used to specify an alternate manifest
56to be used. If no manifest is specified, the manifest default.xml 55to be used. If no manifest is specified, the manifest default.xml
57will be used. 56will be used.
58 57
58If the --standalone-manifest argument is set, the manifest will be downloaded
59directly from the specified --manifest-url as a static file (rather than
60setting up a manifest git checkout). With --standalone-manifest, the manifest
61will be fully static and will not be re-downloaded during subsesquent
62`repo init` and `repo sync` calls.
63
59The --reference option can be used to point to a directory that 64The --reference option can be used to point to a directory that
60has the content of a --mirror sync. This will make the working 65has the content of a --mirror sync. This will make the working
61directory use as much data as possible from the local reference 66directory use as much data as possible from the local reference
@@ -81,109 +86,64 @@ manifest, a subsequent `repo sync` (or `repo sync -d`) is necessary
81to update the working directory files. 86to update the working directory files.
82""" 87"""
83 88
89 def _CommonOptions(self, p):
90 """Disable due to re-use of Wrapper()."""
91
84 def _Options(self, p, gitc_init=False): 92 def _Options(self, p, gitc_init=False):
85 # Logging 93 Wrapper().InitParser(p, gitc_init=gitc_init)
86 g = p.add_option_group('Logging options')
87 g.add_option('-q', '--quiet',
88 dest="quiet", action="store_true", default=False,
89 help="be quiet")
90
91 # Manifest
92 g = p.add_option_group('Manifest options')
93 g.add_option('-u', '--manifest-url',
94 dest='manifest_url',
95 help='manifest repository location', metavar='URL')
96 g.add_option('-b', '--manifest-branch',
97 dest='manifest_branch',
98 help='manifest branch or revision', metavar='REVISION')
99 cbr_opts = ['--current-branch']
100 # The gitc-init subcommand allocates -c itself, but a lot of init users
101 # want -c, so try to satisfy both as best we can.
102 if not gitc_init:
103 cbr_opts += ['-c']
104 g.add_option(*cbr_opts,
105 dest='current_branch_only', action='store_true',
106 help='fetch only current manifest branch from server')
107 g.add_option('-m', '--manifest-name',
108 dest='manifest_name', default='default.xml',
109 help='initial manifest file', metavar='NAME.xml')
110 g.add_option('--mirror',
111 dest='mirror', action='store_true',
112 help='create a replica of the remote repositories '
113 'rather than a client working directory')
114 g.add_option('--reference',
115 dest='reference',
116 help='location of mirror directory', metavar='DIR')
117 g.add_option('--dissociate',
118 dest='dissociate', action='store_true',
119 help='dissociate from reference mirrors after clone')
120 g.add_option('--depth', type='int', default=None,
121 dest='depth',
122 help='create a shallow clone with given depth; see git clone')
123 g.add_option('--partial-clone', action='store_true',
124 dest='partial_clone',
125 help='perform partial clone (https://git-scm.com/'
126 'docs/gitrepository-layout#_code_partialclone_code)')
127 g.add_option('--clone-filter', action='store', default='blob:none',
128 dest='clone_filter',
129 help='filter for use with --partial-clone [default: %default]')
130 g.add_option('--archive',
131 dest='archive', action='store_true',
132 help='checkout an archive instead of a git repository for '
133 'each project. See git archive.')
134 g.add_option('--submodules',
135 dest='submodules', action='store_true',
136 help='sync any submodules associated with the manifest repo')
137 g.add_option('-g', '--groups',
138 dest='groups', default='default',
139 help='restrict manifest projects to ones with specified '
140 'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
141 metavar='GROUP')
142 g.add_option('-p', '--platform',
143 dest='platform', default='auto',
144 help='restrict manifest projects to ones with a specified '
145 'platform group [auto|all|none|linux|darwin|...]',
146 metavar='PLATFORM')
147 g.add_option('--no-clone-bundle',
148 dest='no_clone_bundle', action='store_true',
149 help='disable use of /clone.bundle on HTTP/HTTPS')
150 g.add_option('--no-tags',
151 dest='no_tags', action='store_true',
152 help="don't fetch tags in the manifest")
153
154 # Tool
155 g = p.add_option_group('repo Version options')
156 g.add_option('--repo-url',
157 dest='repo_url',
158 help='repo repository location', metavar='URL')
159 g.add_option('--repo-branch',
160 dest='repo_branch',
161 help='repo branch or revision', metavar='REVISION')
162 g.add_option('--no-repo-verify',
163 dest='no_repo_verify', action='store_true',
164 help='do not verify repo source code')
165
166 # Other
167 g = p.add_option_group('Other options')
168 g.add_option('--config-name',
169 dest='config_name', action="store_true", default=False,
170 help='Always prompt for name/e-mail')
171 94
172 def _RegisteredEnvironmentOptions(self): 95 def _RegisteredEnvironmentOptions(self):
173 return {'REPO_MANIFEST_URL': 'manifest_url', 96 return {'REPO_MANIFEST_URL': 'manifest_url',
174 'REPO_MIRROR_LOCATION': 'reference'} 97 'REPO_MIRROR_LOCATION': 'reference'}
175 98
99 def _CloneSuperproject(self, opt):
100 """Clone the superproject based on the superproject's url and branch.
101
102 Args:
103 opt: Program options returned from optparse. See _Options().
104 """
105 superproject = git_superproject.Superproject(self.manifest,
106 self.repodir,
107 self.git_event_log,
108 quiet=opt.quiet)
109 sync_result = superproject.Sync()
110 if not sync_result.success:
111 print('warning: git update of superproject failed, repo sync will not '
112 'use superproject to fetch source; while this error is not fatal, '
113 'and you can continue to run repo sync, please run repo init with '
114 'the --no-use-superproject option to stop seeing this warning',
115 file=sys.stderr)
116 if sync_result.fatal and opt.use_superproject is not None:
117 sys.exit(1)
118
176 def _SyncManifest(self, opt): 119 def _SyncManifest(self, opt):
177 m = self.manifest.manifestProject 120 m = self.manifest.manifestProject
178 is_new = not m.Exists 121 is_new = not m.Exists
179 122
123 # If repo has already been initialized, we take -u with the absence of
124 # --standalone-manifest to mean "transition to a standard repo set up",
125 # which necessitates starting fresh.
126 # If --standalone-manifest is set, we always tear everything down and start
127 # anew.
128 if not is_new:
129 was_standalone_manifest = m.config.GetString('manifest.standalone')
130 if opt.standalone_manifest or (
131 was_standalone_manifest and opt.manifest_url):
132 m.config.ClearCache()
133 if m.gitdir and os.path.exists(m.gitdir):
134 platform_utils.rmtree(m.gitdir)
135 if m.worktree and os.path.exists(m.worktree):
136 platform_utils.rmtree(m.worktree)
137
138 is_new = not m.Exists
180 if is_new: 139 if is_new:
181 if not opt.manifest_url: 140 if not opt.manifest_url:
182 print('fatal: manifest url (-u) is required.', file=sys.stderr) 141 print('fatal: manifest url is required.', file=sys.stderr)
183 sys.exit(1) 142 sys.exit(1)
184 143
185 if not opt.quiet: 144 if not opt.quiet:
186 print('Get %s' % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url), 145 print('Downloading manifest from %s' %
146 (GitConfig.ForUser().UrlInsteadOf(opt.manifest_url),),
187 file=sys.stderr) 147 file=sys.stderr)
188 148
189 # The manifest project object doesn't keep track of the path on the 149 # The manifest project object doesn't keep track of the path on the
@@ -200,30 +160,52 @@ to update the working directory files.
200 160
201 m._InitGitDir(mirror_git=mirrored_manifest_git) 161 m._InitGitDir(mirror_git=mirrored_manifest_git)
202 162
203 if opt.manifest_branch: 163 # If standalone_manifest is set, mark the project as "standalone" -- we'll
204 m.revisionExpr = opt.manifest_branch 164 # still do much of the manifests.git set up, but will avoid actual syncs to
205 else: 165 # a remote.
206 m.revisionExpr = 'refs/heads/master' 166 standalone_manifest = False
207 else: 167 if opt.standalone_manifest:
208 if opt.manifest_branch: 168 standalone_manifest = True
209 m.revisionExpr = opt.manifest_branch 169 elif not opt.manifest_url:
210 else: 170 # If -u is set and --standalone-manifest is not, then we're not in
211 m.PreSync() 171 # standalone mode. Otherwise, use config to infer what we were in the last
172 # init.
173 standalone_manifest = bool(m.config.GetString('manifest.standalone'))
174 m.config.SetString('manifest.standalone', opt.manifest_url)
212 175
213 self._ConfigureDepth(opt) 176 self._ConfigureDepth(opt)
214 177
178 # Set the remote URL before the remote branch as we might need it below.
215 if opt.manifest_url: 179 if opt.manifest_url:
216 r = m.GetRemote(m.remote.name) 180 r = m.GetRemote(m.remote.name)
217 r.url = opt.manifest_url 181 r.url = opt.manifest_url
218 r.ResetFetch() 182 r.ResetFetch()
219 r.Save() 183 r.Save()
220 184
185 if not standalone_manifest:
186 if opt.manifest_branch:
187 if opt.manifest_branch == 'HEAD':
188 opt.manifest_branch = m.ResolveRemoteHead()
189 if opt.manifest_branch is None:
190 print('fatal: unable to resolve HEAD', file=sys.stderr)
191 sys.exit(1)
192 m.revisionExpr = opt.manifest_branch
193 else:
194 if is_new:
195 default_branch = m.ResolveRemoteHead()
196 if default_branch is None:
197 # If the remote doesn't have HEAD configured, default to master.
198 default_branch = 'refs/heads/master'
199 m.revisionExpr = default_branch
200 else:
201 m.PreSync()
202
221 groups = re.split(r'[,\s]+', opt.groups) 203 groups = re.split(r'[,\s]+', opt.groups)
222 all_platforms = ['linux', 'darwin', 'windows'] 204 all_platforms = ['linux', 'darwin', 'windows']
223 platformize = lambda x: 'platform-' + x 205 platformize = lambda x: 'platform-' + x
224 if opt.platform == 'auto': 206 if opt.platform == 'auto':
225 if (not opt.mirror and 207 if (not opt.mirror and
226 not m.config.GetString('repo.mirror') == 'true'): 208 not m.config.GetString('repo.mirror') == 'true'):
227 groups.append(platformize(platform.system().lower())) 209 groups.append(platformize(platform.system().lower()))
228 elif opt.platform == 'all': 210 elif opt.platform == 'all':
229 groups.extend(map(platformize, all_platforms)) 211 groups.extend(map(platformize, all_platforms))
@@ -235,7 +217,7 @@ to update the working directory files.
235 217
236 groups = [x for x in groups if x] 218 groups = [x for x in groups if x]
237 groupstr = ','.join(groups) 219 groupstr = ','.join(groups)
238 if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower(): 220 if opt.platform == 'auto' and groupstr == self.manifest.GetDefaultGroupsStr():
239 groupstr = None 221 groupstr = None
240 m.config.SetString('manifest.groups', groupstr) 222 m.config.SetString('manifest.groups', groupstr)
241 223
@@ -243,11 +225,25 @@ to update the working directory files.
243 m.config.SetString('repo.reference', opt.reference) 225 m.config.SetString('repo.reference', opt.reference)
244 226
245 if opt.dissociate: 227 if opt.dissociate:
246 m.config.SetString('repo.dissociate', 'true') 228 m.config.SetBoolean('repo.dissociate', opt.dissociate)
229
230 if opt.worktree:
231 if opt.mirror:
232 print('fatal: --mirror and --worktree are incompatible',
233 file=sys.stderr)
234 sys.exit(1)
235 if opt.submodules:
236 print('fatal: --submodules and --worktree are incompatible',
237 file=sys.stderr)
238 sys.exit(1)
239 m.config.SetBoolean('repo.worktree', opt.worktree)
240 if is_new:
241 m.use_git_worktrees = True
242 print('warning: --worktree is experimental!', file=sys.stderr)
247 243
248 if opt.archive: 244 if opt.archive:
249 if is_new: 245 if is_new:
250 m.config.SetString('repo.archive', 'true') 246 m.config.SetBoolean('repo.archive', opt.archive)
251 else: 247 else:
252 print('fatal: --archive is only supported when initializing a new ' 248 print('fatal: --archive is only supported when initializing a new '
253 'workspace.', file=sys.stderr) 249 'workspace.', file=sys.stderr)
@@ -257,7 +253,7 @@ to update the working directory files.
257 253
258 if opt.mirror: 254 if opt.mirror:
259 if is_new: 255 if is_new:
260 m.config.SetString('repo.mirror', 'true') 256 m.config.SetBoolean('repo.mirror', opt.mirror)
261 else: 257 else:
262 print('fatal: --mirror is only supported when initializing a new ' 258 print('fatal: --mirror is only supported when initializing a new '
263 'workspace.', file=sys.stderr) 259 'workspace.', file=sys.stderr)
@@ -265,25 +261,49 @@ to update the working directory files.
265 'in another location.', file=sys.stderr) 261 'in another location.', file=sys.stderr)
266 sys.exit(1) 262 sys.exit(1)
267 263
268 if opt.partial_clone: 264 if opt.partial_clone is not None:
269 if opt.mirror: 265 if opt.mirror:
270 print('fatal: --mirror and --partial-clone are mutually exclusive', 266 print('fatal: --mirror and --partial-clone are mutually exclusive',
271 file=sys.stderr) 267 file=sys.stderr)
272 sys.exit(1) 268 sys.exit(1)
273 m.config.SetString('repo.partialclone', 'true') 269 m.config.SetBoolean('repo.partialclone', opt.partial_clone)
274 if opt.clone_filter: 270 if opt.clone_filter:
275 m.config.SetString('repo.clonefilter', opt.clone_filter) 271 m.config.SetString('repo.clonefilter', opt.clone_filter)
272 elif m.config.GetBoolean('repo.partialclone'):
273 opt.clone_filter = m.config.GetString('repo.clonefilter')
276 else: 274 else:
277 opt.clone_filter = None 275 opt.clone_filter = None
278 276
277 if opt.partial_clone_exclude is not None:
278 m.config.SetString('repo.partialcloneexclude', opt.partial_clone_exclude)
279
280 if opt.clone_bundle is None:
281 opt.clone_bundle = False if opt.partial_clone else True
282 else:
283 m.config.SetBoolean('repo.clonebundle', opt.clone_bundle)
284
279 if opt.submodules: 285 if opt.submodules:
280 m.config.SetString('repo.submodules', 'true') 286 m.config.SetBoolean('repo.submodules', opt.submodules)
287
288 if opt.use_superproject is not None:
289 m.config.SetBoolean('repo.superproject', opt.use_superproject)
290
291 if standalone_manifest:
292 if is_new:
293 manifest_name = 'default.xml'
294 manifest_data = fetch.fetch_file(opt.manifest_url)
295 dest = os.path.join(m.worktree, manifest_name)
296 os.makedirs(os.path.dirname(dest), exist_ok=True)
297 with open(dest, 'wb') as f:
298 f.write(manifest_data)
299 return
281 300
282 if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, 301 if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose,
283 clone_bundle=not opt.no_clone_bundle, 302 clone_bundle=opt.clone_bundle,
284 current_branch_only=opt.current_branch_only, 303 current_branch_only=opt.current_branch_only,
285 no_tags=opt.no_tags, submodules=opt.submodules, 304 tags=opt.tags, submodules=opt.submodules,
286 clone_filter=opt.clone_filter): 305 clone_filter=opt.clone_filter,
306 partial_clone_exclude=self.manifest.PartialCloneExclude):
287 r = m.GetRemote(m.remote.name) 307 r = m.GetRemote(m.remote.name)
288 print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr) 308 print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
289 309
@@ -326,8 +346,8 @@ to update the working directory files.
326 return value 346 return value
327 return a 347 return a
328 348
329 def _ShouldConfigureUser(self): 349 def _ShouldConfigureUser(self, opt):
330 gc = self.manifest.globalConfig 350 gc = self.client.globalConfig
331 mp = self.manifest.manifestProject 351 mp = self.manifest.manifestProject
332 352
333 # If we don't have local settings, get from global. 353 # If we don't have local settings, get from global.
@@ -338,21 +358,24 @@ to update the working directory files.
338 mp.config.SetString('user.name', gc.GetString('user.name')) 358 mp.config.SetString('user.name', gc.GetString('user.name'))
339 mp.config.SetString('user.email', gc.GetString('user.email')) 359 mp.config.SetString('user.email', gc.GetString('user.email'))
340 360
341 print() 361 if not opt.quiet:
342 print('Your identity is: %s <%s>' % (mp.config.GetString('user.name'), 362 print()
343 mp.config.GetString('user.email'))) 363 print('Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
344 print('If you want to change this, please re-run \'repo init\' with --config-name') 364 mp.config.GetString('user.email')))
365 print("If you want to change this, please re-run 'repo init' with --config-name")
345 return False 366 return False
346 367
347 def _ConfigureUser(self): 368 def _ConfigureUser(self, opt):
348 mp = self.manifest.manifestProject 369 mp = self.manifest.manifestProject
349 370
350 while True: 371 while True:
351 print() 372 if not opt.quiet:
352 name = self._Prompt('Your Name', mp.UserName) 373 print()
374 name = self._Prompt('Your Name', mp.UserName)
353 email = self._Prompt('Your Email', mp.UserEmail) 375 email = self._Prompt('Your Email', mp.UserEmail)
354 376
355 print() 377 if not opt.quiet:
378 print()
356 print('Your identity is: %s <%s>' % (name, email)) 379 print('Your identity is: %s <%s>' % (name, email))
357 print('is this correct [y/N]? ', end='') 380 print('is this correct [y/N]? ', end='')
358 # TODO: When we require Python 3, use flush=True w/print above. 381 # TODO: When we require Python 3, use flush=True w/print above.
@@ -373,7 +396,7 @@ to update the working directory files.
373 return False 396 return False
374 397
375 def _ConfigureColor(self): 398 def _ConfigureColor(self):
376 gc = self.manifest.globalConfig 399 gc = self.client.globalConfig
377 if self._HasColorSet(gc): 400 if self._HasColorSet(gc):
378 return 401 return
379 402
@@ -424,15 +447,16 @@ to update the working directory files.
424 # We store the depth in the main manifest project. 447 # We store the depth in the main manifest project.
425 self.manifest.manifestProject.config.SetString('repo.depth', depth) 448 self.manifest.manifestProject.config.SetString('repo.depth', depth)
426 449
427 def _DisplayResult(self): 450 def _DisplayResult(self, opt):
428 if self.manifest.IsMirror: 451 if self.manifest.IsMirror:
429 init_type = 'mirror ' 452 init_type = 'mirror '
430 else: 453 else:
431 init_type = '' 454 init_type = ''
432 455
433 print() 456 if not opt.quiet:
434 print('repo %shas been initialized in %s' 457 print()
435 % (init_type, self.manifest.topdir)) 458 print('repo %shas been initialized in %s' %
459 (init_type, self.manifest.topdir))
436 460
437 current_dir = os.getcwd() 461 current_dir = os.getcwd()
438 if current_dir != self.manifest.topdir: 462 if current_dir != self.manifest.topdir:
@@ -450,15 +474,61 @@ to update the working directory files.
450 if opt.archive and opt.mirror: 474 if opt.archive and opt.mirror:
451 self.OptionParser.error('--mirror and --archive cannot be used together.') 475 self.OptionParser.error('--mirror and --archive cannot be used together.')
452 476
477 if opt.standalone_manifest and (
478 opt.manifest_branch or opt.manifest_name != 'default.xml'):
479 self.OptionParser.error('--manifest-branch and --manifest-name cannot'
480 ' be used with --standalone-manifest.')
481
482 if args:
483 if opt.manifest_url:
484 self.OptionParser.error(
485 '--manifest-url option and URL argument both specified: only use '
486 'one to select the manifest URL.')
487
488 opt.manifest_url = args.pop(0)
489
490 if args:
491 self.OptionParser.error('too many arguments to init')
492
453 def Execute(self, opt, args): 493 def Execute(self, opt, args):
454 git_require(MIN_GIT_VERSION, fail=True) 494 git_require(MIN_GIT_VERSION_HARD, fail=True)
495 if not git_require(MIN_GIT_VERSION_SOFT):
496 print('repo: warning: git-%s+ will soon be required; please upgrade your '
497 'version of git to maintain support.'
498 % ('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT),),
499 file=sys.stderr)
500
501 rp = self.manifest.repoProject
502
503 # Handle new --repo-url requests.
504 if opt.repo_url:
505 remote = rp.GetRemote('origin')
506 remote.url = opt.repo_url
507 remote.Save()
508
509 # Handle new --repo-rev requests.
510 if opt.repo_rev:
511 wrapper = Wrapper()
512 remote_ref, rev = wrapper.check_repo_rev(
513 rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
514 branch = rp.GetBranch('default')
515 branch.merge = remote_ref
516 rp.work_git.reset('--hard', rev)
517 branch.Save()
518
519 if opt.worktree:
520 # Older versions of git supported worktree, but had dangerous gc bugs.
521 git_require((2, 15, 0), fail=True, msg='git gc worktree corruption')
455 522
456 self._SyncManifest(opt) 523 self._SyncManifest(opt)
457 self._LinkManifest(opt.manifest_name) 524 self._LinkManifest(opt.manifest_name)
458 525
526 if self.manifest.manifestProject.config.GetBoolean('repo.superproject'):
527 self._CloneSuperproject(opt)
528
459 if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror: 529 if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
460 if opt.config_name or self._ShouldConfigureUser(): 530 if opt.config_name or self._ShouldConfigureUser(opt):
461 self._ConfigureUser() 531 self._ConfigureUser(opt)
462 self._ConfigureColor() 532 self._ConfigureColor()
463 533
464 self._DisplayResult() 534 self._DisplayResult(opt)
diff --git a/subcmds/list.py b/subcmds/list.py
index 00172f0e..6adf85b7 100644
--- a/subcmds/list.py
+++ b/subcmds/list.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2011 The Android Open Source Project 1# Copyright (C) 2011 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,45 +12,59 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function 15import os
18import sys
19 16
20from command import Command, MirrorSafeCommand 17from command import Command, MirrorSafeCommand
21 18
19
22class List(Command, MirrorSafeCommand): 20class List(Command, MirrorSafeCommand):
23 common = True 21 COMMON = True
24 helpSummary = "List projects and their associated directories" 22 helpSummary = "List projects and their associated directories"
25 helpUsage = """ 23 helpUsage = """
26%prog [-f] [<project>...] 24%prog [-f] [<project>...]
27%prog [-f] -r str1 [str2]..." 25%prog [-f] -r str1 [str2]...
28""" 26"""
29 helpDescription = """ 27 helpDescription = """
30List all projects; pass '.' to list the project for the cwd. 28List all projects; pass '.' to list the project for the cwd.
31 29
30By default, only projects that currently exist in the checkout are shown. If
31you want to list all projects (using the specified filter settings), use the
32--all option. If you want to show all projects regardless of the manifest
33groups, then also pass --groups all.
34
32This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'. 35This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
33""" 36"""
34 37
35 def _Options(self, p): 38 def _Options(self, p):
36 p.add_option('-r', '--regex', 39 p.add_option('-r', '--regex',
37 dest='regex', action='store_true', 40 dest='regex', action='store_true',
38 help="Filter the project list based on regex or wildcard matching of strings") 41 help='filter the project list based on regex or wildcard matching of strings')
39 p.add_option('-g', '--groups', 42 p.add_option('-g', '--groups',
40 dest='groups', 43 dest='groups',
41 help="Filter the project list based on the groups the project is in") 44 help='filter the project list based on the groups the project is in')
42 p.add_option('-f', '--fullpath', 45 p.add_option('-a', '--all',
43 dest='fullpath', action='store_true', 46 action='store_true',
44 help="Display the full work tree path instead of the relative path") 47 help='show projects regardless of checkout state')
45 p.add_option('-n', '--name-only', 48 p.add_option('-n', '--name-only',
46 dest='name_only', action='store_true', 49 dest='name_only', action='store_true',
47 help="Display only the name of the repository") 50 help='display only the name of the repository')
48 p.add_option('-p', '--path-only', 51 p.add_option('-p', '--path-only',
49 dest='path_only', action='store_true', 52 dest='path_only', action='store_true',
50 help="Display only the path of the repository") 53 help='display only the path of the repository')
54 p.add_option('-f', '--fullpath',
55 dest='fullpath', action='store_true',
56 help='display the full work tree path instead of the relative path')
57 p.add_option('--relative-to', metavar='PATH',
58 help='display paths relative to this one (default: top of repo client checkout)')
51 59
52 def ValidateOptions(self, opt, args): 60 def ValidateOptions(self, opt, args):
53 if opt.fullpath and opt.name_only: 61 if opt.fullpath and opt.name_only:
54 self.OptionParser.error('cannot combine -f and -n') 62 self.OptionParser.error('cannot combine -f and -n')
55 63
64 # Resolve any symlinks so the output is stable.
65 if opt.relative_to:
66 opt.relative_to = os.path.realpath(opt.relative_to)
67
56 def Execute(self, opt, args): 68 def Execute(self, opt, args):
57 """List all projects and the associated directories. 69 """List all projects and the associated directories.
58 70
@@ -65,23 +77,26 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
65 args: Positional args. Can be a list of projects to list, or empty. 77 args: Positional args. Can be a list of projects to list, or empty.
66 """ 78 """
67 if not opt.regex: 79 if not opt.regex:
68 projects = self.GetProjects(args, groups=opt.groups) 80 projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all)
69 else: 81 else:
70 projects = self.FindProjects(args) 82 projects = self.FindProjects(args)
71 83
72 def _getpath(x): 84 def _getpath(x):
73 if opt.fullpath: 85 if opt.fullpath:
74 return x.worktree 86 return x.worktree
87 if opt.relative_to:
88 return os.path.relpath(x.worktree, opt.relative_to)
75 return x.relpath 89 return x.relpath
76 90
77 lines = [] 91 lines = []
78 for project in projects: 92 for project in projects:
79 if opt.name_only and not opt.path_only: 93 if opt.name_only and not opt.path_only:
80 lines.append("%s" % ( project.name)) 94 lines.append("%s" % (project.name))
81 elif opt.path_only and not opt.name_only: 95 elif opt.path_only and not opt.name_only:
82 lines.append("%s" % (_getpath(project))) 96 lines.append("%s" % (_getpath(project)))
83 else: 97 else:
84 lines.append("%s : %s" % (_getpath(project), project.name)) 98 lines.append("%s : %s" % (_getpath(project), project.name))
85 99
86 lines.sort() 100 if lines:
87 print('\n'.join(lines)) 101 lines.sort()
102 print('\n'.join(lines))
diff --git a/subcmds/manifest.py b/subcmds/manifest.py
index 9c1b3f0c..0fbdeac0 100644
--- a/subcmds/manifest.py
+++ b/subcmds/manifest.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2009 The Android Open Source Project 1# Copyright (C) 2009 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,25 +12,32 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function 15import json
18import os 16import os
19import sys 17import sys
20 18
21from command import PagedCommand 19from command import PagedCommand
22 20
21
23class Manifest(PagedCommand): 22class Manifest(PagedCommand):
24 common = False 23 COMMON = False
25 helpSummary = "Manifest inspection utility" 24 helpSummary = "Manifest inspection utility"
26 helpUsage = """ 25 helpUsage = """
27%prog [-o {-|NAME.xml} [-r]] 26%prog [-o {-|NAME.xml}] [-m MANIFEST.xml] [-r]
28""" 27"""
29 _helpDescription = """ 28 _helpDescription = """
30 29
31With the -o option, exports the current manifest for inspection. 30With the -o option, exports the current manifest for inspection.
32The manifest and (if present) local_manifest.xml are combined 31The manifest and (if present) local_manifests/ are combined
33together to produce a single manifest file. This file can be stored 32together to produce a single manifest file. This file can be stored
34in a Git repository for use during future 'repo init' invocations. 33in a Git repository for use during future 'repo init' invocations.
35 34
35The -r option can be used to generate a manifest file with project
36revisions set to the current commit hash. These are known as
37"revision locked manifests", as they don't follow a particular branch.
38In this case, the 'upstream' attribute is set to the ref we were on
39when the manifest was generated. The 'dest-branch' attribute is set
40to indicate the remote ref to push changes to via 'repo upload'.
36""" 41"""
37 42
38 @property 43 @property
@@ -48,26 +53,63 @@ in a Git repository for use during future 'repo init' invocations.
48 def _Options(self, p): 53 def _Options(self, p):
49 p.add_option('-r', '--revision-as-HEAD', 54 p.add_option('-r', '--revision-as-HEAD',
50 dest='peg_rev', action='store_true', 55 dest='peg_rev', action='store_true',
51 help='Save revisions as current HEAD') 56 help='save revisions as current HEAD')
57 p.add_option('-m', '--manifest-name',
58 help='temporary manifest to use for this sync', metavar='NAME.xml')
52 p.add_option('--suppress-upstream-revision', dest='peg_rev_upstream', 59 p.add_option('--suppress-upstream-revision', dest='peg_rev_upstream',
53 default=True, action='store_false', 60 default=True, action='store_false',
54 help='If in -r mode, do not write the upstream field. ' 61 help='if in -r mode, do not write the upstream field '
55 'Only of use if the branch names for a sha1 manifest are ' 62 '(only of use if the branch names for a sha1 manifest are '
56 'sensitive.') 63 'sensitive)')
64 p.add_option('--suppress-dest-branch', dest='peg_rev_dest_branch',
65 default=True, action='store_false',
66 help='if in -r mode, do not write the dest-branch field '
67 '(only of use if the branch names for a sha1 manifest are '
68 'sensitive)')
69 p.add_option('--json', default=False, action='store_true',
70 help='output manifest in JSON format (experimental)')
71 p.add_option('--pretty', default=False, action='store_true',
72 help='format output for humans to read')
73 p.add_option('--no-local-manifests', default=False, action='store_true',
74 dest='ignore_local_manifests', help='ignore local manifests')
57 p.add_option('-o', '--output-file', 75 p.add_option('-o', '--output-file',
58 dest='output_file', 76 dest='output_file',
59 default='-', 77 default='-',
60 help='File to save the manifest to', 78 help='file to save the manifest to',
61 metavar='-|NAME.xml') 79 metavar='-|NAME.xml')
62 80
63 def _Output(self, opt): 81 def _Output(self, opt):
82 # If alternate manifest is specified, override the manifest file that we're using.
83 if opt.manifest_name:
84 self.manifest.Override(opt.manifest_name, False)
85
64 if opt.output_file == '-': 86 if opt.output_file == '-':
65 fd = sys.stdout 87 fd = sys.stdout
66 else: 88 else:
67 fd = open(opt.output_file, 'w') 89 fd = open(opt.output_file, 'w')
68 self.manifest.Save(fd, 90
69 peg_rev = opt.peg_rev, 91 self.manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
70 peg_rev_upstream = opt.peg_rev_upstream) 92
93 if opt.json:
94 print('warning: --json is experimental!', file=sys.stderr)
95 doc = self.manifest.ToDict(peg_rev=opt.peg_rev,
96 peg_rev_upstream=opt.peg_rev_upstream,
97 peg_rev_dest_branch=opt.peg_rev_dest_branch)
98
99 json_settings = {
100 # JSON style guide says Uunicode characters are fully allowed.
101 'ensure_ascii': False,
102 # We use 2 space indent to match JSON style guide.
103 'indent': 2 if opt.pretty else None,
104 'separators': (',', ': ') if opt.pretty else (',', ':'),
105 'sort_keys': True,
106 }
107 fd.write(json.dumps(doc, **json_settings))
108 else:
109 self.manifest.Save(fd,
110 peg_rev=opt.peg_rev,
111 peg_rev_upstream=opt.peg_rev_upstream,
112 peg_rev_dest_branch=opt.peg_rev_dest_branch)
71 fd.close() 113 fd.close()
72 if opt.output_file != '-': 114 if opt.output_file != '-':
73 print('Saved manifest to %s' % opt.output_file, file=sys.stderr) 115 print('Saved manifest to %s' % opt.output_file, file=sys.stderr)
diff --git a/subcmds/overview.py b/subcmds/overview.py
index 08b58a6c..63f5a79e 100644
--- a/subcmds/overview.py
+++ b/subcmds/overview.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2012 The Android Open Source Project 1# Copyright (C) 2012 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,13 +12,14 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function 15import optparse
16
18from color import Coloring 17from color import Coloring
19from command import PagedCommand 18from command import PagedCommand
20 19
21 20
22class Overview(PagedCommand): 21class Overview(PagedCommand):
23 common = True 22 COMMON = True
24 helpSummary = "Display overview of unmerged project branches" 23 helpSummary = "Display overview of unmerged project branches"
25 helpUsage = """ 24 helpUsage = """
26%prog [--current-branch] [<project>...] 25%prog [--current-branch] [<project>...]
@@ -29,15 +28,22 @@ class Overview(PagedCommand):
29The '%prog' command is used to display an overview of the projects branches, 28The '%prog' command is used to display an overview of the projects branches,
30and list any local commits that have not yet been merged into the project. 29and list any local commits that have not yet been merged into the project.
31 30
32The -b/--current-branch option can be used to restrict the output to only 31The -c/--current-branch option can be used to restrict the output to only
33branches currently checked out in each project. By default, all branches 32branches currently checked out in each project. By default, all branches
34are displayed. 33are displayed.
35""" 34"""
36 35
37 def _Options(self, p): 36 def _Options(self, p):
38 p.add_option('-b', '--current-branch', 37 p.add_option('-c', '--current-branch',
39 dest="current_branch", action="store_true", 38 dest="current_branch", action="store_true",
40 help="Consider only checked out branches") 39 help="consider only checked out branches")
40 p.add_option('--no-current-branch',
41 dest='current_branch', action='store_false',
42 help='consider all local branches')
43 # Turn this into a warning & remove this someday.
44 p.add_option('-b',
45 dest='current_branch', action='store_true',
46 help=optparse.SUPPRESS_HELP)
41 47
42 def Execute(self, opt, args): 48 def Execute(self, opt, args):
43 all_branches = [] 49 all_branches = []
diff --git a/subcmds/prune.py b/subcmds/prune.py
index ff2fba1d..584ee7ed 100644
--- a/subcmds/prune.py
+++ b/subcmds/prune.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,21 +12,38 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function 15import itertools
16
18from color import Coloring 17from color import Coloring
19from command import PagedCommand 18from command import DEFAULT_LOCAL_JOBS, PagedCommand
19
20 20
21class Prune(PagedCommand): 21class Prune(PagedCommand):
22 common = True 22 COMMON = True
23 helpSummary = "Prune (delete) already merged topics" 23 helpSummary = "Prune (delete) already merged topics"
24 helpUsage = """ 24 helpUsage = """
25%prog [<project>...] 25%prog [<project>...]
26""" 26"""
27 PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
28
29 def _ExecuteOne(self, project):
30 """Process one project."""
31 return project.PruneHeads()
27 32
28 def Execute(self, opt, args): 33 def Execute(self, opt, args):
29 all_branches = [] 34 projects = self.GetProjects(args)
30 for project in self.GetProjects(args): 35
31 all_branches.extend(project.PruneHeads()) 36 # NB: Should be able to refactor this module to display summary as results
37 # come back from children.
38 def _ProcessResults(_pool, _output, results):
39 return list(itertools.chain.from_iterable(results))
40
41 all_branches = self.ExecuteInParallel(
42 opt.jobs,
43 self._ExecuteOne,
44 projects,
45 callback=_ProcessResults,
46 ordered=True)
32 47
33 if not all_branches: 48 if not all_branches:
34 return 49 return
diff --git a/subcmds/rebase.py b/subcmds/rebase.py
index dcb8b2a3..7c53eb7a 100644
--- a/subcmds/rebase.py
+++ b/subcmds/rebase.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2010 The Android Open Source Project 1# Copyright (C) 2010 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +12,6 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function
18import sys 15import sys
19 16
20from color import Coloring 17from color import Coloring
@@ -30,7 +27,7 @@ class RebaseColoring(Coloring):
30 27
31 28
32class Rebase(Command): 29class Rebase(Command):
33 common = True 30 COMMON = True
34 helpSummary = "Rebase local branches on upstream branch" 31 helpSummary = "Rebase local branches on upstream branch"
35 helpUsage = """ 32 helpUsage = """
36%prog {[<project>...] | -i <project>...} 33%prog {[<project>...] | -i <project>...}
@@ -42,36 +39,34 @@ branch but need to incorporate new upstream changes "underneath" them.
42""" 39"""
43 40
44 def _Options(self, p): 41 def _Options(self, p):
45 p.add_option('-i', '--interactive', 42 g = p.get_option_group('--quiet')
46 dest="interactive", action="store_true", 43 g.add_option('-i', '--interactive',
47 help="interactive rebase (single project only)") 44 dest="interactive", action="store_true",
45 help="interactive rebase (single project only)")
48 46
49 p.add_option('--fail-fast', 47 p.add_option('--fail-fast',
50 dest='fail_fast', action='store_true', 48 dest='fail_fast', action='store_true',
51 help='Stop rebasing after first error is hit') 49 help='stop rebasing after first error is hit')
52 p.add_option('-f', '--force-rebase', 50 p.add_option('-f', '--force-rebase',
53 dest='force_rebase', action='store_true', 51 dest='force_rebase', action='store_true',
54 help='Pass --force-rebase to git rebase') 52 help='pass --force-rebase to git rebase')
55 p.add_option('--no-ff', 53 p.add_option('--no-ff',
56 dest='no_ff', action='store_true', 54 dest='ff', default=True, action='store_false',
57 help='Pass --no-ff to git rebase') 55 help='pass --no-ff to git rebase')
58 p.add_option('-q', '--quiet',
59 dest='quiet', action='store_true',
60 help='Pass --quiet to git rebase')
61 p.add_option('--autosquash', 56 p.add_option('--autosquash',
62 dest='autosquash', action='store_true', 57 dest='autosquash', action='store_true',
63 help='Pass --autosquash to git rebase') 58 help='pass --autosquash to git rebase')
64 p.add_option('--whitespace', 59 p.add_option('--whitespace',
65 dest='whitespace', action='store', metavar='WS', 60 dest='whitespace', action='store', metavar='WS',
66 help='Pass --whitespace to git rebase') 61 help='pass --whitespace to git rebase')
67 p.add_option('--auto-stash', 62 p.add_option('--auto-stash',
68 dest='auto_stash', action='store_true', 63 dest='auto_stash', action='store_true',
69 help='Stash local modifications before starting') 64 help='stash local modifications before starting')
70 p.add_option('-m', '--onto-manifest', 65 p.add_option('-m', '--onto-manifest',
71 dest='onto_manifest', action='store_true', 66 dest='onto_manifest', action='store_true',
72 help='Rebase onto the manifest version instead of upstream ' 67 help='rebase onto the manifest version instead of upstream '
73 'HEAD. This helps to make sure the local tree stays ' 68 'HEAD (this helps to make sure the local tree stays '
74 'consistent if you previously synced to a manifest.') 69 'consistent if you previously synced to a manifest)')
75 70
76 def Execute(self, opt, args): 71 def Execute(self, opt, args):
77 all_projects = self.GetProjects(args) 72 all_projects = self.GetProjects(args)
@@ -82,7 +77,7 @@ branch but need to incorporate new upstream changes "underneath" them.
82 file=sys.stderr) 77 file=sys.stderr)
83 if len(args) == 1: 78 if len(args) == 1:
84 print('note: project %s is mapped to more than one path' % (args[0],), 79 print('note: project %s is mapped to more than one path' % (args[0],),
85 file=sys.stderr) 80 file=sys.stderr)
86 return 1 81 return 1
87 82
88 # Setup the common git rebase args that we use for all projects. 83 # Setup the common git rebase args that we use for all projects.
@@ -93,7 +88,7 @@ branch but need to incorporate new upstream changes "underneath" them.
93 common_args.append('--quiet') 88 common_args.append('--quiet')
94 if opt.force_rebase: 89 if opt.force_rebase:
95 common_args.append('--force-rebase') 90 common_args.append('--force-rebase')
96 if opt.no_ff: 91 if not opt.ff:
97 common_args.append('--no-ff') 92 common_args.append('--no-ff')
98 if opt.autosquash: 93 if opt.autosquash:
99 common_args.append('--autosquash') 94 common_args.append('--autosquash')
diff --git a/subcmds/selfupdate.py b/subcmds/selfupdate.py
index a8a09b64..282f518e 100644
--- a/subcmds/selfupdate.py
+++ b/subcmds/selfupdate.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2009 The Android Open Source Project 1# Copyright (C) 2009 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +12,6 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function
18from optparse import SUPPRESS_HELP 15from optparse import SUPPRESS_HELP
19import sys 16import sys
20 17
@@ -22,8 +19,9 @@ from command import Command, MirrorSafeCommand
22from subcmds.sync import _PostRepoUpgrade 19from subcmds.sync import _PostRepoUpgrade
23from subcmds.sync import _PostRepoFetch 20from subcmds.sync import _PostRepoFetch
24 21
22
25class Selfupdate(Command, MirrorSafeCommand): 23class Selfupdate(Command, MirrorSafeCommand):
26 common = False 24 COMMON = False
27 helpSummary = "Update repo to the latest version" 25 helpSummary = "Update repo to the latest version"
28 helpUsage = """ 26 helpUsage = """
29%prog 27%prog
@@ -39,7 +37,7 @@ need to be performed by an end-user.
39 def _Options(self, p): 37 def _Options(self, p):
40 g = p.add_option_group('repo Version options') 38 g = p.add_option_group('repo Version options')
41 g.add_option('--no-repo-verify', 39 g.add_option('--no-repo-verify',
42 dest='no_repo_verify', action='store_true', 40 dest='repo_verify', default=True, action='store_false',
43 help='do not verify repo source code') 41 help='do not verify repo source code')
44 g.add_option('--repo-upgraded', 42 g.add_option('--repo-upgraded',
45 dest='repo_upgraded', action='store_true', 43 dest='repo_upgraded', action='store_true',
@@ -59,5 +57,5 @@ need to be performed by an end-user.
59 57
60 rp.bare_git.gc('--auto') 58 rp.bare_git.gc('--auto')
61 _PostRepoFetch(rp, 59 _PostRepoFetch(rp,
62 no_repo_verify = opt.no_repo_verify, 60 repo_verify=opt.repo_verify,
63 verbose = True) 61 verbose=True)
diff --git a/subcmds/smartsync.py b/subcmds/smartsync.py
index 675b9834..d91d59c6 100644
--- a/subcmds/smartsync.py
+++ b/subcmds/smartsync.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2010 The Android Open Source Project 1# Copyright (C) 2010 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,8 +14,9 @@
16 14
17from subcmds.sync import Sync 15from subcmds.sync import Sync
18 16
17
19class Smartsync(Sync): 18class Smartsync(Sync):
20 common = True 19 COMMON = True
21 helpSummary = "Update working tree to the latest known good revision" 20 helpSummary = "Update working tree to the latest known good revision"
22 helpUsage = """ 21 helpUsage = """
23%prog [<project>...] 22%prog [<project>...]
diff --git a/subcmds/stage.py b/subcmds/stage.py
index aeb49513..0389a4ff 100644
--- a/subcmds/stage.py
+++ b/subcmds/stage.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,13 +12,13 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function
18import sys 15import sys
19 16
20from color import Coloring 17from color import Coloring
21from command import InteractiveCommand 18from command import InteractiveCommand
22from git_command import GitCommand 19from git_command import GitCommand
23 20
21
24class _ProjectList(Coloring): 22class _ProjectList(Coloring):
25 def __init__(self, gc): 23 def __init__(self, gc):
26 Coloring.__init__(self, gc, 'interactive') 24 Coloring.__init__(self, gc, 'interactive')
@@ -28,8 +26,9 @@ class _ProjectList(Coloring):
28 self.header = self.printer('header', attr='bold') 26 self.header = self.printer('header', attr='bold')
29 self.help = self.printer('help', fg='red', attr='bold') 27 self.help = self.printer('help', fg='red', attr='bold')
30 28
29
31class Stage(InteractiveCommand): 30class Stage(InteractiveCommand):
32 common = True 31 COMMON = True
33 helpSummary = "Stage file(s) for commit" 32 helpSummary = "Stage file(s) for commit"
34 helpUsage = """ 33 helpUsage = """
35%prog -i [<project>...] 34%prog -i [<project>...]
@@ -39,7 +38,8 @@ The '%prog' command stages files to prepare the next commit.
39""" 38"""
40 39
41 def _Options(self, p): 40 def _Options(self, p):
42 p.add_option('-i', '--interactive', 41 g = p.get_option_group('--quiet')
42 g.add_option('-i', '--interactive',
43 dest='interactive', action='store_true', 43 dest='interactive', action='store_true',
44 help='use interactive staging') 44 help='use interactive staging')
45 45
@@ -105,6 +105,7 @@ The '%prog' command stages files to prepare the next commit.
105 continue 105 continue
106 print('Bye.') 106 print('Bye.')
107 107
108
108def _AddI(project): 109def _AddI(project):
109 p = GitCommand(project, ['add', '--interactive'], bare=False) 110 p = GitCommand(project, ['add', '--interactive'], bare=False)
110 p.Wait() 111 p.Wait()
diff --git a/subcmds/start.py b/subcmds/start.py
index 6ec0b2ce..2addaf2e 100644
--- a/subcmds/start.py
+++ b/subcmds/start.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,19 +12,20 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function 15import functools
18import os 16import os
19import sys 17import sys
20 18
21from command import Command 19from command import Command, DEFAULT_LOCAL_JOBS
22from git_config import IsImmutable 20from git_config import IsImmutable
23from git_command import git 21from git_command import git
24import gitc_utils 22import gitc_utils
25from progress import Progress 23from progress import Progress
26from project import SyncBuffer 24from project import SyncBuffer
27 25
26
28class Start(Command): 27class Start(Command):
29 common = True 28 COMMON = True
30 helpSummary = "Start a new branch for development" 29 helpSummary = "Start a new branch for development"
31 helpUsage = """ 30 helpUsage = """
32%prog <newbranchname> [--all | <project>...] 31%prog <newbranchname> [--all | <project>...]
@@ -35,6 +34,7 @@ class Start(Command):
35'%prog' begins a new branch of development, starting from the 34'%prog' begins a new branch of development, starting from the
36revision specified in the manifest. 35revision specified in the manifest.
37""" 36"""
37 PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
38 38
39 def _Options(self, p): 39 def _Options(self, p):
40 p.add_option('--all', 40 p.add_option('--all',
@@ -42,7 +42,8 @@ revision specified in the manifest.
42 help='begin branch in all projects') 42 help='begin branch in all projects')
43 p.add_option('-r', '--rev', '--revision', dest='revision', 43 p.add_option('-r', '--rev', '--revision', dest='revision',
44 help='point branch at this revision instead of upstream') 44 help='point branch at this revision instead of upstream')
45 p.add_option('--head', dest='revision', action='store_const', const='HEAD', 45 p.add_option('--head', '--HEAD',
46 dest='revision', action='store_const', const='HEAD',
46 help='abbreviation for --rev HEAD') 47 help='abbreviation for --rev HEAD')
47 48
48 def ValidateOptions(self, opt, args): 49 def ValidateOptions(self, opt, args):
@@ -53,6 +54,26 @@ revision specified in the manifest.
53 if not git.check_ref_format('heads/%s' % nb): 54 if not git.check_ref_format('heads/%s' % nb):
54 self.OptionParser.error("'%s' is not a valid name" % nb) 55 self.OptionParser.error("'%s' is not a valid name" % nb)
55 56
57 def _ExecuteOne(self, revision, nb, project):
58 """Start one project."""
59 # If the current revision is immutable, such as a SHA1, a tag or
60 # a change, then we can't push back to it. Substitute with
61 # dest_branch, if defined; or with manifest default revision instead.
62 branch_merge = ''
63 if IsImmutable(project.revisionExpr):
64 if project.dest_branch:
65 branch_merge = project.dest_branch
66 else:
67 branch_merge = self.manifest.default.revisionExpr
68
69 try:
70 ret = project.StartBranch(
71 nb, branch_merge=branch_merge, revision=revision)
72 except Exception as e:
73 print('error: unable to checkout %s: %s' % (project.name, e), file=sys.stderr)
74 ret = False
75 return (ret, project)
76
56 def Execute(self, opt, args): 77 def Execute(self, opt, args):
57 nb = args[0] 78 nb = args[0]
58 err = [] 79 err = []
@@ -60,7 +81,7 @@ revision specified in the manifest.
60 if not opt.all: 81 if not opt.all:
61 projects = args[1:] 82 projects = args[1:]
62 if len(projects) < 1: 83 if len(projects) < 1:
63 projects = ['.',] # start it in the local project by default 84 projects = ['.'] # start it in the local project by default
64 85
65 all_projects = self.GetProjects(projects, 86 all_projects = self.GetProjects(projects,
66 missing_ok=bool(self.gitc_manifest)) 87 missing_ok=bool(self.gitc_manifest))
@@ -84,11 +105,8 @@ revision specified in the manifest.
84 if not os.path.exists(os.getcwd()): 105 if not os.path.exists(os.getcwd()):
85 os.chdir(self.manifest.topdir) 106 os.chdir(self.manifest.topdir)
86 107
87 pm = Progress('Starting %s' % nb, len(all_projects)) 108 pm = Progress('Syncing %s' % nb, len(all_projects), quiet=opt.quiet)
88 for project in all_projects: 109 for project in all_projects:
89 pm.update()
90
91 if self.gitc_manifest:
92 gitc_project = self.gitc_manifest.paths[project.relpath] 110 gitc_project = self.gitc_manifest.paths[project.relpath]
93 # Sync projects that have not been opened. 111 # Sync projects that have not been opened.
94 if not gitc_project.already_synced: 112 if not gitc_project.already_synced:
@@ -101,21 +119,21 @@ revision specified in the manifest.
101 sync_buf = SyncBuffer(self.manifest.manifestProject.config) 119 sync_buf = SyncBuffer(self.manifest.manifestProject.config)
102 project.Sync_LocalHalf(sync_buf) 120 project.Sync_LocalHalf(sync_buf)
103 project.revisionId = gitc_project.old_revision 121 project.revisionId = gitc_project.old_revision
122 pm.update()
123 pm.end()
104 124
105 # If the current revision is immutable, such as a SHA1, a tag or 125 def _ProcessResults(_pool, pm, results):
106 # a change, then we can't push back to it. Substitute with 126 for (result, project) in results:
107 # dest_branch, if defined; or with manifest default revision instead. 127 if not result:
108 branch_merge = '' 128 err.append(project)
109 if IsImmutable(project.revisionExpr): 129 pm.update()
110 if project.dest_branch:
111 branch_merge = project.dest_branch
112 else:
113 branch_merge = self.manifest.default.revisionExpr
114 130
115 if not project.StartBranch( 131 self.ExecuteInParallel(
116 nb, branch_merge=branch_merge, revision=opt.revision): 132 opt.jobs,
117 err.append(project) 133 functools.partial(self._ExecuteOne, opt.revision, nb),
118 pm.end() 134 all_projects,
135 callback=_ProcessResults,
136 output=Progress('Starting %s' % (nb,), len(all_projects), quiet=opt.quiet))
119 137
120 if err: 138 if err:
121 for p in err: 139 for p in err:
diff --git a/subcmds/status.py b/subcmds/status.py
index 63972d72..5b669547 100644
--- a/subcmds/status.py
+++ b/subcmds/status.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,25 +12,19 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function 15import functools
18
19from command import PagedCommand
20
21try:
22 import threading as _threading
23except ImportError:
24 import dummy_threading as _threading
25
26import glob 16import glob
27 17import io
28import itertools
29import os 18import os
30 19
20from command import DEFAULT_LOCAL_JOBS, PagedCommand
21
31from color import Coloring 22from color import Coloring
32import platform_utils 23import platform_utils
33 24
25
34class Status(PagedCommand): 26class Status(PagedCommand):
35 common = True 27 COMMON = True
36 helpSummary = "Show the working tree status" 28 helpSummary = "Show the working tree status"
37 helpUsage = """ 29 helpUsage = """
38%prog [<project>...] 30%prog [<project>...]
@@ -84,36 +76,29 @@ the following meanings:
84 d: deleted ( in index, not in work tree ) 76 d: deleted ( in index, not in work tree )
85 77
86""" 78"""
79 PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
87 80
88 def _Options(self, p): 81 def _Options(self, p):
89 p.add_option('-j', '--jobs',
90 dest='jobs', action='store', type='int', default=2,
91 help="number of projects to check simultaneously")
92 p.add_option('-o', '--orphans', 82 p.add_option('-o', '--orphans',
93 dest='orphans', action='store_true', 83 dest='orphans', action='store_true',
94 help="include objects in working directory outside of repo projects") 84 help="include objects in working directory outside of repo projects")
95 p.add_option('-q', '--quiet', action='store_true',
96 help="only print the name of modified projects")
97 85
98 def _StatusHelper(self, project, clean_counter, sem, quiet): 86 def _StatusHelper(self, quiet, project):
99 """Obtains the status for a specific project. 87 """Obtains the status for a specific project.
100 88
101 Obtains the status for a project, redirecting the output to 89 Obtains the status for a project, redirecting the output to
102 the specified object. It will release the semaphore 90 the specified object.
103 when done.
104 91
105 Args: 92 Args:
93 quiet: Where to output the status.
106 project: Project to get status of. 94 project: Project to get status of.
107 clean_counter: Counter for clean projects. 95
108 sem: Semaphore, will call release() when complete. 96 Returns:
109 output: Where to output the status. 97 The status of the project.
110 """ 98 """
111 try: 99 buf = io.StringIO()
112 state = project.PrintWorkTreeStatus(quiet=quiet) 100 ret = project.PrintWorkTreeStatus(quiet=quiet, output_redir=buf)
113 if state == 'CLEAN': 101 return (ret, buf.getvalue())
114 next(clean_counter)
115 finally:
116 sem.release()
117 102
118 def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring): 103 def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
119 """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'""" 104 """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'"""
@@ -126,34 +111,31 @@ the following meanings:
126 continue 111 continue
127 if item in proj_dirs_parents: 112 if item in proj_dirs_parents:
128 self._FindOrphans(glob.glob('%s/.*' % item) + 113 self._FindOrphans(glob.glob('%s/.*' % item) +
129 glob.glob('%s/*' % item), 114 glob.glob('%s/*' % item),
130 proj_dirs, proj_dirs_parents, outstring) 115 proj_dirs, proj_dirs_parents, outstring)
131 continue 116 continue
132 outstring.append(''.join([status_header, item, '/'])) 117 outstring.append(''.join([status_header, item, '/']))
133 118
134 def Execute(self, opt, args): 119 def Execute(self, opt, args):
135 all_projects = self.GetProjects(args) 120 all_projects = self.GetProjects(args)
136 counter = itertools.count()
137 121
138 if opt.jobs == 1: 122 def _ProcessResults(_pool, _output, results):
139 for project in all_projects: 123 ret = 0
140 state = project.PrintWorkTreeStatus(quiet=opt.quiet) 124 for (state, output) in results:
125 if output:
126 print(output, end='')
141 if state == 'CLEAN': 127 if state == 'CLEAN':
142 next(counter) 128 ret += 1
143 else: 129 return ret
144 sem = _threading.Semaphore(opt.jobs) 130
145 threads = [] 131 counter = self.ExecuteInParallel(
146 for project in all_projects: 132 opt.jobs,
147 sem.acquire() 133 functools.partial(self._StatusHelper, opt.quiet),
148 134 all_projects,
149 t = _threading.Thread(target=self._StatusHelper, 135 callback=_ProcessResults,
150 args=(project, counter, sem, opt.quiet)) 136 ordered=True)
151 threads.append(t) 137
152 t.daemon = True 138 if not opt.quiet and len(all_projects) == counter:
153 t.start()
154 for t in threads:
155 t.join()
156 if not opt.quiet and len(all_projects) == next(counter):
157 print('nothing to commit (working directory clean)') 139 print('nothing to commit (working directory clean)')
158 140
159 if opt.orphans: 141 if opt.orphans:
@@ -170,8 +152,8 @@ the following meanings:
170 class StatusColoring(Coloring): 152 class StatusColoring(Coloring):
171 def __init__(self, config): 153 def __init__(self, config):
172 Coloring.__init__(self, config, 'status') 154 Coloring.__init__(self, config, 'status')
173 self.project = self.printer('header', attr = 'bold') 155 self.project = self.printer('header', attr='bold')
174 self.untracked = self.printer('untracked', fg = 'red') 156 self.untracked = self.printer('untracked', fg='red')
175 157
176 orig_path = os.getcwd() 158 orig_path = os.getcwd()
177 try: 159 try:
@@ -179,11 +161,11 @@ the following meanings:
179 161
180 outstring = [] 162 outstring = []
181 self._FindOrphans(glob.glob('.*') + 163 self._FindOrphans(glob.glob('.*') +
182 glob.glob('*'), 164 glob.glob('*'),
183 proj_dirs, proj_dirs_parents, outstring) 165 proj_dirs, proj_dirs_parents, outstring)
184 166
185 if outstring: 167 if outstring:
186 output = StatusColoring(self.manifest.globalConfig) 168 output = StatusColoring(self.client.globalConfig)
187 output.project('Objects not within a project (orphans)') 169 output.project('Objects not within a project (orphans)')
188 output.nl() 170 output.nl()
189 for entry in outstring: 171 for entry in outstring:
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 2973a16e..3211cbb1 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,37 +12,23 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function 15import errno
16import functools
17import http.cookiejar as cookielib
18import io
18import json 19import json
20import multiprocessing
19import netrc 21import netrc
20from optparse import SUPPRESS_HELP 22from optparse import SUPPRESS_HELP
21import os 23import os
22import re
23import socket 24import socket
24import subprocess
25import sys 25import sys
26import tempfile 26import tempfile
27import time 27import time
28 28import urllib.error
29from pyversion import is_python3 29import urllib.parse
30if is_python3(): 30import urllib.request
31 import http.cookiejar as cookielib 31import xmlrpc.client
32 import urllib.error
33 import urllib.parse
34 import urllib.request
35 import xmlrpc.client
36else:
37 import cookielib
38 import imp
39 import urllib2
40 import urlparse
41 import xmlrpclib
42 urllib = imp.new_module('urllib')
43 urllib.error = urllib2
44 urllib.parse = urlparse
45 urllib.request = urllib2
46 xmlrpc = imp.new_module('xmlrpc')
47 xmlrpc.client = xmlrpclib
48 32
49try: 33try:
50 import threading as _threading 34 import threading as _threading
@@ -53,44 +37,36 @@ except ImportError:
53 37
54try: 38try:
55 import resource 39 import resource
40
56 def _rlimit_nofile(): 41 def _rlimit_nofile():
57 return resource.getrlimit(resource.RLIMIT_NOFILE) 42 return resource.getrlimit(resource.RLIMIT_NOFILE)
58except ImportError: 43except ImportError:
59 def _rlimit_nofile(): 44 def _rlimit_nofile():
60 return (256, 256) 45 return (256, 256)
61 46
62try:
63 import multiprocessing
64except ImportError:
65 multiprocessing = None
66
67import event_log 47import event_log
68from git_command import GIT, git_require 48from git_command import git_require
69from git_config import GetUrlCookieFile 49from git_config import GetUrlCookieFile
70from git_refs import R_HEADS, HEAD 50from git_refs import R_HEADS, HEAD
51import git_superproject
71import gitc_utils 52import gitc_utils
72from project import Project 53from project import Project
73from project import RemoteSpec 54from project import RemoteSpec
74from command import Command, MirrorSafeCommand 55from command import Command, MirrorSafeCommand, WORKER_BATCH_SIZE
75from error import RepoChangedException, GitError, ManifestParseError 56from error import RepoChangedException, GitError, ManifestParseError
76import platform_utils 57import platform_utils
77from project import SyncBuffer 58from project import SyncBuffer
78from progress import Progress 59from progress import Progress
60import ssh
79from wrapper import Wrapper 61from wrapper import Wrapper
80from manifest_xml import GitcManifest 62from manifest_xml import GitcManifest
81 63
82_ONE_DAY_S = 24 * 60 * 60 64_ONE_DAY_S = 24 * 60 * 60
83 65
84class _FetchError(Exception):
85 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
86 pass
87
88class _CheckoutError(Exception):
89 """Internal error thrown in _CheckoutOne() when we don't want stack trace."""
90 66
91class Sync(Command, MirrorSafeCommand): 67class Sync(Command, MirrorSafeCommand):
92 jobs = 1 68 jobs = 1
93 common = True 69 COMMON = True
94 helpSummary = "Update working tree to the latest revision" 70 helpSummary = "Update working tree to the latest revision"
95 helpUsage = """ 71 helpUsage = """
96%prog [<project>...] 72%prog [<project>...]
@@ -133,11 +109,11 @@ if the manifest server specified in the manifest file already includes
133credentials. 109credentials.
134 110
135By default, all projects will be synced. The --fail-fast option can be used 111By default, all projects will be synced. The --fail-fast option can be used
136to halt syncing as soon as possible when the the first project fails to sync. 112to halt syncing as soon as possible when the first project fails to sync.
137 113
138The --force-sync option can be used to overwrite existing git 114The --force-sync option can be used to overwrite existing git
139directories if they have previously been linked to a different 115directories if they have previously been linked to a different
140object direcotry. WARNING: This may cause data to be lost since 116object directory. WARNING: This may cause data to be lost since
141refs may be removed when overwriting. 117refs may be removed when overwriting.
142 118
143The --force-remove-dirty option can be used to remove previously used 119The --force-remove-dirty option can be used to remove previously used
@@ -191,12 +167,21 @@ If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
191later is required to fix a server side protocol bug. 167later is required to fix a server side protocol bug.
192 168
193""" 169"""
170 PARALLEL_JOBS = 1
171
172 def _CommonOptions(self, p):
173 if self.manifest:
174 try:
175 self.PARALLEL_JOBS = self.manifest.default.sync_j
176 except ManifestParseError:
177 pass
178 super()._CommonOptions(p)
194 179
195 def _Options(self, p, show_smart=True): 180 def _Options(self, p, show_smart=True):
196 try: 181 p.add_option('--jobs-network', default=None, type=int, metavar='JOBS',
197 self.jobs = self.manifest.default.sync_j 182 help='number of network jobs to run in parallel (defaults to --jobs)')
198 except ManifestParseError: 183 p.add_option('--jobs-checkout', default=None, type=int, metavar='JOBS',
199 self.jobs = 1 184 help='number of local checkout jobs to run in parallel (defaults to --jobs)')
200 185
201 p.add_option('-f', '--force-broken', 186 p.add_option('-f', '--force-broken',
202 dest='force_broken', action='store_true', 187 dest='force_broken', action='store_true',
@@ -217,6 +202,10 @@ later is required to fix a server side protocol bug.
217 p.add_option('-l', '--local-only', 202 p.add_option('-l', '--local-only',
218 dest='local_only', action='store_true', 203 dest='local_only', action='store_true',
219 help="only update working tree, don't fetch") 204 help="only update working tree, don't fetch")
205 p.add_option('--no-manifest-update', '--nmu',
206 dest='mp_update', action='store_false', default='true',
207 help='use the existing manifest checkout as-is. '
208 '(do not update to the latest revision)')
220 p.add_option('-n', '--network-only', 209 p.add_option('-n', '--network-only',
221 dest='network_only', action='store_true', 210 dest='network_only', action='store_true',
222 help="fetch only, don't update working tree") 211 help="fetch only, don't update working tree")
@@ -226,17 +215,15 @@ later is required to fix a server side protocol bug.
226 p.add_option('-c', '--current-branch', 215 p.add_option('-c', '--current-branch',
227 dest='current_branch_only', action='store_true', 216 dest='current_branch_only', action='store_true',
228 help='fetch only current branch from server') 217 help='fetch only current branch from server')
229 p.add_option('-q', '--quiet', 218 p.add_option('--no-current-branch',
230 dest='quiet', action='store_true', 219 dest='current_branch_only', action='store_false',
231 help='be more quiet') 220 help='fetch all branches from server')
232 p.add_option('-j', '--jobs',
233 dest='jobs', action='store', type='int',
234 help="projects to fetch simultaneously (default %d)" % self.jobs)
235 p.add_option('-m', '--manifest-name', 221 p.add_option('-m', '--manifest-name',
236 dest='manifest_name', 222 dest='manifest_name',
237 help='temporary manifest to use for this sync', metavar='NAME.xml') 223 help='temporary manifest to use for this sync', metavar='NAME.xml')
238 p.add_option('--no-clone-bundle', 224 p.add_option('--clone-bundle', action='store_true',
239 dest='no_clone_bundle', action='store_true', 225 help='enable use of /clone.bundle on HTTP/HTTPS')
226 p.add_option('--no-clone-bundle', dest='clone_bundle', action='store_false',
240 help='disable use of /clone.bundle on HTTP/HTTPS') 227 help='disable use of /clone.bundle on HTTP/HTTPS')
241 p.add_option('-u', '--manifest-server-username', action='store', 228 p.add_option('-u', '--manifest-server-username', action='store',
242 dest='manifest_server_username', 229 dest='manifest_server_username',
@@ -247,12 +234,23 @@ later is required to fix a server side protocol bug.
247 p.add_option('--fetch-submodules', 234 p.add_option('--fetch-submodules',
248 dest='fetch_submodules', action='store_true', 235 dest='fetch_submodules', action='store_true',
249 help='fetch submodules from server') 236 help='fetch submodules from server')
237 p.add_option('--use-superproject', action='store_true',
238 help='use the manifest superproject to sync projects')
239 p.add_option('--no-use-superproject', action='store_false',
240 dest='use_superproject',
241 help='disable use of manifest superprojects')
242 p.add_option('--tags',
243 action='store_false',
244 help='fetch tags')
250 p.add_option('--no-tags', 245 p.add_option('--no-tags',
251 dest='no_tags', action='store_true', 246 dest='tags', action='store_false',
252 help="don't fetch tags") 247 help="don't fetch tags")
253 p.add_option('--optimized-fetch', 248 p.add_option('--optimized-fetch',
254 dest='optimized_fetch', action='store_true', 249 dest='optimized_fetch', action='store_true',
255 help='only fetch projects fixed to sha1 if revision does not exist locally') 250 help='only fetch projects fixed to sha1 if revision does not exist locally')
251 p.add_option('--retry-fetches',
252 default=0, action='store', type='int',
253 help='number of times to retry fetches on transient errors')
256 p.add_option('--prune', dest='prune', action='store_true', 254 p.add_option('--prune', dest='prune', action='store_true',
257 help='delete refs that no longer exist on the remote') 255 help='delete refs that no longer exist on the remote')
258 if show_smart: 256 if show_smart:
@@ -265,345 +263,400 @@ later is required to fix a server side protocol bug.
265 263
266 g = p.add_option_group('repo Version options') 264 g = p.add_option_group('repo Version options')
267 g.add_option('--no-repo-verify', 265 g.add_option('--no-repo-verify',
268 dest='no_repo_verify', action='store_true', 266 dest='repo_verify', default=True, action='store_false',
269 help='do not verify repo source code') 267 help='do not verify repo source code')
270 g.add_option('--repo-upgraded', 268 g.add_option('--repo-upgraded',
271 dest='repo_upgraded', action='store_true', 269 dest='repo_upgraded', action='store_true',
272 help=SUPPRESS_HELP) 270 help=SUPPRESS_HELP)
273 271
274 def _FetchProjectList(self, opt, projects, sem, *args, **kwargs): 272 def _GetBranch(self):
275 """Main function of the fetch threads. 273 """Returns the branch name for getting the approved manifest."""
274 p = self.manifest.manifestProject
275 b = p.GetBranch(p.CurrentBranch)
276 branch = b.merge
277 if branch.startswith(R_HEADS):
278 branch = branch[len(R_HEADS):]
279 return branch
280
281 def _GetCurrentBranchOnly(self, opt):
282 """Returns True if current-branch or use-superproject options are enabled."""
283 return opt.current_branch_only or git_superproject.UseSuperproject(opt, self.manifest)
284
285 def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data):
286 """Update revisionId of every project with the SHA from superproject.
287
288 This function updates each project's revisionId with SHA from superproject.
289 It writes the updated manifest into a file and reloads the manifest from it.
290
291 Args:
292 opt: Program options returned from optparse. See _Options().
293 args: Arguments to pass to GetProjects. See the GetProjects
294 docstring for details.
295 load_local_manifests: Whether to load local manifests.
296 superproject_logging_data: A dictionary of superproject data that is to be logged.
297
298 Returns:
299 Returns path to the overriding manifest file instead of None.
300 """
301 print_messages = git_superproject.PrintMessages(opt, self.manifest)
302 superproject = git_superproject.Superproject(self.manifest,
303 self.repodir,
304 self.git_event_log,
305 quiet=opt.quiet,
306 print_messages=print_messages)
307 if opt.local_only:
308 manifest_path = superproject.manifest_path
309 if manifest_path:
310 self._ReloadManifest(manifest_path, load_local_manifests)
311 return manifest_path
312
313 all_projects = self.GetProjects(args,
314 missing_ok=True,
315 submodules_ok=opt.fetch_submodules)
316 update_result = superproject.UpdateProjectsRevisionId(all_projects)
317 manifest_path = update_result.manifest_path
318 superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
319 if manifest_path:
320 self._ReloadManifest(manifest_path, load_local_manifests)
321 else:
322 if print_messages:
323 print('warning: Update of revisionId from superproject has failed, '
324 'repo sync will not use superproject to fetch the source. ',
325 'Please resync with the --no-use-superproject option to avoid this repo warning.',
326 file=sys.stderr)
327 if update_result.fatal and opt.use_superproject is not None:
328 sys.exit(1)
329 return manifest_path
330
331 def _FetchProjectList(self, opt, projects):
332 """Main function of the fetch worker.
333
334 The projects we're given share the same underlying git object store, so we
335 have to fetch them in serial.
276 336
277 Delegates most of the work to _FetchHelper. 337 Delegates most of the work to _FetchHelper.
278 338
279 Args: 339 Args:
280 opt: Program options returned from optparse. See _Options(). 340 opt: Program options returned from optparse. See _Options().
281 projects: Projects to fetch. 341 projects: Projects to fetch.
282 sem: We'll release() this semaphore when we exit so that another thread
283 can be started up.
284 *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
285 _FetchHelper docstring for details.
286 """ 342 """
287 try: 343 return [self._FetchOne(opt, x) for x in projects]
288 for project in projects:
289 success = self._FetchHelper(opt, project, *args, **kwargs)
290 if not success and opt.fail_fast:
291 break
292 finally:
293 sem.release()
294 344
295 def _FetchHelper(self, opt, project, lock, fetched, pm, err_event, 345 def _FetchOne(self, opt, project):
296 clone_filter):
297 """Fetch git objects for a single project. 346 """Fetch git objects for a single project.
298 347
299 Args: 348 Args:
300 opt: Program options returned from optparse. See _Options(). 349 opt: Program options returned from optparse. See _Options().
301 project: Project object for the project to fetch. 350 project: Project object for the project to fetch.
302 lock: Lock for accessing objects that are shared amongst multiple
303 _FetchHelper() threads.
304 fetched: set object that we will add project.gitdir to when we're done
305 (with our lock held).
306 pm: Instance of a Project object. We will call pm.update() (with our
307 lock held).
308 err_event: We'll set this event in the case of an error (after printing
309 out info about the error).
310 clone_filter: Filter for use in a partial clone.
311 351
312 Returns: 352 Returns:
313 Whether the fetch was successful. 353 Whether the fetch was successful.
314 """ 354 """
315 # We'll set to true once we've locked the lock.
316 did_lock = False
317
318 # Encapsulate everything in a try/except/finally so that:
319 # - We always set err_event in the case of an exception.
320 # - We always make sure we unlock the lock if we locked it.
321 start = time.time() 355 start = time.time()
322 success = False 356 success = False
357 buf = io.StringIO()
323 try: 358 try:
324 try: 359 success = project.Sync_NetworkHalf(
325 success = project.Sync_NetworkHalf(
326 quiet=opt.quiet, 360 quiet=opt.quiet,
327 current_branch_only=opt.current_branch_only, 361 verbose=opt.verbose,
362 output_redir=buf,
363 current_branch_only=self._GetCurrentBranchOnly(opt),
328 force_sync=opt.force_sync, 364 force_sync=opt.force_sync,
329 clone_bundle=not opt.no_clone_bundle, 365 clone_bundle=opt.clone_bundle,
330 no_tags=opt.no_tags, archive=self.manifest.IsArchive, 366 tags=opt.tags, archive=self.manifest.IsArchive,
331 optimized_fetch=opt.optimized_fetch, 367 optimized_fetch=opt.optimized_fetch,
368 retry_fetches=opt.retry_fetches,
332 prune=opt.prune, 369 prune=opt.prune,
333 clone_filter=clone_filter) 370 ssh_proxy=self.ssh_proxy,
334 self._fetch_times.Set(project, time.time() - start) 371 clone_filter=self.manifest.CloneFilter,
372 partial_clone_exclude=self.manifest.PartialCloneExclude)
335 373
336 # Lock around all the rest of the code, since printing, updating a set 374 output = buf.getvalue()
337 # and Progress.update() are not thread safe. 375 if (opt.verbose or not success) and output:
338 lock.acquire() 376 print('\n' + output.rstrip())
339 did_lock = True
340 377
341 if not success: 378 if not success:
342 err_event.set() 379 print('error: Cannot fetch %s from %s'
343 print('error: Cannot fetch %s from %s' 380 % (project.name, project.remote.url),
344 % (project.name, project.remote.url), 381 file=sys.stderr)
345 file=sys.stderr) 382 except GitError as e:
346 if opt.fail_fast: 383 print('error.GitError: Cannot fetch %s' % str(e), file=sys.stderr)
347 raise _FetchError() 384 except Exception as e:
348 385 print('error: Cannot fetch %s (%s: %s)'
349 fetched.add(project.gitdir)
350 pm.update(msg=project.name)
351 except _FetchError:
352 pass
353 except Exception as e:
354 print('error: Cannot fetch %s (%s: %s)' \
355 % (project.name, type(e).__name__, str(e)), file=sys.stderr) 386 % (project.name, type(e).__name__, str(e)), file=sys.stderr)
356 err_event.set() 387 raise
357 raise 388
358 finally: 389 finish = time.time()
359 if did_lock: 390 return (success, project, start, finish)
360 lock.release()
361 finish = time.time()
362 self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
363 start, finish, success)
364 391
365 return success 392 @classmethod
393 def _FetchInitChild(cls, ssh_proxy):
394 cls.ssh_proxy = ssh_proxy
366 395
367 def _Fetch(self, projects, opt): 396 def _Fetch(self, projects, opt, err_event, ssh_proxy):
397 ret = True
398
399 jobs = opt.jobs_network if opt.jobs_network else self.jobs
368 fetched = set() 400 fetched = set()
369 lock = _threading.Lock() 401 pm = Progress('Fetching', len(projects), delay=False, quiet=opt.quiet)
370 pm = Progress('Fetching projects', len(projects),
371 always_print_percentage=opt.quiet)
372 402
373 objdir_project_map = dict() 403 objdir_project_map = dict()
374 for project in projects: 404 for project in projects:
375 objdir_project_map.setdefault(project.objdir, []).append(project) 405 objdir_project_map.setdefault(project.objdir, []).append(project)
376 406 projects_list = list(objdir_project_map.values())
377 threads = set() 407
378 sem = _threading.Semaphore(self.jobs) 408 def _ProcessResults(results_sets):
379 err_event = _threading.Event() 409 ret = True
380 for project_list in objdir_project_map.values(): 410 for results in results_sets:
381 # Check for any errors before running any more tasks. 411 for (success, project, start, finish) in results:
382 # ...we'll let existing threads finish, though. 412 self._fetch_times.Set(project, finish - start)
383 if err_event.isSet() and opt.fail_fast: 413 self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
384 break 414 start, finish, success)
385 415 # Check for any errors before running any more tasks.
386 sem.acquire() 416 # ...we'll let existing jobs finish, though.
387 kwargs = dict(opt=opt, 417 if not success:
388 projects=project_list, 418 ret = False
389 sem=sem, 419 else:
390 lock=lock, 420 fetched.add(project.gitdir)
391 fetched=fetched, 421 pm.update(msg=project.name)
392 pm=pm, 422 if not ret and opt.fail_fast:
393 err_event=err_event, 423 break
394 clone_filter=self.manifest.CloneFilter) 424 return ret
395 if self.jobs > 1: 425
396 t = _threading.Thread(target = self._FetchProjectList, 426 # We pass the ssh proxy settings via the class. This allows multiprocessing
397 kwargs = kwargs) 427 # to pickle it up when spawning children. We can't pass it as an argument
398 # Ensure that Ctrl-C will not freeze the repo process. 428 # to _FetchProjectList below as multiprocessing is unable to pickle those.
399 t.daemon = True 429 Sync.ssh_proxy = None
400 threads.add(t) 430
401 t.start() 431 # NB: Multiprocessing is heavy, so don't spin it up for one job.
432 if len(projects_list) == 1 or jobs == 1:
433 self._FetchInitChild(ssh_proxy)
434 if not _ProcessResults(self._FetchProjectList(opt, x) for x in projects_list):
435 ret = False
436 else:
437 # Favor throughput over responsiveness when quiet. It seems that imap()
438 # will yield results in batches relative to chunksize, so even as the
439 # children finish a sync, we won't see the result until one child finishes
440 # ~chunksize jobs. When using a large --jobs with large chunksize, this
441 # can be jarring as there will be a large initial delay where repo looks
442 # like it isn't doing anything and sits at 0%, but then suddenly completes
443 # a lot of jobs all at once. Since this code is more network bound, we
444 # can accept a bit more CPU overhead with a smaller chunksize so that the
445 # user sees more immediate & continuous feedback.
446 if opt.quiet:
447 chunksize = WORKER_BATCH_SIZE
402 else: 448 else:
403 self._FetchProjectList(**kwargs) 449 pm.update(inc=0, msg='warming up')
404 450 chunksize = 4
405 for t in threads: 451 with multiprocessing.Pool(
406 t.join() 452 jobs, initializer=self._FetchInitChild, initargs=(ssh_proxy,)) as pool:
407 453 results = pool.imap_unordered(
408 # If we saw an error, exit with code 1 so that other scripts can check. 454 functools.partial(self._FetchProjectList, opt),
409 if err_event.isSet() and opt.fail_fast: 455 projects_list,
410 print('\nerror: Exited sync due to fetch errors', file=sys.stderr) 456 chunksize=chunksize)
411 sys.exit(1) 457 if not _ProcessResults(results):
458 ret = False
459 pool.close()
460
461 # Cleanup the reference now that we're done with it, and we're going to
462 # release any resources it points to. If we don't, later multiprocessing
463 # usage (e.g. checkouts) will try to pickle and then crash.
464 del Sync.ssh_proxy
412 465
413 pm.end() 466 pm.end()
414 self._fetch_times.Save() 467 self._fetch_times.Save()
415 468
416 if not self.manifest.IsArchive: 469 if not self.manifest.IsArchive:
417 self._GCProjects(projects) 470 self._GCProjects(projects, opt, err_event)
418 471
419 return fetched 472 return (ret, fetched)
420 473
421 def _CheckoutWorker(self, opt, sem, project, *args, **kwargs): 474 def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
422 """Main function of the fetch threads. 475 load_local_manifests, ssh_proxy):
423 476 """The main network fetch loop.
424 Delegates most of the work to _CheckoutOne.
425 477
426 Args: 478 Args:
427 opt: Program options returned from optparse. See _Options(). 479 opt: Program options returned from optparse. See _Options().
428 projects: Projects to fetch. 480 args: Command line args used to filter out projects.
429 sem: We'll release() this semaphore when we exit so that another thread 481 all_projects: List of all projects that should be fetched.
430 can be started up. 482 err_event: Whether an error was hit while processing.
431 *args, **kwargs: Remaining arguments to pass to _CheckoutOne. See the 483 manifest_name: Manifest file to be reloaded.
432 _CheckoutOne docstring for details. 484 load_local_manifests: Whether to load local manifests.
485 ssh_proxy: SSH manager for clients & masters.
486
487 Returns:
488 List of all projects that should be checked out.
433 """ 489 """
434 try: 490 rp = self.manifest.repoProject
435 return self._CheckoutOne(opt, project, *args, **kwargs)
436 finally:
437 sem.release()
438 491
439 def _CheckoutOne(self, opt, project, lock, pm, err_event, err_results): 492 to_fetch = []
493 now = time.time()
494 if _ONE_DAY_S <= (now - rp.LastFetch):
495 to_fetch.append(rp)
496 to_fetch.extend(all_projects)
497 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
498
499 success, fetched = self._Fetch(to_fetch, opt, err_event, ssh_proxy)
500 if not success:
501 err_event.set()
502
503 _PostRepoFetch(rp, opt.repo_verify)
504 if opt.network_only:
505 # bail out now; the rest touches the working tree
506 if err_event.is_set():
507 print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
508 sys.exit(1)
509 return
510
511 # Iteratively fetch missing and/or nested unregistered submodules
512 previously_missing_set = set()
513 while True:
514 self._ReloadManifest(manifest_name, load_local_manifests)
515 all_projects = self.GetProjects(args,
516 missing_ok=True,
517 submodules_ok=opt.fetch_submodules)
518 missing = []
519 for project in all_projects:
520 if project.gitdir not in fetched:
521 missing.append(project)
522 if not missing:
523 break
524 # Stop us from non-stopped fetching actually-missing repos: If set of
525 # missing repos has not been changed from last fetch, we break.
526 missing_set = set(p.name for p in missing)
527 if previously_missing_set == missing_set:
528 break
529 previously_missing_set = missing_set
530 success, new_fetched = self._Fetch(missing, opt, err_event, ssh_proxy)
531 if not success:
532 err_event.set()
533 fetched.update(new_fetched)
534
535 return all_projects
536
537 def _CheckoutOne(self, detach_head, force_sync, project):
440 """Checkout work tree for one project 538 """Checkout work tree for one project
441 539
442 Args: 540 Args:
443 opt: Program options returned from optparse. See _Options(). 541 detach_head: Whether to leave a detached HEAD.
542 force_sync: Force checking out of the repo.
444 project: Project object for the project to checkout. 543 project: Project object for the project to checkout.
445 lock: Lock for accessing objects that are shared amongst multiple
446 _CheckoutWorker() threads.
447 pm: Instance of a Project object. We will call pm.update() (with our
448 lock held).
449 err_event: We'll set this event in the case of an error (after printing
450 out info about the error).
451 err_results: A list of strings, paths to git repos where checkout
452 failed.
453 544
454 Returns: 545 Returns:
455 Whether the fetch was successful. 546 Whether the fetch was successful.
456 """ 547 """
457 # We'll set to true once we've locked the lock.
458 did_lock = False
459
460 # Encapsulate everything in a try/except/finally so that:
461 # - We always set err_event in the case of an exception.
462 # - We always make sure we unlock the lock if we locked it.
463 start = time.time() 548 start = time.time()
464 syncbuf = SyncBuffer(self.manifest.manifestProject.config, 549 syncbuf = SyncBuffer(self.manifest.manifestProject.config,
465 detach_head=opt.detach_head) 550 detach_head=detach_head)
466 success = False 551 success = False
467 try: 552 try:
468 try: 553 project.Sync_LocalHalf(syncbuf, force_sync=force_sync)
469 project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync) 554 success = syncbuf.Finish()
470 555 except GitError as e:
471 # Lock around all the rest of the code, since printing, updating a set 556 print('error.GitError: Cannot checkout %s: %s' %
472 # and Progress.update() are not thread safe. 557 (project.name, str(e)), file=sys.stderr)
473 lock.acquire() 558 except Exception as e:
474 success = syncbuf.Finish() 559 print('error: Cannot checkout %s: %s: %s' %
475 did_lock = True 560 (project.name, type(e).__name__, str(e)),
476 561 file=sys.stderr)
477 if not success: 562 raise
478 err_event.set()
479 print('error: Cannot checkout %s' % (project.name),
480 file=sys.stderr)
481 raise _CheckoutError()
482
483 pm.update(msg=project.name)
484 except _CheckoutError:
485 pass
486 except Exception as e:
487 print('error: Cannot checkout %s: %s: %s' %
488 (project.name, type(e).__name__, str(e)),
489 file=sys.stderr)
490 err_event.set()
491 raise
492 finally:
493 if did_lock:
494 if not success:
495 err_results.append(project.relpath)
496 lock.release()
497 finish = time.time()
498 self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
499 start, finish, success)
500 563
501 return success 564 if not success:
565 print('error: Cannot checkout %s' % (project.name), file=sys.stderr)
566 finish = time.time()
567 return (success, project, start, finish)
502 568
503 def _Checkout(self, all_projects, opt): 569 def _Checkout(self, all_projects, opt, err_results):
504 """Checkout projects listed in all_projects 570 """Checkout projects listed in all_projects
505 571
506 Args: 572 Args:
507 all_projects: List of all projects that should be checked out. 573 all_projects: List of all projects that should be checked out.
508 opt: Program options returned from optparse. See _Options(). 574 opt: Program options returned from optparse. See _Options().
575 err_results: A list of strings, paths to git repos where checkout failed.
509 """ 576 """
577 # Only checkout projects with worktrees.
578 all_projects = [x for x in all_projects if x.worktree]
579
580 def _ProcessResults(pool, pm, results):
581 ret = True
582 for (success, project, start, finish) in results:
583 self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
584 start, finish, success)
585 # Check for any errors before running any more tasks.
586 # ...we'll let existing jobs finish, though.
587 if not success:
588 ret = False
589 err_results.append(project.relpath)
590 if opt.fail_fast:
591 if pool:
592 pool.close()
593 return ret
594 pm.update(msg=project.name)
595 return ret
510 596
511 # Perform checkouts in multiple threads when we are using partial clone. 597 return self.ExecuteInParallel(
512 # Without partial clone, all needed git objects are already downloaded, 598 opt.jobs_checkout if opt.jobs_checkout else self.jobs,
513 # in this situation it's better to use only one process because the checkout 599 functools.partial(self._CheckoutOne, opt.detach_head, opt.force_sync),
514 # would be mostly disk I/O; with partial clone, the objects are only 600 all_projects,
515 # downloaded when demanded (at checkout time), which is similar to the 601 callback=_ProcessResults,
516 # Sync_NetworkHalf case and parallelism would be helpful. 602 output=Progress('Checking out', len(all_projects), quiet=opt.quiet)) and not err_results
517 if self.manifest.CloneFilter:
518 syncjobs = self.jobs
519 else:
520 syncjobs = 1
521
522 lock = _threading.Lock()
523 pm = Progress('Checking out projects', len(all_projects))
524
525 threads = set()
526 sem = _threading.Semaphore(syncjobs)
527 err_event = _threading.Event()
528 err_results = []
529
530 for project in all_projects:
531 # Check for any errors before running any more tasks.
532 # ...we'll let existing threads finish, though.
533 if err_event.isSet() and opt.fail_fast:
534 break
535
536 sem.acquire()
537 if project.worktree:
538 kwargs = dict(opt=opt,
539 sem=sem,
540 project=project,
541 lock=lock,
542 pm=pm,
543 err_event=err_event,
544 err_results=err_results)
545 if syncjobs > 1:
546 t = _threading.Thread(target=self._CheckoutWorker,
547 kwargs=kwargs)
548 # Ensure that Ctrl-C will not freeze the repo process.
549 t.daemon = True
550 threads.add(t)
551 t.start()
552 else:
553 self._CheckoutWorker(**kwargs)
554
555 for t in threads:
556 t.join()
557 603
558 pm.end() 604 def _GCProjects(self, projects, opt, err_event):
559 # If we saw an error, exit with code 1 so that other scripts can check. 605 pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
560 if err_event.isSet(): 606 pm.update(inc=0, msg='prescan')
561 print('\nerror: Exited sync due to checkout errors', file=sys.stderr)
562 if err_results:
563 print('Failing repos:\n%s' % '\n'.join(err_results),
564 file=sys.stderr)
565 sys.exit(1)
566 607
567 def _GCProjects(self, projects):
568 gc_gitdirs = {} 608 gc_gitdirs = {}
569 for project in projects: 609 for project in projects:
570 if len(project.manifest.GetProjectsWithName(project.name)) > 1: 610 # Make sure pruning never kicks in with shared projects.
571 print('Shared project %s found, disabling pruning.' % project.name) 611 if (not project.use_git_worktrees and
572 project.bare_git.config('--replace-all', 'gc.pruneExpire', 'never') 612 len(project.manifest.GetProjectsWithName(project.name)) > 1):
613 if not opt.quiet:
614 print('\r%s: Shared project %s found, disabling pruning.' %
615 (project.relpath, project.name))
616 if git_require((2, 7, 0)):
617 project.EnableRepositoryExtension('preciousObjects')
618 else:
619 # This isn't perfect, but it's the best we can do with old git.
620 print('\r%s: WARNING: shared projects are unreliable when using old '
621 'versions of git; please upgrade to git-2.7.0+.'
622 % (project.relpath,),
623 file=sys.stderr)
624 project.config.SetString('gc.pruneExpire', 'never')
573 gc_gitdirs[project.gitdir] = project.bare_git 625 gc_gitdirs[project.gitdir] = project.bare_git
574 626
575 has_dash_c = git_require((1, 7, 2)) 627 pm.update(inc=len(projects) - len(gc_gitdirs), msg='warming up')
576 if multiprocessing and has_dash_c: 628
577 cpu_count = multiprocessing.cpu_count() 629 cpu_count = os.cpu_count()
578 else:
579 cpu_count = 1
580 jobs = min(self.jobs, cpu_count) 630 jobs = min(self.jobs, cpu_count)
581 631
582 if jobs < 2: 632 if jobs < 2:
583 for bare_git in gc_gitdirs.values(): 633 for bare_git in gc_gitdirs.values():
634 pm.update(msg=bare_git._project.name)
584 bare_git.gc('--auto') 635 bare_git.gc('--auto')
636 pm.end()
585 return 637 return
586 638
587 config = {'pack.threads': cpu_count // jobs if cpu_count > jobs else 1} 639 config = {'pack.threads': cpu_count // jobs if cpu_count > jobs else 1}
588 640
589 threads = set() 641 threads = set()
590 sem = _threading.Semaphore(jobs) 642 sem = _threading.Semaphore(jobs)
591 err_event = _threading.Event()
592 643
593 def GC(bare_git): 644 def GC(bare_git):
645 pm.start(bare_git._project.name)
594 try: 646 try:
595 try: 647 try:
596 bare_git.gc('--auto', config=config) 648 bare_git.gc('--auto', config=config)
597 except GitError: 649 except GitError:
598 err_event.set() 650 err_event.set()
599 except: 651 except Exception:
600 err_event.set() 652 err_event.set()
601 raise 653 raise
602 finally: 654 finally:
655 pm.finish(bare_git._project.name)
603 sem.release() 656 sem.release()
604 657
605 for bare_git in gc_gitdirs.values(): 658 for bare_git in gc_gitdirs.values():
606 if err_event.isSet(): 659 if err_event.is_set() and opt.fail_fast:
607 break 660 break
608 sem.acquire() 661 sem.acquire()
609 t = _threading.Thread(target=GC, args=(bare_git,)) 662 t = _threading.Thread(target=GC, args=(bare_git,))
@@ -613,84 +666,30 @@ later is required to fix a server side protocol bug.
613 666
614 for t in threads: 667 for t in threads:
615 t.join() 668 t.join()
669 pm.end()
616 670
617 if err_event.isSet(): 671 def _ReloadManifest(self, manifest_name=None, load_local_manifests=True):
618 print('\nerror: Exited sync due to gc errors', file=sys.stderr) 672 """Reload the manfiest from the file specified by the |manifest_name|.
619 sys.exit(1) 673
674 It unloads the manifest if |manifest_name| is None.
620 675
621 def _ReloadManifest(self, manifest_name=None): 676 Args:
677 manifest_name: Manifest file to be reloaded.
678 load_local_manifests: Whether to load local manifests.
679 """
622 if manifest_name: 680 if manifest_name:
623 # Override calls _Unload already 681 # Override calls _Unload already
624 self.manifest.Override(manifest_name) 682 self.manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
625 else: 683 else:
626 self.manifest._Unload() 684 self.manifest._Unload()
627 685
628 def _DeleteProject(self, path):
629 print('Deleting obsolete path %s' % path, file=sys.stderr)
630
631 # Delete the .git directory first, so we're less likely to have a partially
632 # working git repository around. There shouldn't be any git projects here,
633 # so rmtree works.
634 try:
635 platform_utils.rmtree(os.path.join(path, '.git'))
636 except OSError as e:
637 print('Failed to remove %s (%s)' % (os.path.join(path, '.git'), str(e)), file=sys.stderr)
638 print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
639 print(' remove manually, then run sync again', file=sys.stderr)
640 return 1
641
642 # Delete everything under the worktree, except for directories that contain
643 # another git project
644 dirs_to_remove = []
645 failed = False
646 for root, dirs, files in platform_utils.walk(path):
647 for f in files:
648 try:
649 platform_utils.remove(os.path.join(root, f))
650 except OSError as e:
651 print('Failed to remove %s (%s)' % (os.path.join(root, f), str(e)), file=sys.stderr)
652 failed = True
653 dirs[:] = [d for d in dirs
654 if not os.path.lexists(os.path.join(root, d, '.git'))]
655 dirs_to_remove += [os.path.join(root, d) for d in dirs
656 if os.path.join(root, d) not in dirs_to_remove]
657 for d in reversed(dirs_to_remove):
658 if platform_utils.islink(d):
659 try:
660 platform_utils.remove(d)
661 except OSError as e:
662 print('Failed to remove %s (%s)' % (os.path.join(root, d), str(e)), file=sys.stderr)
663 failed = True
664 elif len(platform_utils.listdir(d)) == 0:
665 try:
666 platform_utils.rmdir(d)
667 except OSError as e:
668 print('Failed to remove %s (%s)' % (os.path.join(root, d), str(e)), file=sys.stderr)
669 failed = True
670 continue
671 if failed:
672 print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
673 print(' remove manually, then run sync again', file=sys.stderr)
674 return 1
675
676 # Try deleting parent dirs if they are empty
677 project_dir = path
678 while project_dir != self.manifest.topdir:
679 if len(platform_utils.listdir(project_dir)) == 0:
680 platform_utils.rmdir(project_dir)
681 else:
682 break
683 project_dir = os.path.dirname(project_dir)
684
685 return 0
686
687 def UpdateProjectList(self, opt): 686 def UpdateProjectList(self, opt):
688 new_project_paths = [] 687 new_project_paths = []
689 for project in self.GetProjects(None, missing_ok=True): 688 for project in self.GetProjects(None, missing_ok=True):
690 if project.relpath: 689 if project.relpath:
691 new_project_paths.append(project.relpath) 690 new_project_paths.append(project.relpath)
692 file_name = 'project.list' 691 file_name = 'project.list'
693 file_path = os.path.join(self.manifest.repodir, file_name) 692 file_path = os.path.join(self.repodir, file_name)
694 old_project_paths = [] 693 old_project_paths = []
695 694
696 if os.path.exists(file_path): 695 if os.path.exists(file_path):
@@ -705,28 +704,20 @@ later is required to fix a server side protocol bug.
705 gitdir = os.path.join(self.manifest.topdir, path, '.git') 704 gitdir = os.path.join(self.manifest.topdir, path, '.git')
706 if os.path.exists(gitdir): 705 if os.path.exists(gitdir):
707 project = Project( 706 project = Project(
708 manifest = self.manifest, 707 manifest=self.manifest,
709 name = path, 708 name=path,
710 remote = RemoteSpec('origin'), 709 remote=RemoteSpec('origin'),
711 gitdir = gitdir, 710 gitdir=gitdir,
712 objdir = gitdir, 711 objdir=gitdir,
713 worktree = os.path.join(self.manifest.topdir, path), 712 use_git_worktrees=os.path.isfile(gitdir),
714 relpath = path, 713 worktree=os.path.join(self.manifest.topdir, path),
715 revisionExpr = 'HEAD', 714 relpath=path,
716 revisionId = None, 715 revisionExpr='HEAD',
717 groups = None) 716 revisionId=None,
718 717 groups=None)
719 if project.IsDirty() and opt.force_remove_dirty: 718 if not project.DeleteWorktree(
720 print('WARNING: Removing dirty project "%s": uncommitted changes ' 719 quiet=opt.quiet,
721 'erased' % project.relpath, file=sys.stderr) 720 force=opt.force_remove_dirty):
722 self._DeleteProject(project.worktree)
723 elif project.IsDirty():
724 print('error: Cannot remove project "%s": uncommitted changes '
725 'are present' % project.relpath, file=sys.stderr)
726 print(' commit changes, then run sync again',
727 file=sys.stderr)
728 return 1
729 elif self._DeleteProject(project.worktree):
730 return 1 721 return 1
731 722
732 new_project_paths.sort() 723 new_project_paths.sort()
@@ -735,6 +726,56 @@ later is required to fix a server side protocol bug.
735 fd.write('\n') 726 fd.write('\n')
736 return 0 727 return 0
737 728
729 def UpdateCopyLinkfileList(self):
730 """Save all dests of copyfile and linkfile, and update them if needed.
731
732 Returns:
733 Whether update was successful.
734 """
735 new_paths = {}
736 new_linkfile_paths = []
737 new_copyfile_paths = []
738 for project in self.GetProjects(None, missing_ok=True):
739 new_linkfile_paths.extend(x.dest for x in project.linkfiles)
740 new_copyfile_paths.extend(x.dest for x in project.copyfiles)
741
742 new_paths = {
743 'linkfile': new_linkfile_paths,
744 'copyfile': new_copyfile_paths,
745 }
746
747 copylinkfile_name = 'copy-link-files.json'
748 copylinkfile_path = os.path.join(self.manifest.repodir, copylinkfile_name)
749 old_copylinkfile_paths = {}
750
751 if os.path.exists(copylinkfile_path):
752 with open(copylinkfile_path, 'rb') as fp:
753 try:
754 old_copylinkfile_paths = json.load(fp)
755 except:
756 print('error: %s is not a json formatted file.' %
757 copylinkfile_path, file=sys.stderr)
758 platform_utils.remove(copylinkfile_path)
759 return False
760
761 need_remove_files = []
762 need_remove_files.extend(
763 set(old_copylinkfile_paths.get('linkfile', [])) -
764 set(new_linkfile_paths))
765 need_remove_files.extend(
766 set(old_copylinkfile_paths.get('copyfile', [])) -
767 set(new_copyfile_paths))
768
769 for need_remove_file in need_remove_files:
770 # Try to remove the updated copyfile or linkfile.
771 # So, if the file is not exist, nothing need to do.
772 platform_utils.remove(need_remove_file, missing_ok=True)
773
774 # Create copy-link-files.json, save dest path of "copyfile" and "linkfile".
775 with open(copylinkfile_path, 'w', encoding='utf-8') as fp:
776 json.dump(new_paths, fp)
777 return True
778
738 def _SmartSyncSetup(self, opt, smart_sync_manifest_path): 779 def _SmartSyncSetup(self, opt, smart_sync_manifest_path):
739 if not self.manifest.manifest_server: 780 if not self.manifest.manifest_server:
740 print('error: cannot smart sync: no manifest server defined in ' 781 print('error: cannot smart sync: no manifest server defined in '
@@ -745,7 +786,7 @@ later is required to fix a server side protocol bug.
745 if not opt.quiet: 786 if not opt.quiet:
746 print('Using manifest server %s' % manifest_server) 787 print('Using manifest server %s' % manifest_server)
747 788
748 if not '@' in manifest_server: 789 if '@' not in manifest_server:
749 username = None 790 username = None
750 password = None 791 password = None
751 if opt.manifest_server_username and opt.manifest_server_password: 792 if opt.manifest_server_username and opt.manifest_server_password:
@@ -782,19 +823,15 @@ later is required to fix a server side protocol bug.
782 try: 823 try:
783 server = xmlrpc.client.Server(manifest_server, transport=transport) 824 server = xmlrpc.client.Server(manifest_server, transport=transport)
784 if opt.smart_sync: 825 if opt.smart_sync:
785 p = self.manifest.manifestProject 826 branch = self._GetBranch()
786 b = p.GetBranch(p.CurrentBranch) 827
787 branch = b.merge 828 if 'SYNC_TARGET' in os.environ:
788 if branch.startswith(R_HEADS): 829 target = os.environ['SYNC_TARGET']
789 branch = branch[len(R_HEADS):]
790
791 env = os.environ.copy()
792 if 'SYNC_TARGET' in env:
793 target = env['SYNC_TARGET']
794 [success, manifest_str] = server.GetApprovedManifest(branch, target) 830 [success, manifest_str] = server.GetApprovedManifest(branch, target)
795 elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env: 831 elif ('TARGET_PRODUCT' in os.environ and
796 target = '%s-%s' % (env['TARGET_PRODUCT'], 832 'TARGET_BUILD_VARIANT' in os.environ):
797 env['TARGET_BUILD_VARIANT']) 833 target = '%s-%s' % (os.environ['TARGET_PRODUCT'],
834 os.environ['TARGET_BUILD_VARIANT'])
798 [success, manifest_str] = server.GetApprovedManifest(branch, target) 835 [success, manifest_str] = server.GetApprovedManifest(branch, target)
799 else: 836 else:
800 [success, manifest_str] = server.GetApprovedManifest(branch) 837 [success, manifest_str] = server.GetApprovedManifest(branch)
@@ -833,12 +870,15 @@ later is required to fix a server side protocol bug.
833 """Fetch & update the local manifest project.""" 870 """Fetch & update the local manifest project."""
834 if not opt.local_only: 871 if not opt.local_only:
835 start = time.time() 872 start = time.time()
836 success = mp.Sync_NetworkHalf(quiet=opt.quiet, 873 success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
837 current_branch_only=opt.current_branch_only, 874 current_branch_only=self._GetCurrentBranchOnly(opt),
838 no_tags=opt.no_tags, 875 force_sync=opt.force_sync,
876 tags=opt.tags,
839 optimized_fetch=opt.optimized_fetch, 877 optimized_fetch=opt.optimized_fetch,
878 retry_fetches=opt.retry_fetches,
840 submodules=self.manifest.HasSubmodules, 879 submodules=self.manifest.HasSubmodules,
841 clone_filter=self.manifest.CloneFilter) 880 clone_filter=self.manifest.CloneFilter,
881 partial_clone_exclude=self.manifest.PartialCloneExclude)
842 finish = time.time() 882 finish = time.time()
843 self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK, 883 self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
844 start, finish, success) 884 start, finish, success)
@@ -852,7 +892,7 @@ later is required to fix a server side protocol bug.
852 start, time.time(), clean) 892 start, time.time(), clean)
853 if not clean: 893 if not clean:
854 sys.exit(1) 894 sys.exit(1)
855 self._ReloadManifest(opt.manifest_name) 895 self._ReloadManifest(manifest_name)
856 if opt.jobs is None: 896 if opt.jobs is None:
857 self.jobs = self.manifest.default.sync_j 897 self.jobs = self.manifest.default.sync_j
858 898
@@ -886,7 +926,10 @@ later is required to fix a server side protocol bug.
886 926
887 manifest_name = opt.manifest_name 927 manifest_name = opt.manifest_name
888 smart_sync_manifest_path = os.path.join( 928 smart_sync_manifest_path = os.path.join(
889 self.manifest.manifestProject.worktree, 'smart_sync_override.xml') 929 self.manifest.manifestProject.worktree, 'smart_sync_override.xml')
930
931 if opt.clone_bundle is None:
932 opt.clone_bundle = self.manifest.CloneBundle
890 933
891 if opt.smart_sync or opt.smart_tag: 934 if opt.smart_sync or opt.smart_tag:
892 manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path) 935 manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path)
@@ -898,8 +941,17 @@ later is required to fix a server side protocol bug.
898 print('error: failed to remove existing smart sync override manifest: %s' % 941 print('error: failed to remove existing smart sync override manifest: %s' %
899 e, file=sys.stderr) 942 e, file=sys.stderr)
900 943
944 err_event = multiprocessing.Event()
945
901 rp = self.manifest.repoProject 946 rp = self.manifest.repoProject
902 rp.PreSync() 947 rp.PreSync()
948 cb = rp.CurrentBranch
949 if cb:
950 base = rp.GetBranch(cb).merge
951 if not base or not base.startswith('refs/heads/'):
952 print('warning: repo is not tracking a remote branch, so it will not '
953 'receive updates; run `repo init --repo-rev=stable` to fix.',
954 file=sys.stderr)
903 955
904 mp = self.manifest.manifestProject 956 mp = self.manifest.manifestProject
905 mp.PreSync() 957 mp.PreSync()
@@ -907,7 +959,21 @@ later is required to fix a server side protocol bug.
907 if opt.repo_upgraded: 959 if opt.repo_upgraded:
908 _PostRepoUpgrade(self.manifest, quiet=opt.quiet) 960 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
909 961
910 self._UpdateManifestProject(opt, mp, manifest_name) 962 if not opt.mp_update:
963 print('Skipping update of local manifest project.')
964 else:
965 self._UpdateManifestProject(opt, mp, manifest_name)
966
967 load_local_manifests = not self.manifest.HasLocalManifests
968 use_superproject = git_superproject.UseSuperproject(opt, self.manifest)
969 superproject_logging_data = {
970 'superproject': use_superproject,
971 'haslocalmanifests': bool(self.manifest.HasLocalManifests),
972 'hassuperprojecttag': bool(self.manifest.superproject),
973 }
974 if use_superproject:
975 manifest_name = self._UpdateProjectsRevisionId(
976 opt, args, load_local_manifests, superproject_logging_data) or opt.manifest_name
911 977
912 if self.gitc_manifest: 978 if self.gitc_manifest:
913 gitc_manifest_projects = self.GetProjects(args, 979 gitc_manifest_projects = self.GetProjects(args,
@@ -948,56 +1014,92 @@ later is required to fix a server side protocol bug.
948 missing_ok=True, 1014 missing_ok=True,
949 submodules_ok=opt.fetch_submodules) 1015 submodules_ok=opt.fetch_submodules)
950 1016
1017 err_network_sync = False
1018 err_update_projects = False
1019
951 self._fetch_times = _FetchTimes(self.manifest) 1020 self._fetch_times = _FetchTimes(self.manifest)
952 if not opt.local_only: 1021 if not opt.local_only:
953 to_fetch = [] 1022 with multiprocessing.Manager() as manager:
954 now = time.time() 1023 with ssh.ProxyManager(manager) as ssh_proxy:
955 if _ONE_DAY_S <= (now - rp.LastFetch): 1024 # Initialize the socket dir once in the parent.
956 to_fetch.append(rp) 1025 ssh_proxy.sock()
957 to_fetch.extend(all_projects) 1026 all_projects = self._FetchMain(opt, args, all_projects, err_event,
958 to_fetch.sort(key=self._fetch_times.Get, reverse=True) 1027 manifest_name, load_local_manifests,
959 1028 ssh_proxy)
960 fetched = self._Fetch(to_fetch, opt) 1029
961 _PostRepoFetch(rp, opt.no_repo_verify)
962 if opt.network_only: 1030 if opt.network_only:
963 # bail out now; the rest touches the working tree
964 return 1031 return
965 1032
966 # Iteratively fetch missing and/or nested unregistered submodules 1033 # If we saw an error, exit with code 1 so that other scripts can check.
967 previously_missing_set = set() 1034 if err_event.is_set():
968 while True: 1035 err_network_sync = True
969 self._ReloadManifest(manifest_name) 1036 if opt.fail_fast:
970 all_projects = self.GetProjects(args, 1037 print('\nerror: Exited sync due to fetch errors.\n'
971 missing_ok=True, 1038 'Local checkouts *not* updated. Resolve network issues & '
972 submodules_ok=opt.fetch_submodules) 1039 'retry.\n'
973 missing = [] 1040 '`repo sync -l` will update some local checkouts.',
974 for project in all_projects: 1041 file=sys.stderr)
975 if project.gitdir not in fetched: 1042 sys.exit(1)
976 missing.append(project)
977 if not missing:
978 break
979 # Stop us from non-stopped fetching actually-missing repos: If set of
980 # missing repos has not been changed from last fetch, we break.
981 missing_set = set(p.name for p in missing)
982 if previously_missing_set == missing_set:
983 break
984 previously_missing_set = missing_set
985 fetched.update(self._Fetch(missing, opt))
986 1043
987 if self.manifest.IsMirror or self.manifest.IsArchive: 1044 if self.manifest.IsMirror or self.manifest.IsArchive:
988 # bail out now, we have no working tree 1045 # bail out now, we have no working tree
989 return 1046 return
990 1047
991 if self.UpdateProjectList(opt): 1048 if self.UpdateProjectList(opt):
992 sys.exit(1) 1049 err_event.set()
1050 err_update_projects = True
1051 if opt.fail_fast:
1052 print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
1053 sys.exit(1)
993 1054
994 self._Checkout(all_projects, opt) 1055 err_update_linkfiles = not self.UpdateCopyLinkfileList()
1056 if err_update_linkfiles:
1057 err_event.set()
1058 if opt.fail_fast:
1059 print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr)
1060 sys.exit(1)
1061
1062 err_results = []
1063 # NB: We don't exit here because this is the last step.
1064 err_checkout = not self._Checkout(all_projects, opt, err_results)
1065 if err_checkout:
1066 err_event.set()
995 1067
996 # If there's a notice that's supposed to print at the end of the sync, print 1068 # If there's a notice that's supposed to print at the end of the sync, print
997 # it now... 1069 # it now...
998 if self.manifest.notice: 1070 if self.manifest.notice:
999 print(self.manifest.notice) 1071 print(self.manifest.notice)
1000 1072
1073 # If we saw an error, exit with code 1 so that other scripts can check.
1074 if err_event.is_set():
1075 print('\nerror: Unable to fully sync the tree.', file=sys.stderr)
1076 if err_network_sync:
1077 print('error: Downloading network changes failed.', file=sys.stderr)
1078 if err_update_projects:
1079 print('error: Updating local project lists failed.', file=sys.stderr)
1080 if err_update_linkfiles:
1081 print('error: Updating copyfiles or linkfiles failed.', file=sys.stderr)
1082 if err_checkout:
1083 print('error: Checking out local projects failed.', file=sys.stderr)
1084 if err_results:
1085 print('Failing repos:\n%s' % '\n'.join(err_results), file=sys.stderr)
1086 print('Try re-running with "-j1 --fail-fast" to exit at the first error.',
1087 file=sys.stderr)
1088 sys.exit(1)
1089
1090 # Log the previous sync analysis state from the config.
1091 self.git_event_log.LogDataConfigEvents(mp.config.GetSyncAnalysisStateData(),
1092 'previous_sync_state')
1093
1094 # Update and log with the new sync analysis state.
1095 mp.config.UpdateSyncAnalysisState(opt, superproject_logging_data)
1096 self.git_event_log.LogDataConfigEvents(mp.config.GetSyncAnalysisStateData(),
1097 'current_sync_state')
1098
1099 if not opt.quiet:
1100 print('repo sync has finished successfully.')
1101
1102
1001def _PostRepoUpgrade(manifest, quiet=False): 1103def _PostRepoUpgrade(manifest, quiet=False):
1002 wrapper = Wrapper() 1104 wrapper = Wrapper()
1003 if wrapper.NeedSetupGnuPG(): 1105 if wrapper.NeedSetupGnuPG():
@@ -1006,15 +1108,29 @@ def _PostRepoUpgrade(manifest, quiet=False):
1006 if project.Exists: 1108 if project.Exists:
1007 project.PostRepoUpgrade() 1109 project.PostRepoUpgrade()
1008 1110
1009def _PostRepoFetch(rp, no_repo_verify=False, verbose=False): 1111
1112def _PostRepoFetch(rp, repo_verify=True, verbose=False):
1010 if rp.HasChanges: 1113 if rp.HasChanges:
1011 print('info: A new version of repo is available', file=sys.stderr) 1114 print('info: A new version of repo is available', file=sys.stderr)
1012 print(file=sys.stderr) 1115 wrapper = Wrapper()
1013 if no_repo_verify or _VerifyTag(rp): 1116 try:
1014 syncbuf = SyncBuffer(rp.config) 1117 rev = rp.bare_git.describe(rp.GetRevisionId())
1015 rp.Sync_LocalHalf(syncbuf) 1118 except GitError:
1016 if not syncbuf.Finish(): 1119 rev = None
1017 sys.exit(1) 1120 _, new_rev = wrapper.check_repo_rev(rp.gitdir, rev, repo_verify=repo_verify)
1121 # See if we're held back due to missing signed tag.
1122 current_revid = rp.bare_git.rev_parse('HEAD')
1123 new_revid = rp.bare_git.rev_parse('--verify', new_rev)
1124 if current_revid != new_revid:
1125 # We want to switch to the new rev, but also not trash any uncommitted
1126 # changes. This helps with local testing/hacking.
1127 # If a local change has been made, we will throw that away.
1128 # We also have to make sure this will switch to an older commit if that's
1129 # the latest tag in order to support release rollback.
1130 try:
1131 rp.work_git.reset('--keep', new_rev)
1132 except GitError as e:
1133 sys.exit(str(e))
1018 print('info: Restarting repo with latest version', file=sys.stderr) 1134 print('info: Restarting repo with latest version', file=sys.stderr)
1019 raise RepoChangedException(['--repo-upgraded']) 1135 raise RepoChangedException(['--repo-upgraded'])
1020 else: 1136 else:
@@ -1024,53 +1140,6 @@ def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
1024 print('repo version %s is current' % rp.work_git.describe(HEAD), 1140 print('repo version %s is current' % rp.work_git.describe(HEAD),
1025 file=sys.stderr) 1141 file=sys.stderr)
1026 1142
1027def _VerifyTag(project):
1028 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
1029 if not os.path.exists(gpg_dir):
1030 print('warning: GnuPG was not available during last "repo init"\n'
1031 'warning: Cannot automatically authenticate repo."""',
1032 file=sys.stderr)
1033 return True
1034
1035 try:
1036 cur = project.bare_git.describe(project.GetRevisionId())
1037 except GitError:
1038 cur = None
1039
1040 if not cur \
1041 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
1042 rev = project.revisionExpr
1043 if rev.startswith(R_HEADS):
1044 rev = rev[len(R_HEADS):]
1045
1046 print(file=sys.stderr)
1047 print("warning: project '%s' branch '%s' is not signed"
1048 % (project.name, rev), file=sys.stderr)
1049 return False
1050
1051 env = os.environ.copy()
1052 env['GIT_DIR'] = project.gitdir.encode()
1053 env['GNUPGHOME'] = gpg_dir.encode()
1054
1055 cmd = [GIT, 'tag', '-v', cur]
1056 proc = subprocess.Popen(cmd,
1057 stdout = subprocess.PIPE,
1058 stderr = subprocess.PIPE,
1059 env = env)
1060 out = proc.stdout.read()
1061 proc.stdout.close()
1062
1063 err = proc.stderr.read()
1064 proc.stderr.close()
1065
1066 if proc.wait() != 0:
1067 print(file=sys.stderr)
1068 print(out, file=sys.stderr)
1069 print(err, file=sys.stderr)
1070 print(file=sys.stderr)
1071 return False
1072 return True
1073
1074 1143
1075class _FetchTimes(object): 1144class _FetchTimes(object):
1076 _ALPHA = 0.5 1145 _ALPHA = 0.5
@@ -1090,7 +1159,7 @@ class _FetchTimes(object):
1090 old = self._times.get(name, t) 1159 old = self._times.get(name, t)
1091 self._seen.add(name) 1160 self._seen.add(name)
1092 a = self._ALPHA 1161 a = self._ALPHA
1093 self._times[name] = (a*t) + ((1-a) * old) 1162 self._times[name] = (a * t) + ((1 - a) * old)
1094 1163
1095 def _Load(self): 1164 def _Load(self):
1096 if self._times is None: 1165 if self._times is None:
@@ -1098,10 +1167,7 @@ class _FetchTimes(object):
1098 with open(self._path) as f: 1167 with open(self._path) as f:
1099 self._times = json.load(f) 1168 self._times = json.load(f)
1100 except (IOError, ValueError): 1169 except (IOError, ValueError):
1101 try: 1170 platform_utils.remove(self._path, missing_ok=True)
1102 platform_utils.remove(self._path)
1103 except OSError:
1104 pass
1105 self._times = {} 1171 self._times = {}
1106 1172
1107 def Save(self): 1173 def Save(self):
@@ -1119,15 +1185,14 @@ class _FetchTimes(object):
1119 with open(self._path, 'w') as f: 1185 with open(self._path, 'w') as f:
1120 json.dump(self._times, f, indent=2) 1186 json.dump(self._times, f, indent=2)
1121 except (IOError, TypeError): 1187 except (IOError, TypeError):
1122 try: 1188 platform_utils.remove(self._path, missing_ok=True)
1123 platform_utils.remove(self._path)
1124 except OSError:
1125 pass
1126 1189
1127# This is a replacement for xmlrpc.client.Transport using urllib2 1190# This is a replacement for xmlrpc.client.Transport using urllib2
1128# and supporting persistent-http[s]. It cannot change hosts from 1191# and supporting persistent-http[s]. It cannot change hosts from
1129# request to request like the normal transport, the real url 1192# request to request like the normal transport, the real url
1130# is passed during initialization. 1193# is passed during initialization.
1194
1195
1131class PersistentTransport(xmlrpc.client.Transport): 1196class PersistentTransport(xmlrpc.client.Transport):
1132 def __init__(self, orig_host): 1197 def __init__(self, orig_host):
1133 self.orig_host = orig_host 1198 self.orig_host = orig_host
@@ -1138,7 +1203,7 @@ class PersistentTransport(xmlrpc.client.Transport):
1138 # Since we're only using them for HTTP, copy the file temporarily, 1203 # Since we're only using them for HTTP, copy the file temporarily,
1139 # stripping those prefixes away. 1204 # stripping those prefixes away.
1140 if cookiefile: 1205 if cookiefile:
1141 tmpcookiefile = tempfile.NamedTemporaryFile() 1206 tmpcookiefile = tempfile.NamedTemporaryFile(mode='w')
1142 tmpcookiefile.write("# HTTP Cookie File") 1207 tmpcookiefile.write("# HTTP Cookie File")
1143 try: 1208 try:
1144 with open(cookiefile) as f: 1209 with open(cookiefile) as f:
@@ -1162,7 +1227,7 @@ class PersistentTransport(xmlrpc.client.Transport):
1162 if proxy: 1227 if proxy:
1163 proxyhandler = urllib.request.ProxyHandler({ 1228 proxyhandler = urllib.request.ProxyHandler({
1164 "http": proxy, 1229 "http": proxy,
1165 "https": proxy }) 1230 "https": proxy})
1166 1231
1167 opener = urllib.request.build_opener( 1232 opener = urllib.request.build_opener(
1168 urllib.request.HTTPCookieProcessor(cookiejar), 1233 urllib.request.HTTPCookieProcessor(cookiejar),
@@ -1219,4 +1284,3 @@ class PersistentTransport(xmlrpc.client.Transport):
1219 1284
1220 def close(self): 1285 def close(self):
1221 pass 1286 pass
1222
diff --git a/subcmds/upload.py b/subcmds/upload.py
index 5c12aaee..c48deab6 100644
--- a/subcmds/upload.py
+++ b/subcmds/upload.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2008 The Android Open Source Project 1# Copyright (C) 2008 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,25 +12,23 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function
18import copy 15import copy
16import functools
17import optparse
19import re 18import re
20import sys 19import sys
21 20
22from command import InteractiveCommand 21from command import DEFAULT_LOCAL_JOBS, InteractiveCommand
23from editor import Editor 22from editor import Editor
24from error import HookError, UploadError 23from error import UploadError
25from git_command import GitCommand 24from git_command import GitCommand
26from project import RepoHook 25from git_refs import R_HEADS
26from hooks import RepoHook
27 27
28from pyversion import is_python3
29if not is_python3():
30 input = raw_input
31else:
32 unicode = str
33 28
34UNUSUAL_COMMIT_THRESHOLD = 5 29UNUSUAL_COMMIT_THRESHOLD = 5
35 30
31
36def _ConfirmManyUploads(multiple_branches=False): 32def _ConfirmManyUploads(multiple_branches=False):
37 if multiple_branches: 33 if multiple_branches:
38 print('ATTENTION: One or more branches has an unusually high number ' 34 print('ATTENTION: One or more branches has an unusually high number '
@@ -44,19 +40,22 @@ def _ConfirmManyUploads(multiple_branches=False):
44 answer = input("If you are sure you intend to do this, type 'yes': ").strip() 40 answer = input("If you are sure you intend to do this, type 'yes': ").strip()
45 return answer == "yes" 41 return answer == "yes"
46 42
43
47def _die(fmt, *args): 44def _die(fmt, *args):
48 msg = fmt % args 45 msg = fmt % args
49 print('error: %s' % msg, file=sys.stderr) 46 print('error: %s' % msg, file=sys.stderr)
50 sys.exit(1) 47 sys.exit(1)
51 48
49
52def _SplitEmails(values): 50def _SplitEmails(values):
53 result = [] 51 result = []
54 for value in values: 52 for value in values:
55 result.extend([s.strip() for s in value.split(',')]) 53 result.extend([s.strip() for s in value.split(',')])
56 return result 54 return result
57 55
56
58class Upload(InteractiveCommand): 57class Upload(InteractiveCommand):
59 common = True 58 COMMON = True
60 helpSummary = "Upload changes for code review" 59 helpSummary = "Upload changes for code review"
61 helpUsage = """ 60 helpUsage = """
62%prog [--re --cc] [<project>]... 61%prog [--re --cc] [<project>]...
@@ -126,74 +125,89 @@ is set to "true" then repo will assume you always want the equivalent
126of the -t option to the repo command. If unset or set to "false" then 125of the -t option to the repo command. If unset or set to "false" then
127repo will make use of only the command line option. 126repo will make use of only the command line option.
128 127
128review.URL.uploadhashtags:
129
130To add hashtags whenever uploading a commit, you can set a per-project
131or global Git option to do so. The value of review.URL.uploadhashtags
132will be used as comma delimited hashtags like the --hashtag option.
133
134review.URL.uploadlabels:
135
136To add labels whenever uploading a commit, you can set a per-project
137or global Git option to do so. The value of review.URL.uploadlabels
138will be used as comma delimited labels like the --label option.
139
140review.URL.uploadnotify:
141
142Control e-mail notifications when uploading.
143https://gerrit-review.googlesource.com/Documentation/user-upload.html#notify
144
129# References 145# References
130 146
131Gerrit Code Review: https://www.gerritcodereview.com/ 147Gerrit Code Review: https://www.gerritcodereview.com/
132 148
133""" 149"""
150 PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
134 151
135 def _Options(self, p): 152 def _Options(self, p):
136 p.add_option('-t', 153 p.add_option('-t',
137 dest='auto_topic', action='store_true', 154 dest='auto_topic', action='store_true',
138 help='Send local branch name to Gerrit Code Review') 155 help='send local branch name to Gerrit Code Review')
156 p.add_option('--hashtag', '--ht',
157 dest='hashtags', action='append', default=[],
158 help='add hashtags (comma delimited) to the review')
159 p.add_option('--hashtag-branch', '--htb',
160 action='store_true',
161 help='add local branch name as a hashtag')
162 p.add_option('-l', '--label',
163 dest='labels', action='append', default=[],
164 help='add a label when uploading')
139 p.add_option('--re', '--reviewers', 165 p.add_option('--re', '--reviewers',
140 type='string', action='append', dest='reviewers', 166 type='string', action='append', dest='reviewers',
141 help='Request reviews from these people.') 167 help='request reviews from these people')
142 p.add_option('--cc', 168 p.add_option('--cc',
143 type='string', action='append', dest='cc', 169 type='string', action='append', dest='cc',
144 help='Also send email to these email addresses.') 170 help='also send email to these email addresses')
145 p.add_option('--br', 171 p.add_option('--br', '--branch',
146 type='string', action='store', dest='branch', 172 type='string', action='store', dest='branch',
147 help='Branch to upload.') 173 help='(local) branch to upload')
148 p.add_option('--cbr', '--current-branch', 174 p.add_option('-c', '--current-branch',
175 dest='current_branch', action='store_true',
176 help='upload current git branch')
177 p.add_option('--no-current-branch',
178 dest='current_branch', action='store_false',
179 help='upload all git branches')
180 # Turn this into a warning & remove this someday.
181 p.add_option('--cbr',
149 dest='current_branch', action='store_true', 182 dest='current_branch', action='store_true',
150 help='Upload current git branch.') 183 help=optparse.SUPPRESS_HELP)
151 p.add_option('-d', '--draft',
152 action='store_true', dest='draft', default=False,
153 help='If specified, upload as a draft.')
154 p.add_option('--ne', '--no-emails', 184 p.add_option('--ne', '--no-emails',
155 action='store_false', dest='notify', default=True, 185 action='store_false', dest='notify', default=True,
156 help='If specified, do not send emails on upload.') 186 help='do not send e-mails on upload')
157 p.add_option('-p', '--private', 187 p.add_option('-p', '--private',
158 action='store_true', dest='private', default=False, 188 action='store_true', dest='private', default=False,
159 help='If specified, upload as a private change.') 189 help='upload as a private change (deprecated; use --wip)')
160 p.add_option('-w', '--wip', 190 p.add_option('-w', '--wip',
161 action='store_true', dest='wip', default=False, 191 action='store_true', dest='wip', default=False,
162 help='If specified, upload as a work-in-progress change.') 192 help='upload as a work-in-progress change')
163 p.add_option('-o', '--push-option', 193 p.add_option('-o', '--push-option',
164 type='string', action='append', dest='push_options', 194 type='string', action='append', dest='push_options',
165 default=[], 195 default=[],
166 help='Additional push options to transmit') 196 help='additional push options to transmit')
167 p.add_option('-D', '--destination', '--dest', 197 p.add_option('-D', '--destination', '--dest',
168 type='string', action='store', dest='dest_branch', 198 type='string', action='store', dest='dest_branch',
169 metavar='BRANCH', 199 metavar='BRANCH',
170 help='Submit for review on this target branch.') 200 help='submit for review on this target branch')
171 201 p.add_option('-n', '--dry-run',
172 # Options relating to upload hook. Note that verify and no-verify are NOT 202 dest='dryrun', default=False, action='store_true',
173 # opposites of each other, which is why they store to different locations. 203 help='do everything except actually upload the CL')
174 # We are using them to match 'git commit' syntax. 204 p.add_option('-y', '--yes',
175 # 205 default=False, action='store_true',
176 # Combinations: 206 help='answer yes to all safe prompts')
177 # - no-verify=False, verify=False (DEFAULT):
178 # If stdout is a tty, can prompt about running upload hooks if needed.
179 # If user denies running hooks, the upload is cancelled. If stdout is
180 # not a tty and we would need to prompt about upload hooks, upload is
181 # cancelled.
182 # - no-verify=False, verify=True:
183 # Always run upload hooks with no prompt.
184 # - no-verify=True, verify=False:
185 # Never run upload hooks, but upload anyway (AKA bypass hooks).
186 # - no-verify=True, verify=True:
187 # Invalid
188 p.add_option('--no-cert-checks', 207 p.add_option('--no-cert-checks',
189 dest='validate_certs', action='store_false', default=True, 208 dest='validate_certs', action='store_false', default=True,
190 help='Disable verifying ssl certs (unsafe).') 209 help='disable verifying ssl certs (unsafe)')
191 p.add_option('--no-verify', 210 RepoHook.AddOptionGroup(p, 'pre-upload')
192 dest='bypass_hooks', action='store_true',
193 help='Do not run the upload hook.')
194 p.add_option('--verify',
195 dest='allow_all_hooks', action='store_true',
196 help='Run the upload hook without prompting.')
197 211
198 def _SingleBranch(self, opt, branch, people): 212 def _SingleBranch(self, opt, branch, people):
199 project = branch.project 213 project = branch.project
@@ -212,20 +226,24 @@ Gerrit Code Review: https://www.gerritcodereview.com/
212 226
213 destination = opt.dest_branch or project.dest_branch or project.revisionExpr 227 destination = opt.dest_branch or project.dest_branch or project.revisionExpr
214 print('Upload project %s/ to remote branch %s%s:' % 228 print('Upload project %s/ to remote branch %s%s:' %
215 (project.relpath, destination, ' (draft)' if opt.draft else '')) 229 (project.relpath, destination, ' (private)' if opt.private else ''))
216 print(' branch %s (%2d commit%s, %s):' % ( 230 print(' branch %s (%2d commit%s, %s):' % (
217 name, 231 name,
218 len(commit_list), 232 len(commit_list),
219 len(commit_list) != 1 and 's' or '', 233 len(commit_list) != 1 and 's' or '',
220 date)) 234 date))
221 for commit in commit_list: 235 for commit in commit_list:
222 print(' %s' % commit) 236 print(' %s' % commit)
223 237
224 print('to %s (y/N)? ' % remote.review, end='') 238 print('to %s (y/N)? ' % remote.review, end='')
225 # TODO: When we require Python 3, use flush=True w/print above. 239 # TODO: When we require Python 3, use flush=True w/print above.
226 sys.stdout.flush() 240 sys.stdout.flush()
227 answer = sys.stdin.readline().strip().lower() 241 if opt.yes:
228 answer = answer in ('y', 'yes', '1', 'true', 't') 242 print('<--yes>')
243 answer = True
244 else:
245 answer = sys.stdin.readline().strip().lower()
246 answer = answer in ('y', 'yes', '1', 'true', 't')
229 247
230 if answer: 248 if answer:
231 if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD: 249 if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
@@ -322,12 +340,12 @@ Gerrit Code Review: https://www.gerritcodereview.com/
322 340
323 key = 'review.%s.autoreviewer' % project.GetBranch(name).remote.review 341 key = 'review.%s.autoreviewer' % project.GetBranch(name).remote.review
324 raw_list = project.config.GetString(key) 342 raw_list = project.config.GetString(key)
325 if not raw_list is None: 343 if raw_list is not None:
326 people[0].extend([entry.strip() for entry in raw_list.split(',')]) 344 people[0].extend([entry.strip() for entry in raw_list.split(',')])
327 345
328 key = 'review.%s.autocopy' % project.GetBranch(name).remote.review 346 key = 'review.%s.autocopy' % project.GetBranch(name).remote.review
329 raw_list = project.config.GetString(key) 347 raw_list = project.config.GetString(key)
330 if not raw_list is None and len(people[0]) > 0: 348 if raw_list is not None and len(people[0]) > 0:
331 people[1].extend([entry.strip() for entry in raw_list.split(',')]) 349 people[1].extend([entry.strip() for entry in raw_list.split(',')])
332 350
333 def _FindGerritChange(self, branch): 351 def _FindGerritChange(self, branch):
@@ -364,7 +382,11 @@ Gerrit Code Review: https://www.gerritcodereview.com/
364 print('Continue uploading? (y/N) ', end='') 382 print('Continue uploading? (y/N) ', end='')
365 # TODO: When we require Python 3, use flush=True w/print above. 383 # TODO: When we require Python 3, use flush=True w/print above.
366 sys.stdout.flush() 384 sys.stdout.flush()
367 a = sys.stdin.readline().strip().lower() 385 if opt.yes:
386 print('<--yes>')
387 a = 'yes'
388 else:
389 a = sys.stdin.readline().strip().lower()
368 if a not in ('y', 'yes', 't', 'true', 'on'): 390 if a not in ('y', 'yes', 't', 'true', 'on'):
369 print("skipping upload", file=sys.stderr) 391 print("skipping upload", file=sys.stderr)
370 branch.uploaded = False 392 branch.uploaded = False
@@ -376,12 +398,51 @@ Gerrit Code Review: https://www.gerritcodereview.com/
376 key = 'review.%s.uploadtopic' % branch.project.remote.review 398 key = 'review.%s.uploadtopic' % branch.project.remote.review
377 opt.auto_topic = branch.project.config.GetBoolean(key) 399 opt.auto_topic = branch.project.config.GetBoolean(key)
378 400
401 def _ExpandCommaList(value):
402 """Split |value| up into comma delimited entries."""
403 if not value:
404 return
405 for ret in value.split(','):
406 ret = ret.strip()
407 if ret:
408 yield ret
409
410 # Check if hashtags should be included.
411 key = 'review.%s.uploadhashtags' % branch.project.remote.review
412 hashtags = set(_ExpandCommaList(branch.project.config.GetString(key)))
413 for tag in opt.hashtags:
414 hashtags.update(_ExpandCommaList(tag))
415 if opt.hashtag_branch:
416 hashtags.add(branch.name)
417
418 # Check if labels should be included.
419 key = 'review.%s.uploadlabels' % branch.project.remote.review
420 labels = set(_ExpandCommaList(branch.project.config.GetString(key)))
421 for label in opt.labels:
422 labels.update(_ExpandCommaList(label))
423 # Basic sanity check on label syntax.
424 for label in labels:
425 if not re.match(r'^.+[+-][0-9]+$', label):
426 print('repo: error: invalid label syntax "%s": labels use forms '
427 'like CodeReview+1 or Verified-1' % (label,), file=sys.stderr)
428 sys.exit(1)
429
430 # Handle e-mail notifications.
431 if opt.notify is False:
432 notify = 'NONE'
433 else:
434 key = 'review.%s.uploadnotify' % branch.project.remote.review
435 notify = branch.project.config.GetString(key)
436
379 destination = opt.dest_branch or branch.project.dest_branch 437 destination = opt.dest_branch or branch.project.dest_branch
380 438
381 # Make sure our local branch is not setup to track a different remote branch 439 # Make sure our local branch is not setup to track a different remote branch
382 merge_branch = self._GetMergeBranch(branch.project) 440 merge_branch = self._GetMergeBranch(branch.project)
383 if destination: 441 if destination:
384 full_dest = 'refs/heads/%s' % destination 442 full_dest = destination
443 if not full_dest.startswith(R_HEADS):
444 full_dest = R_HEADS + full_dest
445
385 if not opt.dest_branch and merge_branch and merge_branch != full_dest: 446 if not opt.dest_branch and merge_branch and merge_branch != full_dest:
386 print('merge branch %s does not match destination branch %s' 447 print('merge branch %s does not match destination branch %s'
387 % (merge_branch, full_dest)) 448 % (merge_branch, full_dest))
@@ -392,10 +453,12 @@ Gerrit Code Review: https://www.gerritcodereview.com/
392 continue 453 continue
393 454
394 branch.UploadForReview(people, 455 branch.UploadForReview(people,
456 dryrun=opt.dryrun,
395 auto_topic=opt.auto_topic, 457 auto_topic=opt.auto_topic,
396 draft=opt.draft, 458 hashtags=hashtags,
459 labels=labels,
397 private=opt.private, 460 private=opt.private,
398 notify=None if opt.notify else 'NONE', 461 notify=notify,
399 wip=opt.wip, 462 wip=opt.wip,
400 dest_branch=destination, 463 dest_branch=destination,
401 validate_certs=opt.validate_certs, 464 validate_certs=opt.validate_certs,
@@ -418,18 +481,18 @@ Gerrit Code Review: https://www.gerritcodereview.com/
418 else: 481 else:
419 fmt = '\n (%s)' 482 fmt = '\n (%s)'
420 print(('[FAILED] %-15s %-15s' + fmt) % ( 483 print(('[FAILED] %-15s %-15s' + fmt) % (
421 branch.project.relpath + '/', \ 484 branch.project.relpath + '/',
422 branch.name, \ 485 branch.name,
423 str(branch.error)), 486 str(branch.error)),
424 file=sys.stderr) 487 file=sys.stderr)
425 print() 488 print()
426 489
427 for branch in todo: 490 for branch in todo:
428 if branch.uploaded: 491 if branch.uploaded:
429 print('[OK ] %-15s %s' % ( 492 print('[OK ] %-15s %s' % (
430 branch.project.relpath + '/', 493 branch.project.relpath + '/',
431 branch.name), 494 branch.name),
432 file=sys.stderr) 495 file=sys.stderr)
433 496
434 if have_errors: 497 if have_errors:
435 sys.exit(1) 498 sys.exit(1)
@@ -437,68 +500,72 @@ Gerrit Code Review: https://www.gerritcodereview.com/
437 def _GetMergeBranch(self, project): 500 def _GetMergeBranch(self, project):
438 p = GitCommand(project, 501 p = GitCommand(project,
439 ['rev-parse', '--abbrev-ref', 'HEAD'], 502 ['rev-parse', '--abbrev-ref', 'HEAD'],
440 capture_stdout = True, 503 capture_stdout=True,
441 capture_stderr = True) 504 capture_stderr=True)
442 p.Wait() 505 p.Wait()
443 local_branch = p.stdout.strip() 506 local_branch = p.stdout.strip()
444 p = GitCommand(project, 507 p = GitCommand(project,
445 ['config', '--get', 'branch.%s.merge' % local_branch], 508 ['config', '--get', 'branch.%s.merge' % local_branch],
446 capture_stdout = True, 509 capture_stdout=True,
447 capture_stderr = True) 510 capture_stderr=True)
448 p.Wait() 511 p.Wait()
449 merge_branch = p.stdout.strip() 512 merge_branch = p.stdout.strip()
450 return merge_branch 513 return merge_branch
451 514
515 @staticmethod
516 def _GatherOne(opt, project):
517 """Figure out the upload status for |project|."""
518 if opt.current_branch:
519 cbr = project.CurrentBranch
520 up_branch = project.GetUploadableBranch(cbr)
521 avail = [up_branch] if up_branch else None
522 else:
523 avail = project.GetUploadableBranches(opt.branch)
524 return (project, avail)
525
452 def Execute(self, opt, args): 526 def Execute(self, opt, args):
453 project_list = self.GetProjects(args) 527 projects = self.GetProjects(args)
454 pending = [] 528
455 reviewers = [] 529 def _ProcessResults(_pool, _out, results):
456 cc = [] 530 pending = []
457 branch = None 531 for result in results:
458 532 project, avail = result
459 if opt.branch: 533 if avail is None:
460 branch = opt.branch 534 print('repo: error: %s: Unable to upload branch "%s". '
461 535 'You might be able to fix the branch by running:\n'
462 for project in project_list: 536 ' git branch --set-upstream-to m/%s' %
463 if opt.current_branch: 537 (project.relpath, project.CurrentBranch, self.manifest.branch),
464 cbr = project.CurrentBranch
465 up_branch = project.GetUploadableBranch(cbr)
466 if up_branch:
467 avail = [up_branch]
468 else:
469 avail = None
470 print('ERROR: Current branch (%s) not uploadable. '
471 'You may be able to type '
472 '"git branch --set-upstream-to m/master" to fix '
473 'your branch.' % str(cbr),
474 file=sys.stderr) 538 file=sys.stderr)
475 else: 539 elif avail:
476 avail = project.GetUploadableBranches(branch) 540 pending.append(result)
477 if avail: 541 return pending
478 pending.append((project, avail)) 542
543 pending = self.ExecuteInParallel(
544 opt.jobs,
545 functools.partial(self._GatherOne, opt),
546 projects,
547 callback=_ProcessResults)
479 548
480 if not pending: 549 if not pending:
481 print("no branches ready for upload", file=sys.stderr) 550 if opt.branch is None:
482 return 551 print('repo: error: no branches ready for upload', file=sys.stderr)
483 552 else:
484 if not opt.bypass_hooks: 553 print('repo: error: no branches named "%s" ready for upload' %
485 hook = RepoHook('pre-upload', self.manifest.repo_hooks_project, 554 (opt.branch,), file=sys.stderr)
486 self.manifest.topdir, 555 return 1
487 self.manifest.manifestProject.GetRemote('origin').url, 556
488 abort_if_user_denies=True) 557 pending_proj_names = [project.name for (project, available) in pending]
489 pending_proj_names = [project.name for (project, available) in pending] 558 pending_worktrees = [project.worktree for (project, available) in pending]
490 pending_worktrees = [project.worktree for (project, available) in pending] 559 hook = RepoHook.FromSubcmd(
491 try: 560 hook_type='pre-upload', manifest=self.manifest,
492 hook.Run(opt.allow_all_hooks, project_list=pending_proj_names, 561 opt=opt, abort_if_user_denies=True)
493 worktree_list=pending_worktrees) 562 if not hook.Run(
494 except HookError as e: 563 project_list=pending_proj_names,
495 print("ERROR: %s" % str(e), file=sys.stderr) 564 worktree_list=pending_worktrees):
496 return 565 return 1
497 566
498 if opt.reviewers: 567 reviewers = _SplitEmails(opt.reviewers) if opt.reviewers else []
499 reviewers = _SplitEmails(opt.reviewers) 568 cc = _SplitEmails(opt.cc) if opt.cc else []
500 if opt.cc:
501 cc = _SplitEmails(opt.cc)
502 people = (reviewers, cc) 569 people = (reviewers, cc)
503 570
504 if len(pending) == 1 and len(pending[0][1]) == 1: 571 if len(pending) == 1 and len(pending[0][1]) == 1:
diff --git a/subcmds/version.py b/subcmds/version.py
index 761172b7..09b053ea 100644
--- a/subcmds/version.py
+++ b/subcmds/version.py
@@ -1,5 +1,3 @@
1# -*- coding:utf-8 -*-
2#
3# Copyright (C) 2009 The Android Open Source Project 1# Copyright (C) 2009 The Android Open Source Project
4# 2#
5# Licensed under the Apache License, Version 2.0 (the "License"); 3# Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,17 +12,20 @@
14# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
15# limitations under the License. 13# limitations under the License.
16 14
17from __future__ import print_function 15import platform
18import sys 16import sys
17
19from command import Command, MirrorSafeCommand 18from command import Command, MirrorSafeCommand
20from git_command import git, RepoSourceVersion, user_agent 19from git_command import git, RepoSourceVersion, user_agent
21from git_refs import HEAD 20from git_refs import HEAD
21from wrapper import Wrapper
22
22 23
23class Version(Command, MirrorSafeCommand): 24class Version(Command, MirrorSafeCommand):
24 wrapper_version = None 25 wrapper_version = None
25 wrapper_path = None 26 wrapper_path = None
26 27
27 common = False 28 COMMON = False
28 helpSummary = "Display the version of repo" 29 helpSummary = "Display the version of repo"
29 helpUsage = """ 30 helpUsage = """
30%prog 31%prog
@@ -33,16 +34,19 @@ class Version(Command, MirrorSafeCommand):
33 def Execute(self, opt, args): 34 def Execute(self, opt, args):
34 rp = self.manifest.repoProject 35 rp = self.manifest.repoProject
35 rem = rp.GetRemote(rp.remote.name) 36 rem = rp.GetRemote(rp.remote.name)
37 branch = rp.GetBranch('default')
36 38
37 # These might not be the same. Report them both. 39 # These might not be the same. Report them both.
38 src_ver = RepoSourceVersion() 40 src_ver = RepoSourceVersion()
39 rp_ver = rp.bare_git.describe(HEAD) 41 rp_ver = rp.bare_git.describe(HEAD)
40 print('repo version %s' % rp_ver) 42 print('repo version %s' % rp_ver)
41 print(' (from %s)' % rem.url) 43 print(' (from %s)' % rem.url)
44 print(' (tracking %s)' % branch.merge)
45 print(' (%s)' % rp.bare_git.log('-1', '--format=%cD', HEAD))
42 46
43 if Version.wrapper_path is not None: 47 if self.wrapper_path is not None:
44 print('repo launcher version %s' % Version.wrapper_version) 48 print('repo launcher version %s' % self.wrapper_version)
45 print(' (from %s)' % Version.wrapper_path) 49 print(' (from %s)' % self.wrapper_path)
46 50
47 if src_ver != rp_ver: 51 if src_ver != rp_ver:
48 print(' (currently at %s)' % src_ver) 52 print(' (currently at %s)' % src_ver)
@@ -51,3 +55,12 @@ class Version(Command, MirrorSafeCommand):
51 print('git %s' % git.version_tuple().full) 55 print('git %s' % git.version_tuple().full)
52 print('git User-Agent %s' % user_agent.git) 56 print('git User-Agent %s' % user_agent.git)
53 print('Python %s' % sys.version) 57 print('Python %s' % sys.version)
58 uname = platform.uname()
59 if sys.version_info.major < 3:
60 # Python 3 returns a named tuple, but Python 2 is simpler.
61 print(uname)
62 else:
63 print('OS %s %s (%s)' % (uname.system, uname.release, uname.version))
64 print('CPU %s (%s)' %
65 (uname.machine, uname.processor if uname.processor else 'unknown'))
66 print('Bug reports:', Wrapper().BUG_URL)
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2025-05-16 15:55:30 +0000