summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--SUBMITTING_PATCHES30
-rw-r--r--command.py65
-rw-r--r--docs/manifest-format.txt31
-rw-r--r--git_command.py3
-rw-r--r--git_config.py39
-rw-r--r--gitc_utils.py148
-rwxr-xr-xhooks/commit-msg19
-rwxr-xr-xmain.py20
-rw-r--r--manifest_xml.py44
-rw-r--r--project.py352
-rwxr-xr-xrepo197
-rw-r--r--subcmds/forall.py17
-rw-r--r--subcmds/gitc_delete.py55
-rw-r--r--subcmds/gitc_init.py82
-rw-r--r--subcmds/help.py17
-rw-r--r--subcmds/init.py4
-rw-r--r--subcmds/rebase.py9
-rw-r--r--subcmds/start.py49
-rw-r--r--subcmds/sync.py178
-rw-r--r--tests/fixtures/gitc_config1
-rw-r--r--tests/test_wrapper.py75
21 files changed, 1165 insertions, 270 deletions
diff --git a/SUBMITTING_PATCHES b/SUBMITTING_PATCHES
index 50e2cf77..8656ee7d 100644
--- a/SUBMITTING_PATCHES
+++ b/SUBMITTING_PATCHES
@@ -4,7 +4,9 @@ Short Version:
4 - Provide a meaningful commit message. 4 - Provide a meaningful commit message.
5 - Check for coding errors with pylint 5 - Check for coding errors with pylint
6 - Make sure all code is under the Apache License, 2.0. 6 - Make sure all code is under the Apache License, 2.0.
7 - Publish your changes for review: 7 - Publish your changes for review.
8 - Make corrections if requested.
9 - Verify your changes on gerrit so they can be submitted.
8 10
9 git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/master 11 git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/master
10 12
@@ -75,6 +77,17 @@ Ensure you have obtained an HTTP password to authenticate:
75 77
76 https://gerrit-review.googlesource.com/new-password 78 https://gerrit-review.googlesource.com/new-password
77 79
80Ensure that you have the local commit hook installed to automatically
81add a ChangeId to your commits:
82
83 curl -Lo `git rev-parse --git-dir`/hooks/commit-msg https://gerrit-review.googlesource.com/tools/hooks/commit-msg
84 chmod +x `git rev-parse --git-dir`/hooks/commit-msg
85
86If you have already committed your changes you will need to amend the commit
87to get the ChangeId added.
88
89 git commit --amend
90
78Push your patches over HTTPS to the review server, possibly through 91Push your patches over HTTPS to the review server, possibly through
79a remembered remote to make this easier in the future: 92a remembered remote to make this easier in the future:
80 93
@@ -85,3 +98,18 @@ a remembered remote to make this easier in the future:
85 98
86You will be automatically emailed a copy of your commits, and any 99You will be automatically emailed a copy of your commits, and any
87comments made by the project maintainers. 100comments made by the project maintainers.
101
102
103(5) Make changes if requested
104
105The project maintainer who reviews your changes might request changes to your
106commit. If you make the requested changes you will need to amend your commit
107and push it to the review server again.
108
109
110(6) Verify your changes on gerrit
111
112After you receive a Code-Review+2 from the maintainer, select the Verified
113button on the gerrit page for the change. This verifies that you have tested
114your changes and notifies the maintainer that they are ready to be submitted.
115The maintainer will then submit your changes to the repository.
diff --git a/command.py b/command.py
index 38cacd3b..bc2f9501 100644
--- a/command.py
+++ b/command.py
@@ -31,7 +31,7 @@ class Command(object):
31 manifest = None 31 manifest = None
32 _optparse = None 32 _optparse = None
33 33
34 def WantPager(self, opt): 34 def WantPager(self, _opt):
35 return False 35 return False
36 36
37 def ReadEnvironmentOptions(self, opts): 37 def ReadEnvironmentOptions(self, opts):
@@ -63,7 +63,7 @@ class Command(object):
63 usage = self.helpUsage.strip().replace('%prog', me) 63 usage = self.helpUsage.strip().replace('%prog', me)
64 except AttributeError: 64 except AttributeError:
65 usage = 'repo %s' % self.NAME 65 usage = 'repo %s' % self.NAME
66 self._optparse = optparse.OptionParser(usage = usage) 66 self._optparse = optparse.OptionParser(usage=usage)
67 self._Options(self._optparse) 67 self._Options(self._optparse)
68 return self._optparse 68 return self._optparse
69 69
@@ -106,13 +106,13 @@ class Command(object):
106 def _UpdatePathToProjectMap(self, project): 106 def _UpdatePathToProjectMap(self, project):
107 self._by_path[project.worktree] = project 107 self._by_path[project.worktree] = project
108 108
109 def _GetProjectByPath(self, path): 109 def _GetProjectByPath(self, manifest, path):
110 project = None 110 project = None
111 if os.path.exists(path): 111 if os.path.exists(path):
112 oldpath = None 112 oldpath = None
113 while path \ 113 while path and \
114 and path != oldpath \ 114 path != oldpath and \
115 and path != self.manifest.topdir: 115 path != manifest.topdir:
116 try: 116 try:
117 project = self._by_path[path] 117 project = self._by_path[path]
118 break 118 break
@@ -126,16 +126,19 @@ class Command(object):
126 pass 126 pass
127 return project 127 return project
128 128
129 def GetProjects(self, args, groups='', missing_ok=False, submodules_ok=False): 129 def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
130 submodules_ok=False):
130 """A list of projects that match the arguments. 131 """A list of projects that match the arguments.
131 """ 132 """
132 all_projects_list = self.manifest.projects 133 if not manifest:
134 manifest = self.manifest
135 all_projects_list = manifest.projects
133 result = [] 136 result = []
134 137
135 mp = self.manifest.manifestProject 138 mp = manifest.manifestProject
136 139
137 if not groups: 140 if not groups:
138 groups = mp.config.GetString('manifest.groups') 141 groups = mp.config.GetString('manifest.groups')
139 if not groups: 142 if not groups:
140 groups = 'default,platform-' + platform.system().lower() 143 groups = 'default,platform-' + platform.system().lower()
141 groups = [x for x in re.split(r'[,\s]+', groups) if x] 144 groups = [x for x in re.split(r'[,\s]+', groups) if x]
@@ -148,29 +151,28 @@ class Command(object):
148 for p in project.GetDerivedSubprojects()) 151 for p in project.GetDerivedSubprojects())
149 all_projects_list.extend(derived_projects.values()) 152 all_projects_list.extend(derived_projects.values())
150 for project in all_projects_list: 153 for project in all_projects_list:
151 if ((missing_ok or project.Exists) and 154 if (missing_ok or project.Exists) and project.MatchesGroups(groups):
152 project.MatchesGroups(groups)):
153 result.append(project) 155 result.append(project)
154 else: 156 else:
155 self._ResetPathToProjectMap(all_projects_list) 157 self._ResetPathToProjectMap(all_projects_list)
156 158
157 for arg in args: 159 for arg in args:
158 projects = self.manifest.GetProjectsWithName(arg) 160 projects = manifest.GetProjectsWithName(arg)
159 161
160 if not projects: 162 if not projects:
161 path = os.path.abspath(arg).replace('\\', '/') 163 path = os.path.abspath(arg).replace('\\', '/')
162 project = self._GetProjectByPath(path) 164 project = self._GetProjectByPath(manifest, path)
163 165
164 # If it's not a derived project, update path->project mapping and 166 # If it's not a derived project, update path->project mapping and
165 # search again, as arg might actually point to a derived subproject. 167 # search again, as arg might actually point to a derived subproject.
166 if (project and not project.Derived and 168 if (project and not project.Derived and (submodules_ok or
167 (submodules_ok or project.sync_s)): 169 project.sync_s)):
168 search_again = False 170 search_again = False
169 for subproject in project.GetDerivedSubprojects(): 171 for subproject in project.GetDerivedSubprojects():
170 self._UpdatePathToProjectMap(subproject) 172 self._UpdatePathToProjectMap(subproject)
171 search_again = True 173 search_again = True
172 if search_again: 174 if search_again:
173 project = self._GetProjectByPath(path) or project 175 project = self._GetProjectByPath(manifest, path) or project
174 176
175 if project: 177 if project:
176 projects = [project] 178 projects = [project]
@@ -191,17 +193,24 @@ class Command(object):
191 result.sort(key=_getpath) 193 result.sort(key=_getpath)
192 return result 194 return result
193 195
194 def FindProjects(self, args): 196 def FindProjects(self, args, inverse=False):
195 result = [] 197 result = []
196 patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args] 198 patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
197 for project in self.GetProjects(''): 199 for project in self.GetProjects(''):
198 for pattern in patterns: 200 for pattern in patterns:
199 if pattern.search(project.name) or pattern.search(project.relpath): 201 match = pattern.search(project.name) or pattern.search(project.relpath)
202 if not inverse and match:
200 result.append(project) 203 result.append(project)
201 break 204 break
205 if inverse and match:
206 break
207 else:
208 if inverse:
209 result.append(project)
202 result.sort(key=lambda project: project.relpath) 210 result.sort(key=lambda project: project.relpath)
203 return result 211 return result
204 212
213
205# pylint: disable=W0223 214# pylint: disable=W0223
206# Pylint warns that the `InteractiveCommand` and `PagedCommand` classes do not 215# Pylint warns that the `InteractiveCommand` and `PagedCommand` classes do not
207# override method `Execute` which is abstract in `Command`. Since that method 216# override method `Execute` which is abstract in `Command`. Since that method
@@ -211,19 +220,33 @@ class InteractiveCommand(Command):
211 """Command which requires user interaction on the tty and 220 """Command which requires user interaction on the tty and
212 must not run within a pager, even if the user asks to. 221 must not run within a pager, even if the user asks to.
213 """ 222 """
214 def WantPager(self, opt): 223 def WantPager(self, _opt):
215 return False 224 return False
216 225
226
217class PagedCommand(Command): 227class PagedCommand(Command):
218 """Command which defaults to output in a pager, as its 228 """Command which defaults to output in a pager, as its
219 display tends to be larger than one screen full. 229 display tends to be larger than one screen full.
220 """ 230 """
221 def WantPager(self, opt): 231 def WantPager(self, _opt):
222 return True 232 return True
223 233
224# pylint: enable=W0223 234# pylint: enable=W0223
225 235
236
226class MirrorSafeCommand(object): 237class MirrorSafeCommand(object):
227 """Command permits itself to run within a mirror, 238 """Command permits itself to run within a mirror,
228 and does not require a working directory. 239 and does not require a working directory.
229 """ 240 """
241
242
243class GitcAvailableCommand(object):
244 """Command that requires GITC to be available, but does
245 not require the local client to be a GITC client.
246 """
247
248
249class GitcClientCommand(object):
250 """Command that requires the local client to be a GITC
251 client.
252 """
diff --git a/docs/manifest-format.txt b/docs/manifest-format.txt
index 1aa93965..140a782f 100644
--- a/docs/manifest-format.txt
+++ b/docs/manifest-format.txt
@@ -47,10 +47,12 @@ following DTD:
47 <!ATTLIST default sync-s CDATA #IMPLIED> 47 <!ATTLIST default sync-s CDATA #IMPLIED>
48 48
49 <!ELEMENT manifest-server (EMPTY)> 49 <!ELEMENT manifest-server (EMPTY)>
50 <!ATTLIST url CDATA #REQUIRED> 50 <!ATTLIST manifest-server url CDATA #REQUIRED>
51 51
52 <!ELEMENT project (annotation*, 52 <!ELEMENT project (annotation*,
53 project*)> 53 project*,
54 copyfile*,
55 linkfile*)>
54 <!ATTLIST project name CDATA #REQUIRED> 56 <!ATTLIST project name CDATA #REQUIRED>
55 <!ATTLIST project path CDATA #IMPLIED> 57 <!ATTLIST project path CDATA #IMPLIED>
56 <!ATTLIST project remote IDREF #IMPLIED> 58 <!ATTLIST project remote IDREF #IMPLIED>
@@ -68,7 +70,15 @@ following DTD:
68 <!ATTLIST annotation value CDATA #REQUIRED> 70 <!ATTLIST annotation value CDATA #REQUIRED>
69 <!ATTLIST annotation keep CDATA "true"> 71 <!ATTLIST annotation keep CDATA "true">
70 72
71 <!ELEMENT extend-project> 73 <!ELEMENT copyfile (EMPTY)>
74 <!ATTLIST copyfile src CDATA #REQUIRED>
75 <!ATTLIST copyfile dest CDATA #REQUIRED>
76
77 <!ELEMENT linkfile (EMPTY)>
78 <!ATTLIST linkfile src CDATA #REQUIRED>
79 <!ATTLIST linkfile dest CDATA #REQUIRED>
80
81 <!ELEMENT extend-project (EMPTY)>
72 <!ATTLIST extend-project name CDATA #REQUIRED> 82 <!ATTLIST extend-project name CDATA #REQUIRED>
73 <!ATTLIST extend-project path CDATA #IMPLIED> 83 <!ATTLIST extend-project path CDATA #IMPLIED>
74 <!ATTLIST extend-project groups CDATA #IMPLIED> 84 <!ATTLIST extend-project groups CDATA #IMPLIED>
@@ -285,6 +295,21 @@ prefixed with REPO__. In addition, there is an optional attribute
285"false". This attribute determines whether or not the annotation will 295"false". This attribute determines whether or not the annotation will
286be kept when exported with the manifest subcommand. 296be kept when exported with the manifest subcommand.
287 297
298Element copyfile
299----------------
300
301Zero or more copyfile elements may be specified as children of a
302project element. Each element describes a src-dest pair of files;
303the "src" file will be copied to the "dest" place during 'repo sync'
304command.
305"src" is project relative, "dest" is relative to the top of the tree.
306
307Element linkfile
308----------------
309
310It's just like copyfile and runs at the same time as copyfile but
311instead of copying it creates a symlink.
312
288Element remove-project 313Element remove-project
289---------------------- 314----------------------
290 315
diff --git a/git_command.py b/git_command.py
index 0893bff7..9f7d2930 100644
--- a/git_command.py
+++ b/git_command.py
@@ -168,6 +168,9 @@ class GitCommand(object):
168 if p is not None: 168 if p is not None:
169 s = p + ' ' + s 169 s = p + ' ' + s
170 _setenv(env, 'GIT_CONFIG_PARAMETERS', s) 170 _setenv(env, 'GIT_CONFIG_PARAMETERS', s)
171 if 'GIT_ALLOW_PROTOCOL' not in env:
172 _setenv(env, 'GIT_ALLOW_PROTOCOL',
173 'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
171 174
172 if project: 175 if project:
173 if not cwd: 176 if not cwd:
diff --git a/git_config.py b/git_config.py
index 8ded7c25..0379181a 100644
--- a/git_config.py
+++ b/git_config.py
@@ -15,6 +15,8 @@
15 15
16from __future__ import print_function 16from __future__ import print_function
17 17
18import contextlib
19import errno
18import json 20import json
19import os 21import os
20import re 22import re
@@ -502,6 +504,43 @@ def GetSchemeFromUrl(url):
502 return m.group(1) 504 return m.group(1)
503 return None 505 return None
504 506
507@contextlib.contextmanager
508def GetUrlCookieFile(url, quiet):
509 if url.startswith('persistent-'):
510 try:
511 p = subprocess.Popen(
512 ['git-remote-persistent-https', '-print_config', url],
513 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
514 stderr=subprocess.PIPE)
515 try:
516 cookieprefix = 'http.cookiefile='
517 proxyprefix = 'http.proxy='
518 cookiefile = None
519 proxy = None
520 for line in p.stdout:
521 line = line.strip()
522 if line.startswith(cookieprefix):
523 cookiefile = line[len(cookieprefix):]
524 if line.startswith(proxyprefix):
525 proxy = line[len(proxyprefix):]
526 # Leave subprocess open, as cookie file may be transient.
527 if cookiefile or proxy:
528 yield cookiefile, proxy
529 return
530 finally:
531 p.stdin.close()
532 if p.wait():
533 err_msg = p.stderr.read()
534 if ' -print_config' in err_msg:
535 pass # Persistent proxy doesn't support -print_config.
536 elif not quiet:
537 print(err_msg, file=sys.stderr)
538 except OSError as e:
539 if e.errno == errno.ENOENT:
540 pass # No persistent proxy.
541 raise
542 yield GitConfig.ForUser().GetString('http.cookiefile'), None
543
505def _preconnect(url): 544def _preconnect(url):
506 m = URI_ALL.match(url) 545 m = URI_ALL.match(url)
507 if m: 546 if m:
diff --git a/gitc_utils.py b/gitc_utils.py
new file mode 100644
index 00000000..0f3e1818
--- /dev/null
+++ b/gitc_utils.py
@@ -0,0 +1,148 @@
1#
2# Copyright (C) 2015 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16from __future__ import print_function
17import os
18import platform
19import re
20import sys
21import time
22
23import git_command
24import git_config
25import wrapper
26
27NUM_BATCH_RETRIEVE_REVISIONID = 300
28
29def get_gitc_manifest_dir():
30 return wrapper.Wrapper().get_gitc_manifest_dir()
31
32def parse_clientdir(gitc_fs_path):
33 return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path)
34
35def _set_project_revisions(projects):
36 """Sets the revisionExpr for a list of projects.
37
38 Because of the limit of open file descriptors allowed, length of projects
39 should not be overly large. Recommend calling this function multiple times
40 with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
41
42 @param projects: List of project objects to set the revionExpr for.
43 """
44 # Retrieve the commit id for each project based off of it's current
45 # revisionExpr and it is not already a commit id.
46 project_gitcmds = [(
47 project, git_command.GitCommand(None,
48 ['ls-remote',
49 project.remote.url,
50 project.revisionExpr],
51 capture_stdout=True, cwd='/tmp'))
52 for project in projects if not git_config.IsId(project.revisionExpr)]
53 for proj, gitcmd in project_gitcmds:
54 if gitcmd.Wait():
55 print('FATAL: Failed to retrieve revisionExpr for %s' % proj)
56 sys.exit(1)
57 proj.revisionExpr = gitcmd.stdout.split('\t')[0]
58
59def _manifest_groups(manifest):
60 """Returns the manifest group string that should be synced
61
62 This is the same logic used by Command.GetProjects(), which is used during
63 repo sync
64
65 @param manifest: The XmlManifest object
66 """
67 mp = manifest.manifestProject
68 groups = mp.config.GetString('manifest.groups')
69 if not groups:
70 groups = 'default,platform-' + platform.system().lower()
71 return groups
72
73def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
74 """Generate a manifest for shafsd to use for this GITC client.
75
76 @param gitc_manifest: Current gitc manifest, or None if there isn't one yet.
77 @param manifest: A GitcManifest object loaded with the current repo manifest.
78 @param paths: List of project paths we want to update.
79 """
80
81 print('Generating GITC Manifest by fetching revision SHAs for each '
82 'project.')
83 if paths is None:
84 paths = manifest.paths.keys()
85
86 groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x]
87
88 # Convert the paths to projects, and filter them to the matched groups.
89 projects = [manifest.paths[p] for p in paths]
90 projects = [p for p in projects if p.MatchesGroups(groups)]
91
92 if gitc_manifest is not None:
93 for path, proj in manifest.paths.iteritems():
94 if not proj.MatchesGroups(groups):
95 continue
96
97 if not proj.upstream and not git_config.IsId(proj.revisionExpr):
98 proj.upstream = proj.revisionExpr
99
100 if not path in gitc_manifest.paths:
101 # Any new projects need their first revision, even if we weren't asked
102 # for them.
103 projects.append(proj)
104 elif not path in paths:
105 # And copy revisions from the previous manifest if we're not updating
106 # them now.
107 gitc_proj = gitc_manifest.paths[path]
108 if gitc_proj.old_revision:
109 proj.revisionExpr = None
110 proj.old_revision = gitc_proj.old_revision
111 else:
112 proj.revisionExpr = gitc_proj.revisionExpr
113
114 index = 0
115 while index < len(projects):
116 _set_project_revisions(
117 projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
118 index += NUM_BATCH_RETRIEVE_REVISIONID
119
120 if gitc_manifest is not None:
121 for path, proj in gitc_manifest.paths.iteritems():
122 if proj.old_revision and path in paths:
123 # If we updated a project that has been started, keep the old-revision
124 # updated.
125 repo_proj = manifest.paths[path]
126 repo_proj.old_revision = repo_proj.revisionExpr
127 repo_proj.revisionExpr = None
128
129 # Convert URLs from relative to absolute.
130 for name, remote in manifest.remotes.iteritems():
131 remote.fetchUrl = remote.resolvedFetchUrl
132
133 # Save the manifest.
134 save_manifest(manifest)
135
136def save_manifest(manifest, client_dir=None):
137 """Save the manifest file in the client_dir.
138
139 @param client_dir: Client directory to save the manifest in.
140 @param manifest: Manifest object to save.
141 """
142 if not client_dir:
143 client_dir = manifest.gitc_client_dir
144 with open(os.path.join(client_dir, '.manifest'), 'w') as f:
145 manifest.Save(f, groups=_manifest_groups(manifest))
146 # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
147 # Give the GITC filesystem time to register the manifest changes.
148 time.sleep(3)
diff --git a/hooks/commit-msg b/hooks/commit-msg
index d8f009b6..40ac237a 100755
--- a/hooks/commit-msg
+++ b/hooks/commit-msg
@@ -1,6 +1,7 @@
1#!/bin/sh 1#!/bin/sh
2# From Gerrit Code Review 2.12.1
2# 3#
3# Part of Gerrit Code Review (http://code.google.com/p/gerrit/) 4# Part of Gerrit Code Review (https://www.gerritcodereview.com/)
4# 5#
5# Copyright (C) 2009 The Android Open Source Project 6# Copyright (C) 2009 The Android Open Source Project
6# 7#
@@ -19,7 +20,7 @@
19 20
20unset GREP_OPTIONS 21unset GREP_OPTIONS
21 22
22CHANGE_ID_AFTER="Bug|Issue" 23CHANGE_ID_AFTER="Bug|Issue|Test"
23MSG="$1" 24MSG="$1"
24 25
25# Check for, and add if missing, a unique Change-Id 26# Check for, and add if missing, a unique Change-Id
@@ -38,6 +39,12 @@ add_ChangeId() {
38 return 39 return
39 fi 40 fi
40 41
42 # Do not add Change-Id to temp commits
43 if echo "$clean_message" | head -1 | grep -q '^\(fixup\|squash\)!'
44 then
45 return
46 fi
47
41 if test "false" = "`git config --bool --get gerrit.createChangeId`" 48 if test "false" = "`git config --bool --get gerrit.createChangeId`"
42 then 49 then
43 return 50 return
@@ -57,6 +64,10 @@ add_ChangeId() {
57 AWK=/usr/xpg4/bin/awk 64 AWK=/usr/xpg4/bin/awk
58 fi 65 fi
59 66
67 # Get core.commentChar from git config or use default symbol
68 commentChar=`git config --get core.commentChar`
69 commentChar=${commentChar:-#}
70
60 # How this works: 71 # How this works:
61 # - parse the commit message as (textLine+ blankLine*)* 72 # - parse the commit message as (textLine+ blankLine*)*
62 # - assume textLine+ to be a footer until proven otherwise 73 # - assume textLine+ to be a footer until proven otherwise
@@ -75,8 +86,8 @@ add_ChangeId() {
75 blankLines = 0 86 blankLines = 0
76 } 87 }
77 88
78 # Skip lines starting with "#" without any spaces before it. 89 # Skip lines starting with commentChar without any spaces before it.
79 /^#/ { next } 90 /^'"$commentChar"'/ { next }
80 91
81 # Skip the line starting with the diff command and everything after it, 92 # Skip the line starting with the diff command and everything after it,
82 # up to the end of the file, assuming it is only patch data. 93 # up to the end of the file, assuming it is only patch data.
diff --git a/main.py b/main.py
index 6736abc9..4f4eb9fc 100755
--- a/main.py
+++ b/main.py
@@ -42,6 +42,7 @@ from git_command import git, GitCommand
42from git_config import init_ssh, close_ssh 42from git_config import init_ssh, close_ssh
43from command import InteractiveCommand 43from command import InteractiveCommand
44from command import MirrorSafeCommand 44from command import MirrorSafeCommand
45from command import GitcAvailableCommand, GitcClientCommand
45from subcmds.version import Version 46from subcmds.version import Version
46from editor import Editor 47from editor import Editor
47from error import DownloadError 48from error import DownloadError
@@ -51,7 +52,8 @@ from error import ManifestParseError
51from error import NoManifestException 52from error import NoManifestException
52from error import NoSuchProjectError 53from error import NoSuchProjectError
53from error import RepoChangedException 54from error import RepoChangedException
54from manifest_xml import XmlManifest 55import gitc_utils
56from manifest_xml import GitcManifest, XmlManifest
55from pager import RunPager 57from pager import RunPager
56from wrapper import WrapperPath, Wrapper 58from wrapper import WrapperPath, Wrapper
57 59
@@ -129,6 +131,12 @@ class _Repo(object):
129 131
130 cmd.repodir = self.repodir 132 cmd.repodir = self.repodir
131 cmd.manifest = XmlManifest(cmd.repodir) 133 cmd.manifest = XmlManifest(cmd.repodir)
134 cmd.gitc_manifest = None
135 gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
136 if gitc_client_name:
137 cmd.gitc_manifest = GitcManifest(cmd.repodir, gitc_client_name)
138 cmd.manifest.isGitcClient = True
139
132 Editor.globalConfig = cmd.manifest.globalConfig 140 Editor.globalConfig = cmd.manifest.globalConfig
133 141
134 if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror: 142 if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
@@ -136,6 +144,16 @@ class _Repo(object):
136 file=sys.stderr) 144 file=sys.stderr)
137 return 1 145 return 1
138 146
147 if isinstance(cmd, GitcAvailableCommand) and not gitc_utils.get_gitc_manifest_dir():
148 print("fatal: '%s' requires GITC to be available" % name,
149 file=sys.stderr)
150 return 1
151
152 if isinstance(cmd, GitcClientCommand) and not gitc_client_name:
153 print("fatal: '%s' requires a GITC client" % name,
154 file=sys.stderr)
155 return 1
156
139 try: 157 try:
140 copts, cargs = cmd.OptionParser.parse_args(argv) 158 copts, cargs = cmd.OptionParser.parse_args(argv)
141 copts = cmd.ReadEnvironmentOptions(copts) 159 copts = cmd.ReadEnvironmentOptions(copts)
diff --git a/manifest_xml.py b/manifest_xml.py
index 7e719600..3ac607ec 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -29,6 +29,7 @@ else:
29 urllib = imp.new_module('urllib') 29 urllib = imp.new_module('urllib')
30 urllib.parse = urlparse 30 urllib.parse = urlparse
31 31
32import gitc_utils
32from git_config import GitConfig 33from git_config import GitConfig
33from git_refs import R_HEADS, HEAD 34from git_refs import R_HEADS, HEAD
34from project import RemoteSpec, Project, MetaProject 35from project import RemoteSpec, Project, MetaProject
@@ -112,6 +113,7 @@ class XmlManifest(object):
112 self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME) 113 self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
113 self.globalConfig = GitConfig.ForUser() 114 self.globalConfig = GitConfig.ForUser()
114 self.localManifestWarning = False 115 self.localManifestWarning = False
116 self.isGitcClient = False
115 117
116 self.repoProject = MetaProject(self, 'repo', 118 self.repoProject = MetaProject(self, 'repo',
117 gitdir = os.path.join(repodir, 'repo/.git'), 119 gitdir = os.path.join(repodir, 'repo/.git'),
@@ -165,12 +167,13 @@ class XmlManifest(object):
165 def _ParseGroups(self, groups): 167 def _ParseGroups(self, groups):
166 return [x for x in re.split(r'[,\s]+', groups) if x] 168 return [x for x in re.split(r'[,\s]+', groups) if x]
167 169
168 def Save(self, fd, peg_rev=False, peg_rev_upstream=True): 170 def Save(self, fd, peg_rev=False, peg_rev_upstream=True, groups=None):
169 """Write the current manifest out to the given file descriptor. 171 """Write the current manifest out to the given file descriptor.
170 """ 172 """
171 mp = self.manifestProject 173 mp = self.manifestProject
172 174
173 groups = mp.config.GetString('manifest.groups') 175 if groups is None:
176 groups = mp.config.GetString('manifest.groups')
174 if groups: 177 if groups:
175 groups = self._ParseGroups(groups) 178 groups = self._ParseGroups(groups)
176 179
@@ -303,6 +306,11 @@ class XmlManifest(object):
303 if p.sync_s: 306 if p.sync_s:
304 e.setAttribute('sync-s', 'true') 307 e.setAttribute('sync-s', 'true')
305 308
309 if p.clone_depth:
310 e.setAttribute('clone-depth', str(p.clone_depth))
311
312 self._output_manifest_project_extras(p, e)
313
306 if p.subprojects: 314 if p.subprojects:
307 subprojects = set(subp.name for subp in p.subprojects) 315 subprojects = set(subp.name for subp in p.subprojects)
308 output_projects(p, e, list(sorted(subprojects))) 316 output_projects(p, e, list(sorted(subprojects)))
@@ -320,6 +328,10 @@ class XmlManifest(object):
320 328
321 doc.writexml(fd, '', ' ', '\n', 'UTF-8') 329 doc.writexml(fd, '', ' ', '\n', 'UTF-8')
322 330
331 def _output_manifest_project_extras(self, p, e):
332 """Manifests can modify e if they support extra project attributes."""
333 pass
334
323 @property 335 @property
324 def paths(self): 336 def paths(self):
325 self._Load() 337 self._Load()
@@ -709,7 +721,7 @@ class XmlManifest(object):
709 def _UnjoinName(self, parent_name, name): 721 def _UnjoinName(self, parent_name, name):
710 return os.path.relpath(name, parent_name) 722 return os.path.relpath(name, parent_name)
711 723
712 def _ParseProject(self, node, parent = None): 724 def _ParseProject(self, node, parent = None, **extra_proj_attrs):
713 """ 725 """
714 reads a <project> element from the manifest file 726 reads a <project> element from the manifest file
715 """ 727 """
@@ -804,7 +816,8 @@ class XmlManifest(object):
804 clone_depth = clone_depth, 816 clone_depth = clone_depth,
805 upstream = upstream, 817 upstream = upstream,
806 parent = parent, 818 parent = parent,
807 dest_branch = dest_branch) 819 dest_branch = dest_branch,
820 **extra_proj_attrs)
808 821
809 for n in node.childNodes: 822 for n in node.childNodes:
810 if n.nodeName == 'copyfile': 823 if n.nodeName == 'copyfile':
@@ -935,3 +948,26 @@ class XmlManifest(object):
935 diff['added'].append(toProjects[proj]) 948 diff['added'].append(toProjects[proj])
936 949
937 return diff 950 return diff
951
952
953class GitcManifest(XmlManifest):
954
955 def __init__(self, repodir, gitc_client_name):
956 """Initialize the GitcManifest object."""
957 super(GitcManifest, self).__init__(repodir)
958 self.isGitcClient = True
959 self.gitc_client_name = gitc_client_name
960 self.gitc_client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
961 gitc_client_name)
962 self.manifestFile = os.path.join(self.gitc_client_dir, '.manifest')
963
964 def _ParseProject(self, node, parent = None):
965 """Override _ParseProject and add support for GITC specific attributes."""
966 return super(GitcManifest, self)._ParseProject(
967 node, parent=parent, old_revision=node.getAttribute('old-revision'))
968
969 def _output_manifest_project_extras(self, p, e):
970 """Output GITC Specific Project attributes"""
971 if p.old_revision:
972 e.setAttribute('old-revision', str(p.old_revision))
973
diff --git a/project.py b/project.py
index a117f4df..e3c3bd51 100644
--- a/project.py
+++ b/project.py
@@ -13,7 +13,6 @@
13# limitations under the License. 13# limitations under the License.
14 14
15from __future__ import print_function 15from __future__ import print_function
16import contextlib
17import errno 16import errno
18import filecmp 17import filecmp
19import glob 18import glob
@@ -31,7 +30,8 @@ import traceback
31 30
32from color import Coloring 31from color import Coloring
33from git_command import GitCommand, git_require 32from git_command import GitCommand, git_require
34from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE 33from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
34 ID_RE
35from error import GitError, HookError, UploadError, DownloadError 35from error import GitError, HookError, UploadError, DownloadError
36from error import ManifestInvalidRevisionError 36from error import ManifestInvalidRevisionError
37from error import NoManifestException 37from error import NoManifestException
@@ -45,6 +45,7 @@ if not is_python3():
45 input = raw_input 45 input = raw_input
46 # pylint:enable=W0622 46 # pylint:enable=W0622
47 47
48
48def _lwrite(path, content): 49def _lwrite(path, content):
49 lock = '%s.lock' % path 50 lock = '%s.lock' % path
50 51
@@ -60,17 +61,27 @@ def _lwrite(path, content):
60 os.remove(lock) 61 os.remove(lock)
61 raise 62 raise
62 63
64
63def _error(fmt, *args): 65def _error(fmt, *args):
64 msg = fmt % args 66 msg = fmt % args
65 print('error: %s' % msg, file=sys.stderr) 67 print('error: %s' % msg, file=sys.stderr)
66 68
69
70def _warn(fmt, *args):
71 msg = fmt % args
72 print('warn: %s' % msg, file=sys.stderr)
73
74
67def not_rev(r): 75def not_rev(r):
68 return '^' + r 76 return '^' + r
69 77
78
70def sq(r): 79def sq(r):
71 return "'" + r.replace("'", "'\''") + "'" 80 return "'" + r.replace("'", "'\''") + "'"
72 81
73_project_hook_list = None 82_project_hook_list = None
83
84
74def _ProjectHooks(): 85def _ProjectHooks():
75 """List the hooks present in the 'hooks' directory. 86 """List the hooks present in the 'hooks' directory.
76 87
@@ -104,15 +115,14 @@ class DownloadedChange(object):
104 @property 115 @property
105 def commits(self): 116 def commits(self):
106 if self._commit_cache is None: 117 if self._commit_cache is None:
107 self._commit_cache = self.project.bare_git.rev_list( 118 self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
108 '--abbrev=8', 119 '--abbrev-commit',
109 '--abbrev-commit', 120 '--pretty=oneline',
110 '--pretty=oneline', 121 '--reverse',
111 '--reverse', 122 '--date-order',
112 '--date-order', 123 not_rev(self.base),
113 not_rev(self.base), 124 self.commit,
114 self.commit, 125 '--')
115 '--')
116 return self._commit_cache 126 return self._commit_cache
117 127
118 128
@@ -131,36 +141,36 @@ class ReviewableBranch(object):
131 @property 141 @property
132 def commits(self): 142 def commits(self):
133 if self._commit_cache is None: 143 if self._commit_cache is None:
134 self._commit_cache = self.project.bare_git.rev_list( 144 self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
135 '--abbrev=8', 145 '--abbrev-commit',
136 '--abbrev-commit', 146 '--pretty=oneline',
137 '--pretty=oneline', 147 '--reverse',
138 '--reverse', 148 '--date-order',
139 '--date-order', 149 not_rev(self.base),
140 not_rev(self.base), 150 R_HEADS + self.name,
141 R_HEADS + self.name, 151 '--')
142 '--')
143 return self._commit_cache 152 return self._commit_cache
144 153
145 @property 154 @property
146 def unabbrev_commits(self): 155 def unabbrev_commits(self):
147 r = dict() 156 r = dict()
148 for commit in self.project.bare_git.rev_list( 157 for commit in self.project.bare_git.rev_list(not_rev(self.base),
149 not_rev(self.base), 158 R_HEADS + self.name,
150 R_HEADS + self.name, 159 '--'):
151 '--'):
152 r[commit[0:8]] = commit 160 r[commit[0:8]] = commit
153 return r 161 return r
154 162
155 @property 163 @property
156 def date(self): 164 def date(self):
157 return self.project.bare_git.log( 165 return self.project.bare_git.log('--pretty=format:%cd',
158 '--pretty=format:%cd', 166 '-n', '1',
159 '-n', '1', 167 R_HEADS + self.name,
160 R_HEADS + self.name, 168 '--')
161 '--')
162 169
163 def UploadForReview(self, people, auto_topic=False, draft=False, dest_branch=None): 170 def UploadForReview(self, people,
171 auto_topic=False,
172 draft=False,
173 dest_branch=None):
164 self.project.UploadForReview(self.name, 174 self.project.UploadForReview(self.name,
165 people, 175 people,
166 auto_topic=auto_topic, 176 auto_topic=auto_topic,
@@ -170,8 +180,8 @@ class ReviewableBranch(object):
170 def GetPublishedRefs(self): 180 def GetPublishedRefs(self):
171 refs = {} 181 refs = {}
172 output = self.project.bare_git.ls_remote( 182 output = self.project.bare_git.ls_remote(
173 self.branch.remote.SshReviewUrl(self.project.UserEmail), 183 self.branch.remote.SshReviewUrl(self.project.UserEmail),
174 'refs/changes/*') 184 'refs/changes/*')
175 for line in output.split('\n'): 185 for line in output.split('\n'):
176 try: 186 try:
177 (sha, ref) = line.split() 187 (sha, ref) = line.split()
@@ -181,7 +191,9 @@ class ReviewableBranch(object):
181 191
182 return refs 192 return refs
183 193
194
184class StatusColoring(Coloring): 195class StatusColoring(Coloring):
196
185 def __init__(self, config): 197 def __init__(self, config):
186 Coloring.__init__(self, config, 'status') 198 Coloring.__init__(self, config, 'status')
187 self.project = self.printer('header', attr='bold') 199 self.project = self.printer('header', attr='bold')
@@ -195,17 +207,22 @@ class StatusColoring(Coloring):
195 207
196 208
197class DiffColoring(Coloring): 209class DiffColoring(Coloring):
210
198 def __init__(self, config): 211 def __init__(self, config):
199 Coloring.__init__(self, config, 'diff') 212 Coloring.__init__(self, config, 'diff')
200 self.project = self.printer('header', attr='bold') 213 self.project = self.printer('header', attr='bold')
201 214
215
202class _Annotation(object): 216class _Annotation(object):
217
203 def __init__(self, name, value, keep): 218 def __init__(self, name, value, keep):
204 self.name = name 219 self.name = name
205 self.value = value 220 self.value = value
206 self.keep = keep 221 self.keep = keep
207 222
223
208class _CopyFile(object): 224class _CopyFile(object):
225
209 def __init__(self, src, dest, abssrc, absdest): 226 def __init__(self, src, dest, abssrc, absdest):
210 self.src = src 227 self.src = src
211 self.dest = dest 228 self.dest = dest
@@ -233,7 +250,9 @@ class _CopyFile(object):
233 except IOError: 250 except IOError:
234 _error('Cannot copy file %s to %s', src, dest) 251 _error('Cannot copy file %s to %s', src, dest)
235 252
253
236class _LinkFile(object): 254class _LinkFile(object):
255
237 def __init__(self, git_worktree, src, dest, relsrc, absdest): 256 def __init__(self, git_worktree, src, dest, relsrc, absdest):
238 self.git_worktree = git_worktree 257 self.git_worktree = git_worktree
239 self.src = src 258 self.src = src
@@ -246,7 +265,7 @@ class _LinkFile(object):
246 if not os.path.islink(absDest) or (os.readlink(absDest) != relSrc): 265 if not os.path.islink(absDest) or (os.readlink(absDest) != relSrc):
247 try: 266 try:
248 # remove existing file first, since it might be read-only 267 # remove existing file first, since it might be read-only
249 if os.path.exists(absDest): 268 if os.path.lexists(absDest):
250 os.remove(absDest) 269 os.remove(absDest)
251 else: 270 else:
252 dest_dir = os.path.dirname(absDest) 271 dest_dir = os.path.dirname(absDest)
@@ -272,7 +291,7 @@ class _LinkFile(object):
272 absDestDir = self.abs_dest 291 absDestDir = self.abs_dest
273 if os.path.exists(absDestDir) and not os.path.isdir(absDestDir): 292 if os.path.exists(absDestDir) and not os.path.isdir(absDestDir):
274 _error('Link error: src with wildcard, %s must be a directory', 293 _error('Link error: src with wildcard, %s must be a directory',
275 absDestDir) 294 absDestDir)
276 else: 295 else:
277 absSrcFiles = glob.glob(absSrc) 296 absSrcFiles = glob.glob(absSrc)
278 for absSrcFile in absSrcFiles: 297 for absSrcFile in absSrcFiles:
@@ -289,7 +308,9 @@ class _LinkFile(object):
289 relSrc = os.path.join(relSrcDir, srcFile) 308 relSrc = os.path.join(relSrcDir, srcFile)
290 self.__linkIt(relSrc, absDest) 309 self.__linkIt(relSrc, absDest)
291 310
311
292class RemoteSpec(object): 312class RemoteSpec(object):
313
293 def __init__(self, 314 def __init__(self,
294 name, 315 name,
295 url=None, 316 url=None,
@@ -300,7 +321,9 @@ class RemoteSpec(object):
300 self.review = review 321 self.review = review
301 self.revision = revision 322 self.revision = revision
302 323
324
303class RepoHook(object): 325class RepoHook(object):
326
304 """A RepoHook contains information about a script to run as a hook. 327 """A RepoHook contains information about a script to run as a hook.
305 328
306 Hooks are used to run a python script before running an upload (for instance, 329 Hooks are used to run a python script before running an upload (for instance,
@@ -313,6 +336,7 @@ class RepoHook(object):
313 Hooks are always python. When a hook is run, we will load the hook into the 336 Hooks are always python. When a hook is run, we will load the hook into the
314 interpreter and execute its main() function. 337 interpreter and execute its main() function.
315 """ 338 """
339
316 def __init__(self, 340 def __init__(self,
317 hook_type, 341 hook_type,
318 hooks_project, 342 hooks_project,
@@ -427,8 +451,8 @@ class RepoHook(object):
427 ' %s\n' 451 ' %s\n'
428 '\n' 452 '\n'
429 'Do you want to allow this script to run ' 453 'Do you want to allow this script to run '
430 '(yes/yes-never-ask-again/NO)? ') % ( 454 '(yes/yes-never-ask-again/NO)? ') % (self._GetMustVerb(),
431 self._GetMustVerb(), self._script_fullpath) 455 self._script_fullpath)
432 response = input(prompt).lower() 456 response = input(prompt).lower()
433 print() 457 print()
434 458
@@ -472,19 +496,18 @@ class RepoHook(object):
472 496
473 # Exec, storing global context in the context dict. We catch exceptions 497 # Exec, storing global context in the context dict. We catch exceptions
474 # and convert to a HookError w/ just the failing traceback. 498 # and convert to a HookError w/ just the failing traceback.
475 context = {} 499 context = {'__file__': self._script_fullpath}
476 try: 500 try:
477 exec(compile(open(self._script_fullpath).read(), 501 exec(compile(open(self._script_fullpath).read(),
478 self._script_fullpath, 'exec'), context) 502 self._script_fullpath, 'exec'), context)
479 except Exception: 503 except Exception:
480 raise HookError('%s\nFailed to import %s hook; see traceback above.' % ( 504 raise HookError('%s\nFailed to import %s hook; see traceback above.' %
481 traceback.format_exc(), self._hook_type)) 505 (traceback.format_exc(), self._hook_type))
482 506
483 # Running the script should have defined a main() function. 507 # Running the script should have defined a main() function.
484 if 'main' not in context: 508 if 'main' not in context:
485 raise HookError('Missing main() in: "%s"' % self._script_fullpath) 509 raise HookError('Missing main() in: "%s"' % self._script_fullpath)
486 510
487
488 # Add 'hook_should_take_kwargs' to the arguments to be passed to main. 511 # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
489 # We don't actually want hooks to define their main with this argument-- 512 # We don't actually want hooks to define their main with this argument--
490 # it's there to remind them that their hook should always take **kwargs. 513 # it's there to remind them that their hook should always take **kwargs.
@@ -502,8 +525,8 @@ class RepoHook(object):
502 context['main'](**kwargs) 525 context['main'](**kwargs)
503 except Exception: 526 except Exception:
504 raise HookError('%s\nFailed to run main() for %s hook; see traceback ' 527 raise HookError('%s\nFailed to run main() for %s hook; see traceback '
505 'above.' % ( 528 'above.' % (traceback.format_exc(),
506 traceback.format_exc(), self._hook_type)) 529 self._hook_type))
507 finally: 530 finally:
508 # Restore sys.path and CWD. 531 # Restore sys.path and CWD.
509 sys.path = orig_syspath 532 sys.path = orig_syspath
@@ -527,8 +550,8 @@ class RepoHook(object):
527 to run a required hook (from _CheckForHookApproval). 550 to run a required hook (from _CheckForHookApproval).
528 """ 551 """
529 # No-op if there is no hooks project or if hook is disabled. 552 # No-op if there is no hooks project or if hook is disabled.
530 if ((not self._hooks_project) or 553 if ((not self._hooks_project) or (self._hook_type not in
531 (self._hook_type not in self._hooks_project.enabled_repo_hooks)): 554 self._hooks_project.enabled_repo_hooks)):
532 return 555 return
533 556
534 # Bail with a nice error if we can't find the hook. 557 # Bail with a nice error if we can't find the hook.
@@ -550,6 +573,7 @@ class Project(object):
550 # These objects can only be used by a single working tree. 573 # These objects can only be used by a single working tree.
551 working_tree_files = ['config', 'packed-refs', 'shallow'] 574 working_tree_files = ['config', 'packed-refs', 'shallow']
552 working_tree_dirs = ['logs', 'refs'] 575 working_tree_dirs = ['logs', 'refs']
576
553 def __init__(self, 577 def __init__(self,
554 manifest, 578 manifest,
555 name, 579 name,
@@ -569,7 +593,8 @@ class Project(object):
569 parent=None, 593 parent=None,
570 is_derived=False, 594 is_derived=False,
571 dest_branch=None, 595 dest_branch=None,
572 optimized_fetch=False): 596 optimized_fetch=False,
597 old_revision=None):
573 """Init a Project object. 598 """Init a Project object.
574 599
575 Args: 600 Args:
@@ -593,6 +618,7 @@ class Project(object):
593 dest_branch: The branch to which to push changes for review by default. 618 dest_branch: The branch to which to push changes for review by default.
594 optimized_fetch: If True, when a project is set to a sha1 revision, only 619 optimized_fetch: If True, when a project is set to a sha1 revision, only
595 fetch from the remote if the sha1 is not present locally. 620 fetch from the remote if the sha1 is not present locally.
621 old_revision: saved git commit id for open GITC projects.
596 """ 622 """
597 self.manifest = manifest 623 self.manifest = manifest
598 self.name = name 624 self.name = name
@@ -606,9 +632,9 @@ class Project(object):
606 self.relpath = relpath 632 self.relpath = relpath
607 self.revisionExpr = revisionExpr 633 self.revisionExpr = revisionExpr
608 634
609 if revisionId is None \ 635 if revisionId is None \
610 and revisionExpr \ 636 and revisionExpr \
611 and IsId(revisionExpr): 637 and IsId(revisionExpr):
612 self.revisionId = revisionExpr 638 self.revisionId = revisionExpr
613 else: 639 else:
614 self.revisionId = revisionId 640 self.revisionId = revisionId
@@ -628,9 +654,8 @@ class Project(object):
628 self.copyfiles = [] 654 self.copyfiles = []
629 self.linkfiles = [] 655 self.linkfiles = []
630 self.annotations = [] 656 self.annotations = []
631 self.config = GitConfig.ForRepository( 657 self.config = GitConfig.ForRepository(gitdir=self.gitdir,
632 gitdir=self.gitdir, 658 defaults=self.manifest.globalConfig)
633 defaults=self.manifest.globalConfig)
634 659
635 if self.worktree: 660 if self.worktree:
636 self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir) 661 self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
@@ -640,6 +665,7 @@ class Project(object):
640 self.bare_ref = GitRefs(gitdir) 665 self.bare_ref = GitRefs(gitdir)
641 self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir) 666 self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
642 self.dest_branch = dest_branch 667 self.dest_branch = dest_branch
668 self.old_revision = old_revision
643 669
644 # This will be filled in if a project is later identified to be the 670 # This will be filled in if a project is later identified to be the
645 # project containing repo hooks. 671 # project containing repo hooks.
@@ -767,7 +793,7 @@ class Project(object):
767 """ 793 """
768 expanded_manifest_groups = manifest_groups or ['default'] 794 expanded_manifest_groups = manifest_groups or ['default']
769 expanded_project_groups = ['all'] + (self.groups or []) 795 expanded_project_groups = ['all'] + (self.groups or [])
770 if not 'notdefault' in expanded_project_groups: 796 if 'notdefault' not in expanded_project_groups:
771 expanded_project_groups += ['default'] 797 expanded_project_groups += ['default']
772 798
773 matched = False 799 matched = False
@@ -779,7 +805,7 @@ class Project(object):
779 805
780 return matched 806 return matched
781 807
782## Status Display ## 808# Status Display ##
783 def UncommitedFiles(self, get_all=True): 809 def UncommitedFiles(self, get_all=True):
784 """Returns a list of strings, uncommitted files in the git tree. 810 """Returns a list of strings, uncommitted files in the git tree.
785 811
@@ -831,7 +857,7 @@ class Project(object):
831 output: If specified, redirect the output to this object. 857 output: If specified, redirect the output to this object.
832 """ 858 """
833 if not os.path.isdir(self.worktree): 859 if not os.path.isdir(self.worktree):
834 if output_redir == None: 860 if output_redir is None:
835 output_redir = sys.stdout 861 output_redir = sys.stdout
836 print(file=output_redir) 862 print(file=output_redir)
837 print('project %s/' % self.relpath, file=output_redir) 863 print('project %s/' % self.relpath, file=output_redir)
@@ -850,7 +876,7 @@ class Project(object):
850 return 'CLEAN' 876 return 'CLEAN'
851 877
852 out = StatusColoring(self.config) 878 out = StatusColoring(self.config)
853 if not output_redir == None: 879 if output_redir is not None:
854 out.redirect(output_redir) 880 out.redirect(output_redir)
855 out.project('project %-40s', self.relpath + '/ ') 881 out.project('project %-40s', self.relpath + '/ ')
856 882
@@ -893,7 +919,7 @@ class Project(object):
893 919
894 if i and i.src_path: 920 if i and i.src_path:
895 line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status, 921 line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
896 i.src_path, p, i.level) 922 i.src_path, p, i.level)
897 else: 923 else:
898 line = ' %s%s\t%s' % (i_status, f_status, p) 924 line = ' %s%s\t%s' % (i_status, f_status, p)
899 925
@@ -936,7 +962,7 @@ class Project(object):
936 p.Wait() 962 p.Wait()
937 963
938 964
939## Publish / Upload ## 965# Publish / Upload ##
940 966
941 def WasPublished(self, branch, all_refs=None): 967 def WasPublished(self, branch, all_refs=None):
942 """Was the branch published (uploaded) for code review? 968 """Was the branch published (uploaded) for code review?
@@ -1079,7 +1105,7 @@ class Project(object):
1079 message=msg) 1105 message=msg)
1080 1106
1081 1107
1082## Sync ## 1108# Sync ##
1083 1109
1084 def _ExtractArchive(self, tarpath, path=None): 1110 def _ExtractArchive(self, tarpath, path=None):
1085 """Extract the given tar on its current location 1111 """Extract the given tar on its current location
@@ -1093,26 +1119,25 @@ class Project(object):
1093 tar.extractall(path=path) 1119 tar.extractall(path=path)
1094 return True 1120 return True
1095 except (IOError, tarfile.TarError) as e: 1121 except (IOError, tarfile.TarError) as e:
1096 print("error: Cannot extract archive %s: " 1122 _error("Cannot extract archive %s: %s", tarpath, str(e))
1097 "%s" % (tarpath, str(e)), file=sys.stderr)
1098 return False 1123 return False
1099 1124
1100 def Sync_NetworkHalf(self, 1125 def Sync_NetworkHalf(self,
1101 quiet=False, 1126 quiet=False,
1102 is_new=None, 1127 is_new=None,
1103 current_branch_only=False, 1128 current_branch_only=False,
1104 force_sync=False, 1129 force_sync=False,
1105 clone_bundle=True, 1130 clone_bundle=True,
1106 no_tags=False, 1131 no_tags=False,
1107 archive=False, 1132 archive=False,
1108 optimized_fetch=False): 1133 optimized_fetch=False,
1134 prune=False):
1109 """Perform only the network IO portion of the sync process. 1135 """Perform only the network IO portion of the sync process.
1110 Local working directory/branch state is not affected. 1136 Local working directory/branch state is not affected.
1111 """ 1137 """
1112 if archive and not isinstance(self, MetaProject): 1138 if archive and not isinstance(self, MetaProject):
1113 if self.remote.url.startswith(('http://', 'https://')): 1139 if self.remote.url.startswith(('http://', 'https://')):
1114 print("error: %s: Cannot fetch archives from http/https " 1140 _error("%s: Cannot fetch archives from http/https remotes.", self.name)
1115 "remotes." % self.name, file=sys.stderr)
1116 return False 1141 return False
1117 1142
1118 name = self.relpath.replace('\\', '/') 1143 name = self.relpath.replace('\\', '/')
@@ -1123,7 +1148,7 @@ class Project(object):
1123 try: 1148 try:
1124 self._FetchArchive(tarpath, cwd=topdir) 1149 self._FetchArchive(tarpath, cwd=topdir)
1125 except GitError as e: 1150 except GitError as e:
1126 print('error: %s' % str(e), file=sys.stderr) 1151 _error('%s', e)
1127 return False 1152 return False
1128 1153
1129 # From now on, we only need absolute tarpath 1154 # From now on, we only need absolute tarpath
@@ -1134,8 +1159,7 @@ class Project(object):
1134 try: 1159 try:
1135 os.remove(tarpath) 1160 os.remove(tarpath)
1136 except OSError as e: 1161 except OSError as e:
1137 print("warn: Cannot remove archive %s: " 1162 _warn("Cannot remove archive %s: %s", tarpath, str(e))
1138 "%s" % (tarpath, str(e)), file=sys.stderr)
1139 self._CopyAndLinkFiles() 1163 self._CopyAndLinkFiles()
1140 return True 1164 return True
1141 if is_new is None: 1165 if is_new is None:
@@ -1160,8 +1184,8 @@ class Project(object):
1160 alt_dir = None 1184 alt_dir = None
1161 1185
1162 if clone_bundle \ 1186 if clone_bundle \
1163 and alt_dir is None \ 1187 and alt_dir is None \
1164 and self._ApplyCloneBundle(initial=is_new, quiet=quiet): 1188 and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
1165 is_new = False 1189 is_new = False
1166 1190
1167 if not current_branch_only: 1191 if not current_branch_only:
@@ -1173,12 +1197,13 @@ class Project(object):
1173 elif self.manifest.default.sync_c: 1197 elif self.manifest.default.sync_c:
1174 current_branch_only = True 1198 current_branch_only = True
1175 1199
1176 need_to_fetch = not (optimized_fetch and \ 1200 need_to_fetch = not (optimized_fetch and
1177 (ID_RE.match(self.revisionExpr) and self._CheckForSha1())) 1201 (ID_RE.match(self.revisionExpr) and
1178 if (need_to_fetch 1202 self._CheckForSha1()))
1179 and not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir, 1203 if (need_to_fetch and
1180 current_branch_only=current_branch_only, 1204 not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
1181 no_tags=no_tags)): 1205 current_branch_only=current_branch_only,
1206 no_tags=no_tags, prune=prune)):
1182 return False 1207 return False
1183 1208
1184 if self.worktree: 1209 if self.worktree:
@@ -1195,6 +1220,8 @@ class Project(object):
1195 self._InitHooks() 1220 self._InitHooks()
1196 1221
1197 def _CopyAndLinkFiles(self): 1222 def _CopyAndLinkFiles(self):
1223 if self.manifest.isGitcClient:
1224 return
1198 for copyfile in self.copyfiles: 1225 for copyfile in self.copyfiles:
1199 copyfile._Copy() 1226 copyfile._Copy()
1200 for linkfile in self.linkfiles: 1227 for linkfile in self.linkfiles:
@@ -1213,9 +1240,8 @@ class Project(object):
1213 try: 1240 try:
1214 return self.bare_git.rev_list(self.revisionExpr, '-1')[0] 1241 return self.bare_git.rev_list(self.revisionExpr, '-1')[0]
1215 except GitError: 1242 except GitError:
1216 raise ManifestInvalidRevisionError( 1243 raise ManifestInvalidRevisionError('revision %s in %s not found' %
1217 'revision %s in %s not found' % (self.revisionExpr, 1244 (self.revisionExpr, self.name))
1218 self.name))
1219 1245
1220 def GetRevisionId(self, all_refs=None): 1246 def GetRevisionId(self, all_refs=None):
1221 if self.revisionId: 1247 if self.revisionId:
@@ -1230,9 +1256,8 @@ class Project(object):
1230 try: 1256 try:
1231 return self.bare_git.rev_parse('--verify', '%s^0' % rev) 1257 return self.bare_git.rev_parse('--verify', '%s^0' % rev)
1232 except GitError: 1258 except GitError:
1233 raise ManifestInvalidRevisionError( 1259 raise ManifestInvalidRevisionError('revision %s in %s not found' %
1234 'revision %s in %s not found' % (self.revisionExpr, 1260 (self.revisionExpr, self.name))
1235 self.name))
1236 1261
1237 def Sync_LocalHalf(self, syncbuf, force_sync=False): 1262 def Sync_LocalHalf(self, syncbuf, force_sync=False):
1238 """Perform only the local IO portion of the sync process. 1263 """Perform only the local IO portion of the sync process.
@@ -1270,6 +1295,8 @@ class Project(object):
1270 # Except if the head needs to be detached 1295 # Except if the head needs to be detached
1271 # 1296 #
1272 if not syncbuf.detach_head: 1297 if not syncbuf.detach_head:
1298 # The copy/linkfile config may have changed.
1299 self._CopyAndLinkFiles()
1273 return 1300 return
1274 else: 1301 else:
1275 lost = self._revlist(not_rev(revid), HEAD) 1302 lost = self._revlist(not_rev(revid), HEAD)
@@ -1287,6 +1314,8 @@ class Project(object):
1287 if head == revid: 1314 if head == revid:
1288 # No changes; don't do anything further. 1315 # No changes; don't do anything further.
1289 # 1316 #
1317 # The copy/linkfile config may have changed.
1318 self._CopyAndLinkFiles()
1290 return 1319 return
1291 1320
1292 branch = self.GetBranch(branch) 1321 branch = self.GetBranch(branch)
@@ -1317,8 +1346,8 @@ class Project(object):
1317 # to rewrite the published commits so we punt. 1346 # to rewrite the published commits so we punt.
1318 # 1347 #
1319 syncbuf.fail(self, 1348 syncbuf.fail(self,
1320 "branch %s is published (but not merged) and is now %d commits behind" 1349 "branch %s is published (but not merged) and is now "
1321 % (branch.name, len(upstream_gain))) 1350 "%d commits behind" % (branch.name, len(upstream_gain)))
1322 return 1351 return
1323 elif pub == head: 1352 elif pub == head:
1324 # All published commits are merged, and thus we are a 1353 # All published commits are merged, and thus we are a
@@ -1412,7 +1441,7 @@ class Project(object):
1412 remote = self.GetRemote(self.remote.name) 1441 remote = self.GetRemote(self.remote.name)
1413 1442
1414 cmd = ['fetch', remote.name] 1443 cmd = ['fetch', remote.name]
1415 cmd.append('refs/changes/%2.2d/%d/%d' \ 1444 cmd.append('refs/changes/%2.2d/%d/%d'
1416 % (change_id % 100, change_id, patch_id)) 1445 % (change_id % 100, change_id, patch_id))
1417 if GitCommand(self, cmd, bare=True).Wait() != 0: 1446 if GitCommand(self, cmd, bare=True).Wait() != 0:
1418 return None 1447 return None
@@ -1423,11 +1452,13 @@ class Project(object):
1423 self.bare_git.rev_parse('FETCH_HEAD')) 1452 self.bare_git.rev_parse('FETCH_HEAD'))
1424 1453
1425 1454
1426## Branch Management ## 1455# Branch Management ##
1427 1456
1428 def StartBranch(self, name): 1457 def StartBranch(self, name, branch_merge=''):
1429 """Create a new branch off the manifest's revision. 1458 """Create a new branch off the manifest's revision.
1430 """ 1459 """
1460 if not branch_merge:
1461 branch_merge = self.revisionExpr
1431 head = self.work_git.GetHead() 1462 head = self.work_git.GetHead()
1432 if head == (R_HEADS + name): 1463 if head == (R_HEADS + name):
1433 return True 1464 return True
@@ -1441,9 +1472,9 @@ class Project(object):
1441 1472
1442 branch = self.GetBranch(name) 1473 branch = self.GetBranch(name)
1443 branch.remote = self.GetRemote(self.remote.name) 1474 branch.remote = self.GetRemote(self.remote.name)
1444 branch.merge = self.revisionExpr 1475 branch.merge = branch_merge
1445 if not branch.merge.startswith('refs/') and not ID_RE.match(self.revisionExpr): 1476 if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
1446 branch.merge = R_HEADS + self.revisionExpr 1477 branch.merge = R_HEADS + branch_merge
1447 revid = self.GetRevisionId(all_refs) 1478 revid = self.GetRevisionId(all_refs)
1448 1479
1449 if head.startswith(R_HEADS): 1480 if head.startswith(R_HEADS):
@@ -1451,7 +1482,6 @@ class Project(object):
1451 head = all_refs[head] 1482 head = all_refs[head]
1452 except KeyError: 1483 except KeyError:
1453 head = None 1484 head = None
1454
1455 if revid and head and revid == head: 1485 if revid and head and revid == head:
1456 ref = os.path.join(self.gitdir, R_HEADS + name) 1486 ref = os.path.join(self.gitdir, R_HEADS + name)
1457 try: 1487 try:
@@ -1572,8 +1602,6 @@ class Project(object):
1572 1602
1573 if kill: 1603 if kill:
1574 old = self.bare_git.GetHead() 1604 old = self.bare_git.GetHead()
1575 if old is None:
1576 old = 'refs/heads/please_never_use_this_as_a_branch_name'
1577 1605
1578 try: 1606 try:
1579 self.bare_git.DetachHead(rev) 1607 self.bare_git.DetachHead(rev)
@@ -1585,7 +1613,10 @@ class Project(object):
1585 capture_stderr=True) 1613 capture_stderr=True)
1586 b.Wait() 1614 b.Wait()
1587 finally: 1615 finally:
1588 self.bare_git.SetHead(old) 1616 if ID_RE.match(old):
1617 self.bare_git.DetachHead(old)
1618 else:
1619 self.bare_git.SetHead(old)
1589 left = self._allrefs 1620 left = self._allrefs
1590 1621
1591 for branch in kill: 1622 for branch in kill:
@@ -1608,10 +1639,11 @@ class Project(object):
1608 return kept 1639 return kept
1609 1640
1610 1641
1611## Submodule Management ## 1642# Submodule Management ##
1612 1643
1613 def GetRegisteredSubprojects(self): 1644 def GetRegisteredSubprojects(self):
1614 result = [] 1645 result = []
1646
1615 def rec(subprojects): 1647 def rec(subprojects):
1616 if not subprojects: 1648 if not subprojects:
1617 return 1649 return
@@ -1646,6 +1678,7 @@ class Project(object):
1646 1678
1647 re_path = re.compile(r'^submodule\.([^.]+)\.path=(.*)$') 1679 re_path = re.compile(r'^submodule\.([^.]+)\.path=(.*)$')
1648 re_url = re.compile(r'^submodule\.([^.]+)\.url=(.*)$') 1680 re_url = re.compile(r'^submodule\.([^.]+)\.url=(.*)$')
1681
1649 def parse_gitmodules(gitdir, rev): 1682 def parse_gitmodules(gitdir, rev):
1650 cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev] 1683 cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
1651 try: 1684 try:
@@ -1755,7 +1788,7 @@ class Project(object):
1755 return result 1788 return result
1756 1789
1757 1790
1758## Direct Git Commands ## 1791# Direct Git Commands ##
1759 def _CheckForSha1(self): 1792 def _CheckForSha1(self):
1760 try: 1793 try:
1761 # if revision (sha or tag) is not present then following function 1794 # if revision (sha or tag) is not present then following function
@@ -1779,13 +1812,13 @@ class Project(object):
1779 if command.Wait() != 0: 1812 if command.Wait() != 0:
1780 raise GitError('git archive %s: %s' % (self.name, command.stderr)) 1813 raise GitError('git archive %s: %s' % (self.name, command.stderr))
1781 1814
1782
1783 def _RemoteFetch(self, name=None, 1815 def _RemoteFetch(self, name=None,
1784 current_branch_only=False, 1816 current_branch_only=False,
1785 initial=False, 1817 initial=False,
1786 quiet=False, 1818 quiet=False,
1787 alt_dir=None, 1819 alt_dir=None,
1788 no_tags=False): 1820 no_tags=False,
1821 prune=False):
1789 1822
1790 is_sha1 = False 1823 is_sha1 = False
1791 tag_name = None 1824 tag_name = None
@@ -1898,6 +1931,9 @@ class Project(object):
1898 else: 1931 else:
1899 cmd.append('--tags') 1932 cmd.append('--tags')
1900 1933
1934 if prune:
1935 cmd.append('--prune')
1936
1901 spec = [] 1937 spec = []
1902 if not current_branch_only: 1938 if not current_branch_only:
1903 # Fetch whole repo 1939 # Fetch whole repo
@@ -1939,9 +1975,9 @@ class Project(object):
1939 break 1975 break
1940 continue 1976 continue
1941 elif current_branch_only and is_sha1 and ret == 128: 1977 elif current_branch_only and is_sha1 and ret == 128:
1942 # Exit code 128 means "couldn't find the ref you asked for"; if we're in sha1 1978 # Exit code 128 means "couldn't find the ref you asked for"; if we're
1943 # mode, we just tried sync'ing from the upstream field; it doesn't exist, thus 1979 # in sha1 mode, we just tried sync'ing from the upstream field; it
1944 # abort the optimization attempt and do a full sync. 1980 # doesn't exist, thus abort the optimization attempt and do a full sync.
1945 break 1981 break
1946 elif ret < 0: 1982 elif ret < 0:
1947 # Git died with a signal, exit immediately 1983 # Git died with a signal, exit immediately
@@ -1968,20 +2004,24 @@ class Project(object):
1968 initial=False, quiet=quiet, alt_dir=alt_dir) 2004 initial=False, quiet=quiet, alt_dir=alt_dir)
1969 if self.clone_depth: 2005 if self.clone_depth:
1970 self.clone_depth = None 2006 self.clone_depth = None
1971 return self._RemoteFetch(name=name, current_branch_only=current_branch_only, 2007 return self._RemoteFetch(name=name,
2008 current_branch_only=current_branch_only,
1972 initial=False, quiet=quiet, alt_dir=alt_dir) 2009 initial=False, quiet=quiet, alt_dir=alt_dir)
1973 2010
1974 return ok 2011 return ok
1975 2012
1976 def _ApplyCloneBundle(self, initial=False, quiet=False): 2013 def _ApplyCloneBundle(self, initial=False, quiet=False):
1977 if initial and (self.manifest.manifestProject.config.GetString('repo.depth') or self.clone_depth): 2014 if initial and \
2015 (self.manifest.manifestProject.config.GetString('repo.depth') or
2016 self.clone_depth):
1978 return False 2017 return False
1979 2018
1980 remote = self.GetRemote(self.remote.name) 2019 remote = self.GetRemote(self.remote.name)
1981 bundle_url = remote.url + '/clone.bundle' 2020 bundle_url = remote.url + '/clone.bundle'
1982 bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url) 2021 bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
1983 if GetSchemeFromUrl(bundle_url) not in ( 2022 if GetSchemeFromUrl(bundle_url) not in ('http', 'https',
1984 'http', 'https', 'persistent-http', 'persistent-https'): 2023 'persistent-http',
2024 'persistent-https'):
1985 return False 2025 return False
1986 2026
1987 bundle_dst = os.path.join(self.gitdir, 'clone.bundle') 2027 bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
@@ -2030,7 +2070,7 @@ class Project(object):
2030 os.remove(tmpPath) 2070 os.remove(tmpPath)
2031 if 'http_proxy' in os.environ and 'darwin' == sys.platform: 2071 if 'http_proxy' in os.environ and 'darwin' == sys.platform:
2032 cmd += ['--proxy', os.environ['http_proxy']] 2072 cmd += ['--proxy', os.environ['http_proxy']]
2033 with self._GetBundleCookieFile(srcUrl, quiet) as cookiefile: 2073 with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, _proxy):
2034 if cookiefile: 2074 if cookiefile:
2035 cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile] 2075 cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
2036 if srcUrl.startswith('persistent-'): 2076 if srcUrl.startswith('persistent-'):
@@ -2078,40 +2118,6 @@ class Project(object):
2078 except OSError: 2118 except OSError:
2079 return False 2119 return False
2080 2120
2081 @contextlib.contextmanager
2082 def _GetBundleCookieFile(self, url, quiet):
2083 if url.startswith('persistent-'):
2084 try:
2085 p = subprocess.Popen(
2086 ['git-remote-persistent-https', '-print_config', url],
2087 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2088 stderr=subprocess.PIPE)
2089 try:
2090 prefix = 'http.cookiefile='
2091 cookiefile = None
2092 for line in p.stdout:
2093 line = line.strip()
2094 if line.startswith(prefix):
2095 cookiefile = line[len(prefix):]
2096 break
2097 # Leave subprocess open, as cookie file may be transient.
2098 if cookiefile:
2099 yield cookiefile
2100 return
2101 finally:
2102 p.stdin.close()
2103 if p.wait():
2104 err_msg = p.stderr.read()
2105 if ' -print_config' in err_msg:
2106 pass # Persistent proxy doesn't support -print_config.
2107 elif not quiet:
2108 print(err_msg, file=sys.stderr)
2109 except OSError as e:
2110 if e.errno == errno.ENOENT:
2111 pass # No persistent proxy.
2112 raise
2113 yield GitConfig.ForUser().GetString('http.cookiefile')
2114
2115 def _Checkout(self, rev, quiet=False): 2121 def _Checkout(self, rev, quiet=False):
2116 cmd = ['checkout'] 2122 cmd = ['checkout']
2117 if quiet: 2123 if quiet:
@@ -2182,12 +2188,13 @@ class Project(object):
2182 try: 2188 try:
2183 self._CheckDirReference(self.objdir, self.gitdir, share_refs=False) 2189 self._CheckDirReference(self.objdir, self.gitdir, share_refs=False)
2184 except GitError as e: 2190 except GitError as e:
2185 print("Retrying clone after deleting %s" % force_sync, file=sys.stderr)
2186 if force_sync: 2191 if force_sync:
2192 print("Retrying clone after deleting %s" %
2193 self.gitdir, file=sys.stderr)
2187 try: 2194 try:
2188 shutil.rmtree(os.path.realpath(self.gitdir)) 2195 shutil.rmtree(os.path.realpath(self.gitdir))
2189 if self.worktree and os.path.exists( 2196 if self.worktree and os.path.exists(os.path.realpath
2190 os.path.realpath(self.worktree)): 2197 (self.worktree)):
2191 shutil.rmtree(os.path.realpath(self.worktree)) 2198 shutil.rmtree(os.path.realpath(self.worktree))
2192 return self._InitGitDir(mirror_git=mirror_git, force_sync=False) 2199 return self._InitGitDir(mirror_git=mirror_git, force_sync=False)
2193 except: 2200 except:
@@ -2246,7 +2253,7 @@ class Project(object):
2246 name = os.path.basename(stock_hook) 2253 name = os.path.basename(stock_hook)
2247 2254
2248 if name in ('commit-msg',) and not self.remote.review \ 2255 if name in ('commit-msg',) and not self.remote.review \
2249 and not self is self.manifest.manifestProject: 2256 and self is not self.manifest.manifestProject:
2250 # Don't install a Gerrit Code Review hook if this 2257 # Don't install a Gerrit Code Review hook if this
2251 # project does not appear to use it for reviews. 2258 # project does not appear to use it for reviews.
2252 # 2259 #
@@ -2261,7 +2268,8 @@ class Project(object):
2261 if filecmp.cmp(stock_hook, dst, shallow=False): 2268 if filecmp.cmp(stock_hook, dst, shallow=False):
2262 os.remove(dst) 2269 os.remove(dst)
2263 else: 2270 else:
2264 _error("%s: Not replacing %s hook", self.relpath, name) 2271 _warn("%s: Not replacing locally modified %s hook",
2272 self.relpath, name)
2265 continue 2273 continue
2266 try: 2274 try:
2267 os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst) 2275 os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
@@ -2320,7 +2328,10 @@ class Project(object):
2320 # Fail if the links are pointing to the wrong place 2328 # Fail if the links are pointing to the wrong place
2321 if src != dst: 2329 if src != dst:
2322 raise GitError('--force-sync not enabled; cannot overwrite a local ' 2330 raise GitError('--force-sync not enabled; cannot overwrite a local '
2323 'work tree') 2331 'work tree. If you\'re comfortable with the '
2332 'possibility of losing the work tree\'s git metadata,'
2333 ' use `repo sync --force-sync {0}` to '
2334 'proceed.'.format(self.relpath))
2324 2335
2325 def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all): 2336 def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
2326 """Update |dotgit| to reference |gitdir|, using symlinks where possible. 2337 """Update |dotgit| to reference |gitdir|, using symlinks where possible.
@@ -2464,6 +2475,7 @@ class Project(object):
2464 return logs 2475 return logs
2465 2476
2466 class _GitGetByExec(object): 2477 class _GitGetByExec(object):
2478
2467 def __init__(self, project, bare, gitdir): 2479 def __init__(self, project, bare, gitdir):
2468 self._project = project 2480 self._project = project
2469 self._bare = bare 2481 self._bare = bare
@@ -2482,8 +2494,8 @@ class Project(object):
2482 if p.Wait() == 0: 2494 if p.Wait() == 0:
2483 out = p.stdout 2495 out = p.stdout
2484 if out: 2496 if out:
2497 # Backslash is not anomalous
2485 return out[:-1].split('\0') # pylint: disable=W1401 2498 return out[:-1].split('\0') # pylint: disable=W1401
2486 # Backslash is not anomalous
2487 return [] 2499 return []
2488 2500
2489 def DiffZ(self, name, *args): 2501 def DiffZ(self, name, *args):
@@ -2509,6 +2521,7 @@ class Project(object):
2509 break 2521 break
2510 2522
2511 class _Info(object): 2523 class _Info(object):
2524
2512 def __init__(self, path, omode, nmode, oid, nid, state): 2525 def __init__(self, path, omode, nmode, oid, nid, state):
2513 self.path = path 2526 self.path = path
2514 self.src_path = None 2527 self.src_path = None
@@ -2611,10 +2624,8 @@ class Project(object):
2611 line = line[:-1] 2624 line = line[:-1]
2612 r.append(line) 2625 r.append(line)
2613 if p.Wait() != 0: 2626 if p.Wait() != 0:
2614 raise GitError('%s rev-list %s: %s' % ( 2627 raise GitError('%s rev-list %s: %s' %
2615 self._project.name, 2628 (self._project.name, str(args), p.stderr))
2616 str(args),
2617 p.stderr))
2618 return r 2629 return r
2619 2630
2620 def __getattr__(self, name): 2631 def __getattr__(self, name):
@@ -2637,6 +2648,7 @@ class Project(object):
2637 A callable object that will try to call git with the named command. 2648 A callable object that will try to call git with the named command.
2638 """ 2649 """
2639 name = name.replace('_', '-') 2650 name = name.replace('_', '-')
2651
2640 def runner(*args, **kwargs): 2652 def runner(*args, **kwargs):
2641 cmdv = [] 2653 cmdv = []
2642 config = kwargs.pop('config', None) 2654 config = kwargs.pop('config', None)
@@ -2659,10 +2671,8 @@ class Project(object):
2659 capture_stdout=True, 2671 capture_stdout=True,
2660 capture_stderr=True) 2672 capture_stderr=True)
2661 if p.Wait() != 0: 2673 if p.Wait() != 0:
2662 raise GitError('%s %s: %s' % ( 2674 raise GitError('%s %s: %s' %
2663 self._project.name, 2675 (self._project.name, name, p.stderr))
2664 name,
2665 p.stderr))
2666 r = p.stdout 2676 r = p.stdout
2667 try: 2677 try:
2668 r = r.decode('utf-8') 2678 r = r.decode('utf-8')
@@ -2675,14 +2685,19 @@ class Project(object):
2675 2685
2676 2686
2677class _PriorSyncFailedError(Exception): 2687class _PriorSyncFailedError(Exception):
2688
2678 def __str__(self): 2689 def __str__(self):
2679 return 'prior sync failed; rebase still in progress' 2690 return 'prior sync failed; rebase still in progress'
2680 2691
2692
2681class _DirtyError(Exception): 2693class _DirtyError(Exception):
2694
2682 def __str__(self): 2695 def __str__(self):
2683 return 'contains uncommitted changes' 2696 return 'contains uncommitted changes'
2684 2697
2698
2685class _InfoMessage(object): 2699class _InfoMessage(object):
2700
2686 def __init__(self, project, text): 2701 def __init__(self, project, text):
2687 self.project = project 2702 self.project = project
2688 self.text = text 2703 self.text = text
@@ -2691,7 +2706,9 @@ class _InfoMessage(object):
2691 syncbuf.out.info('%s/: %s', self.project.relpath, self.text) 2706 syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
2692 syncbuf.out.nl() 2707 syncbuf.out.nl()
2693 2708
2709
2694class _Failure(object): 2710class _Failure(object):
2711
2695 def __init__(self, project, why): 2712 def __init__(self, project, why):
2696 self.project = project 2713 self.project = project
2697 self.why = why 2714 self.why = why
@@ -2702,7 +2719,9 @@ class _Failure(object):
2702 str(self.why)) 2719 str(self.why))
2703 syncbuf.out.nl() 2720 syncbuf.out.nl()
2704 2721
2722
2705class _Later(object): 2723class _Later(object):
2724
2706 def __init__(self, project, action): 2725 def __init__(self, project, action):
2707 self.project = project 2726 self.project = project
2708 self.action = action 2727 self.action = action
@@ -2719,14 +2738,18 @@ class _Later(object):
2719 out.nl() 2738 out.nl()
2720 return False 2739 return False
2721 2740
2741
2722class _SyncColoring(Coloring): 2742class _SyncColoring(Coloring):
2743
2723 def __init__(self, config): 2744 def __init__(self, config):
2724 Coloring.__init__(self, config, 'reposync') 2745 Coloring.__init__(self, config, 'reposync')
2725 self.project = self.printer('header', attr='bold') 2746 self.project = self.printer('header', attr='bold')
2726 self.info = self.printer('info') 2747 self.info = self.printer('info')
2727 self.fail = self.printer('fail', fg='red') 2748 self.fail = self.printer('fail', fg='red')
2728 2749
2750
2729class SyncBuffer(object): 2751class SyncBuffer(object):
2752
2730 def __init__(self, config, detach_head=False): 2753 def __init__(self, config, detach_head=False):
2731 self._messages = [] 2754 self._messages = []
2732 self._failures = [] 2755 self._failures = []
@@ -2782,8 +2805,10 @@ class SyncBuffer(object):
2782 2805
2783 2806
2784class MetaProject(Project): 2807class MetaProject(Project):
2808
2785 """A special project housed under .repo. 2809 """A special project housed under .repo.
2786 """ 2810 """
2811
2787 def __init__(self, manifest, name, gitdir, worktree): 2812 def __init__(self, manifest, name, gitdir, worktree):
2788 Project.__init__(self, 2813 Project.__init__(self,
2789 manifest=manifest, 2814 manifest=manifest,
@@ -2817,10 +2842,9 @@ class MetaProject(Project):
2817 syncbuf.Finish() 2842 syncbuf.Finish()
2818 2843
2819 return GitCommand(self, 2844 return GitCommand(self,
2820 ['update-ref', '-d', 'refs/heads/default'], 2845 ['update-ref', '-d', 'refs/heads/default'],
2821 capture_stdout=True, 2846 capture_stdout=True,
2822 capture_stderr=True).Wait() == 0 2847 capture_stderr=True).Wait() == 0
2823
2824 2848
2825 @property 2849 @property
2826 def LastFetch(self): 2850 def LastFetch(self):
diff --git a/repo b/repo
index 77695651..e5cb8904 100755
--- a/repo
+++ b/repo
@@ -1,8 +1,11 @@
1#!/usr/bin/env python 1#!/usr/bin/env python
2 2
3## repo default configuration 3# repo default configuration
4## 4#
5REPO_URL = 'https://gerrit.googlesource.com/git-repo' 5import os
6REPO_URL = os.environ.get('REPO_URL', None)
7if not REPO_URL:
8 REPO_URL = 'https://gerrit.googlesource.com/git-repo'
6REPO_REV = 'stable' 9REPO_REV = 'stable'
7 10
8# Copyright (C) 2008 Google Inc. 11# Copyright (C) 2008 Google Inc.
@@ -20,7 +23,7 @@ REPO_REV = 'stable'
20# limitations under the License. 23# limitations under the License.
21 24
22# increment this whenever we make important changes to this script 25# increment this whenever we make important changes to this script
23VERSION = (1, 21) 26VERSION = (1, 22)
24 27
25# increment this if the MAINTAINER_KEYS block is modified 28# increment this if the MAINTAINER_KEYS block is modified
26KEYRING_VERSION = (1, 2) 29KEYRING_VERSION = (1, 2)
@@ -101,18 +104,19 @@ JuinEP+AwLAUZ1Bsx9ISC0Agpk2VeHXPL3FGhroEmoMvBzO0kTFGyoeT7PR/BfKv
101-----END PGP PUBLIC KEY BLOCK----- 104-----END PGP PUBLIC KEY BLOCK-----
102""" 105"""
103 106
104GIT = 'git' # our git command 107GIT = 'git' # our git command
105MIN_GIT_VERSION = (1, 7, 2) # minimum supported git version 108MIN_GIT_VERSION = (1, 7, 2) # minimum supported git version
106repodir = '.repo' # name of repo's private directory 109repodir = '.repo' # name of repo's private directory
107S_repo = 'repo' # special repo repository 110S_repo = 'repo' # special repo repository
108S_manifests = 'manifests' # special manifest repository 111S_manifests = 'manifests' # special manifest repository
109REPO_MAIN = S_repo + '/main.py' # main script 112REPO_MAIN = S_repo + '/main.py' # main script
110MIN_PYTHON_VERSION = (2, 6) # minimum supported python version 113MIN_PYTHON_VERSION = (2, 6) # minimum supported python version
114GITC_CONFIG_FILE = '/gitc/.config'
115GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
111 116
112 117
113import errno 118import errno
114import optparse 119import optparse
115import os
116import re 120import re
117import shutil 121import shutil
118import stat 122import stat
@@ -212,14 +216,69 @@ group.add_option('--config-name',
212 dest='config_name', action="store_true", default=False, 216 dest='config_name', action="store_true", default=False,
213 help='Always prompt for name/e-mail') 217 help='Always prompt for name/e-mail')
214 218
219
220def _GitcInitOptions(init_optparse_arg):
221 init_optparse_arg.set_usage("repo gitc-init -u url -c client [options]")
222 g = init_optparse_arg.add_option_group('GITC options')
223 g.add_option('-f', '--manifest-file',
224 dest='manifest_file',
225 help='Optional manifest file to use for this GITC client.')
226 g.add_option('-c', '--gitc-client',
227 dest='gitc_client',
228 help='The name of the gitc_client instance to create or modify.')
229
230_gitc_manifest_dir = None
231
232
233def get_gitc_manifest_dir():
234 global _gitc_manifest_dir
235 if _gitc_manifest_dir is None:
236 _gitc_manifest_dir = ''
237 try:
238 with open(GITC_CONFIG_FILE, 'r') as gitc_config:
239 for line in gitc_config:
240 match = re.match('gitc_dir=(?P<gitc_manifest_dir>.*)', line)
241 if match:
242 _gitc_manifest_dir = match.group('gitc_manifest_dir')
243 except IOError:
244 pass
245 return _gitc_manifest_dir
246
247
248def gitc_parse_clientdir(gitc_fs_path):
249 """Parse a path in the GITC FS and return its client name.
250
251 @param gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
252
253 @returns: The GITC client name
254 """
255 if gitc_fs_path == GITC_FS_ROOT_DIR:
256 return None
257 if not gitc_fs_path.startswith(GITC_FS_ROOT_DIR):
258 manifest_dir = get_gitc_manifest_dir()
259 if manifest_dir == '':
260 return None
261 if manifest_dir[-1] != '/':
262 manifest_dir += '/'
263 if gitc_fs_path == manifest_dir:
264 return None
265 if not gitc_fs_path.startswith(manifest_dir):
266 return None
267 return gitc_fs_path.split(manifest_dir)[1].split('/')[0]
268 return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split('/')[0]
269
270
215class CloneFailure(Exception): 271class CloneFailure(Exception):
272
216 """Indicate the remote clone of repo itself failed. 273 """Indicate the remote clone of repo itself failed.
217 """ 274 """
218 275
219 276
220def _Init(args): 277def _Init(args, gitc_init=False):
221 """Installs repo by cloning it over the network. 278 """Installs repo by cloning it over the network.
222 """ 279 """
280 if gitc_init:
281 _GitcInitOptions(init_optparse)
223 opt, args = init_optparse.parse_args(args) 282 opt, args = init_optparse.parse_args(args)
224 if args: 283 if args:
225 init_optparse.print_usage() 284 init_optparse.print_usage()
@@ -242,6 +301,26 @@ def _Init(args):
242 raise CloneFailure() 301 raise CloneFailure()
243 302
244 try: 303 try:
304 if gitc_init:
305 gitc_manifest_dir = get_gitc_manifest_dir()
306 if not gitc_manifest_dir:
307 _print('fatal: GITC filesystem is not available. Exiting...',
308 file=sys.stderr)
309 sys.exit(1)
310 gitc_client = opt.gitc_client
311 if not gitc_client:
312 gitc_client = gitc_parse_clientdir(os.getcwd())
313 if not gitc_client:
314 _print('fatal: GITC client (-c) is required.', file=sys.stderr)
315 sys.exit(1)
316 client_dir = os.path.join(gitc_manifest_dir, gitc_client)
317 if not os.path.exists(client_dir):
318 os.makedirs(client_dir)
319 os.chdir(client_dir)
320 if os.path.exists(repodir):
321 # This GITC Client has already initialized repo so continue.
322 return
323
245 os.mkdir(repodir) 324 os.mkdir(repodir)
246 except OSError as e: 325 except OSError as e:
247 if e.errno != errno.EEXIST: 326 if e.errno != errno.EEXIST:
@@ -358,8 +437,8 @@ def SetupGnuPG(quiet):
358 cmd = ['gpg', '--import'] 437 cmd = ['gpg', '--import']
359 try: 438 try:
360 proc = subprocess.Popen(cmd, 439 proc = subprocess.Popen(cmd,
361 env = env, 440 env=env,
362 stdin = subprocess.PIPE) 441 stdin=subprocess.PIPE)
363 except OSError as e: 442 except OSError as e:
364 if not quiet: 443 if not quiet:
365 _print('warning: gpg (GnuPG) is not available.', file=sys.stderr) 444 _print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
@@ -385,7 +464,7 @@ def _SetConfig(local, name, value):
385 """Set a git configuration option to the specified value. 464 """Set a git configuration option to the specified value.
386 """ 465 """
387 cmd = [GIT, 'config', name, value] 466 cmd = [GIT, 'config', name, value]
388 if subprocess.Popen(cmd, cwd = local).wait() != 0: 467 if subprocess.Popen(cmd, cwd=local).wait() != 0:
389 raise CloneFailure() 468 raise CloneFailure()
390 469
391 470
@@ -398,9 +477,9 @@ def _InitHttp():
398 n = netrc.netrc() 477 n = netrc.netrc()
399 for host in n.hosts: 478 for host in n.hosts:
400 p = n.hosts[host] 479 p = n.hosts[host]
401 mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2]) 480 mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2])
402 mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2]) 481 mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
403 except: 482 except: # pylint: disable=bare-except
404 pass 483 pass
405 handlers.append(urllib.request.HTTPBasicAuthHandler(mgr)) 484 handlers.append(urllib.request.HTTPBasicAuthHandler(mgr))
406 handlers.append(urllib.request.HTTPDigestAuthHandler(mgr)) 485 handlers.append(urllib.request.HTTPDigestAuthHandler(mgr))
@@ -413,6 +492,7 @@ def _InitHttp():
413 handlers.append(urllib.request.HTTPSHandler(debuglevel=1)) 492 handlers.append(urllib.request.HTTPSHandler(debuglevel=1))
414 urllib.request.install_opener(urllib.request.build_opener(*handlers)) 493 urllib.request.install_opener(urllib.request.build_opener(*handlers))
415 494
495
416def _Fetch(url, local, src, quiet): 496def _Fetch(url, local, src, quiet):
417 if not quiet: 497 if not quiet:
418 _print('Get %s' % url, file=sys.stderr) 498 _print('Get %s' % url, file=sys.stderr)
@@ -427,22 +507,23 @@ def _Fetch(url, local, src, quiet):
427 cmd.append('+refs/heads/*:refs/remotes/origin/*') 507 cmd.append('+refs/heads/*:refs/remotes/origin/*')
428 cmd.append('refs/tags/*:refs/tags/*') 508 cmd.append('refs/tags/*:refs/tags/*')
429 509
430 proc = subprocess.Popen(cmd, cwd = local, stderr = err) 510 proc = subprocess.Popen(cmd, cwd=local, stderr=err)
431 if err: 511 if err:
432 proc.stderr.read() 512 proc.stderr.read()
433 proc.stderr.close() 513 proc.stderr.close()
434 if proc.wait() != 0: 514 if proc.wait() != 0:
435 raise CloneFailure() 515 raise CloneFailure()
436 516
517
437def _DownloadBundle(url, local, quiet): 518def _DownloadBundle(url, local, quiet):
438 if not url.endswith('/'): 519 if not url.endswith('/'):
439 url += '/' 520 url += '/'
440 url += 'clone.bundle' 521 url += 'clone.bundle'
441 522
442 proc = subprocess.Popen( 523 proc = subprocess.Popen(
443 [GIT, 'config', '--get-regexp', 'url.*.insteadof'], 524 [GIT, 'config', '--get-regexp', 'url.*.insteadof'],
444 cwd = local, 525 cwd=local,
445 stdout = subprocess.PIPE) 526 stdout=subprocess.PIPE)
446 for line in proc.stdout: 527 for line in proc.stdout:
447 m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line) 528 m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line)
448 if m: 529 if m:
@@ -484,6 +565,7 @@ def _DownloadBundle(url, local, quiet):
484 finally: 565 finally:
485 dest.close() 566 dest.close()
486 567
568
487def _ImportBundle(local): 569def _ImportBundle(local):
488 path = os.path.join(local, '.git', 'clone.bundle') 570 path = os.path.join(local, '.git', 'clone.bundle')
489 try: 571 try:
@@ -491,6 +573,7 @@ def _ImportBundle(local):
491 finally: 573 finally:
492 os.remove(path) 574 os.remove(path)
493 575
576
494def _Clone(url, local, quiet): 577def _Clone(url, local, quiet):
495 """Clones a git repository to a new subdirectory of repodir 578 """Clones a git repository to a new subdirectory of repodir
496 """ 579 """
@@ -503,14 +586,14 @@ def _Clone(url, local, quiet):
503 586
504 cmd = [GIT, 'init', '--quiet'] 587 cmd = [GIT, 'init', '--quiet']
505 try: 588 try:
506 proc = subprocess.Popen(cmd, cwd = local) 589 proc = subprocess.Popen(cmd, cwd=local)
507 except OSError as e: 590 except OSError as e:
508 _print(file=sys.stderr) 591 _print(file=sys.stderr)
509 _print("fatal: '%s' is not available" % GIT, file=sys.stderr) 592 _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
510 _print('fatal: %s' % e, file=sys.stderr) 593 _print('fatal: %s' % e, file=sys.stderr)
511 _print(file=sys.stderr) 594 _print(file=sys.stderr)
512 _print('Please make sure %s is installed and in your path.' % GIT, 595 _print('Please make sure %s is installed and in your path.' % GIT,
513 file=sys.stderr) 596 file=sys.stderr)
514 raise CloneFailure() 597 raise CloneFailure()
515 if proc.wait() != 0: 598 if proc.wait() != 0:
516 _print('fatal: could not create %s' % local, file=sys.stderr) 599 _print('fatal: could not create %s' % local, file=sys.stderr)
@@ -518,12 +601,12 @@ def _Clone(url, local, quiet):
518 601
519 _InitHttp() 602 _InitHttp()
520 _SetConfig(local, 'remote.origin.url', url) 603 _SetConfig(local, 'remote.origin.url', url)
521 _SetConfig(local, 'remote.origin.fetch', 604 _SetConfig(local,
522 '+refs/heads/*:refs/remotes/origin/*') 605 'remote.origin.fetch',
606 '+refs/heads/*:refs/remotes/origin/*')
523 if _DownloadBundle(url, local, quiet): 607 if _DownloadBundle(url, local, quiet):
524 _ImportBundle(local) 608 _ImportBundle(local)
525 else: 609 _Fetch(url, local, 'origin', quiet)
526 _Fetch(url, local, 'origin', quiet)
527 610
528 611
529def _Verify(cwd, branch, quiet): 612def _Verify(cwd, branch, quiet):
@@ -533,7 +616,7 @@ def _Verify(cwd, branch, quiet):
533 proc = subprocess.Popen(cmd, 616 proc = subprocess.Popen(cmd,
534 stdout=subprocess.PIPE, 617 stdout=subprocess.PIPE,
535 stderr=subprocess.PIPE, 618 stderr=subprocess.PIPE,
536 cwd = cwd) 619 cwd=cwd)
537 cur = proc.stdout.read().strip() 620 cur = proc.stdout.read().strip()
538 proc.stdout.close() 621 proc.stdout.close()
539 622
@@ -551,7 +634,7 @@ def _Verify(cwd, branch, quiet):
551 if not quiet: 634 if not quiet:
552 _print(file=sys.stderr) 635 _print(file=sys.stderr)
553 _print("info: Ignoring branch '%s'; using tagged release '%s'" 636 _print("info: Ignoring branch '%s'; using tagged release '%s'"
554 % (branch, cur), file=sys.stderr) 637 % (branch, cur), file=sys.stderr)
555 _print(file=sys.stderr) 638 _print(file=sys.stderr)
556 639
557 env = os.environ.copy() 640 env = os.environ.copy()
@@ -559,10 +642,10 @@ def _Verify(cwd, branch, quiet):
559 642
560 cmd = [GIT, 'tag', '-v', cur] 643 cmd = [GIT, 'tag', '-v', cur]
561 proc = subprocess.Popen(cmd, 644 proc = subprocess.Popen(cmd,
562 stdout = subprocess.PIPE, 645 stdout=subprocess.PIPE,
563 stderr = subprocess.PIPE, 646 stderr=subprocess.PIPE,
564 cwd = cwd, 647 cwd=cwd,
565 env = env) 648 env=env)
566 out = proc.stdout.read() 649 out = proc.stdout.read()
567 proc.stdout.close() 650 proc.stdout.close()
568 651
@@ -582,21 +665,21 @@ def _Checkout(cwd, branch, rev, quiet):
582 """Checkout an upstream branch into the repository and track it. 665 """Checkout an upstream branch into the repository and track it.
583 """ 666 """
584 cmd = [GIT, 'update-ref', 'refs/heads/default', rev] 667 cmd = [GIT, 'update-ref', 'refs/heads/default', rev]
585 if subprocess.Popen(cmd, cwd = cwd).wait() != 0: 668 if subprocess.Popen(cmd, cwd=cwd).wait() != 0:
586 raise CloneFailure() 669 raise CloneFailure()
587 670
588 _SetConfig(cwd, 'branch.default.remote', 'origin') 671 _SetConfig(cwd, 'branch.default.remote', 'origin')
589 _SetConfig(cwd, 'branch.default.merge', 'refs/heads/%s' % branch) 672 _SetConfig(cwd, 'branch.default.merge', 'refs/heads/%s' % branch)
590 673
591 cmd = [GIT, 'symbolic-ref', 'HEAD', 'refs/heads/default'] 674 cmd = [GIT, 'symbolic-ref', 'HEAD', 'refs/heads/default']
592 if subprocess.Popen(cmd, cwd = cwd).wait() != 0: 675 if subprocess.Popen(cmd, cwd=cwd).wait() != 0:
593 raise CloneFailure() 676 raise CloneFailure()
594 677
595 cmd = [GIT, 'read-tree', '--reset', '-u'] 678 cmd = [GIT, 'read-tree', '--reset', '-u']
596 if not quiet: 679 if not quiet:
597 cmd.append('-v') 680 cmd.append('-v')
598 cmd.append('HEAD') 681 cmd.append('HEAD')
599 if subprocess.Popen(cmd, cwd = cwd).wait() != 0: 682 if subprocess.Popen(cmd, cwd=cwd).wait() != 0:
600 raise CloneFailure() 683 raise CloneFailure()
601 684
602 685
@@ -608,8 +691,8 @@ def _FindRepo():
608 691
609 olddir = None 692 olddir = None
610 while curdir != '/' \ 693 while curdir != '/' \
611 and curdir != olddir \ 694 and curdir != olddir \
612 and not repo: 695 and not repo:
613 repo = os.path.join(curdir, repodir, REPO_MAIN) 696 repo = os.path.join(curdir, repodir, REPO_MAIN)
614 if not os.path.isfile(repo): 697 if not os.path.isfile(repo):
615 repo = None 698 repo = None
@@ -618,7 +701,7 @@ def _FindRepo():
618 return (repo, os.path.join(curdir, repodir)) 701 return (repo, os.path.join(curdir, repodir))
619 702
620 703
621class _Options: 704class _Options(object):
622 help = False 705 help = False
623 706
624 707
@@ -640,15 +723,20 @@ def _ParseArguments(args):
640 723
641 724
642def _Usage(): 725def _Usage():
726 gitc_usage = ""
727 if get_gitc_manifest_dir():
728 gitc_usage = " gitc-init Initialize a GITC Client.\n"
729
643 _print( 730 _print(
644"""usage: repo COMMAND [ARGS] 731 """usage: repo COMMAND [ARGS]
645 732
646repo is not yet installed. Use "repo init" to install it here. 733repo is not yet installed. Use "repo init" to install it here.
647 734
648The most commonly used repo commands are: 735The most commonly used repo commands are:
649 736
650 init Install repo in the current working directory 737 init Install repo in the current working directory
651 help Display detailed help on a command 738""" + gitc_usage +
739 """ help Display detailed help on a command
652 740
653For access to the full online help, install repo ("repo init"). 741For access to the full online help, install repo ("repo init").
654""", file=sys.stderr) 742""", file=sys.stderr)
@@ -660,6 +748,10 @@ def _Help(args):
660 if args[0] == 'init': 748 if args[0] == 'init':
661 init_optparse.print_help() 749 init_optparse.print_help()
662 sys.exit(0) 750 sys.exit(0)
751 elif args[0] == 'gitc-init':
752 _GitcInitOptions(init_optparse)
753 init_optparse.print_help()
754 sys.exit(0)
663 else: 755 else:
664 _print("error: '%s' is not a bootstrap command.\n" 756 _print("error: '%s' is not a bootstrap command.\n"
665 ' For access to online help, install repo ("repo init").' 757 ' For access to online help, install repo ("repo init").'
@@ -705,8 +797,8 @@ def _SetDefaultsTo(gitdir):
705 '--git-dir=%s' % gitdir, 797 '--git-dir=%s' % gitdir,
706 'symbolic-ref', 798 'symbolic-ref',
707 'HEAD'], 799 'HEAD'],
708 stdout = subprocess.PIPE, 800 stdout=subprocess.PIPE,
709 stderr = subprocess.PIPE) 801 stderr=subprocess.PIPE)
710 REPO_REV = proc.stdout.read().strip() 802 REPO_REV = proc.stdout.read().strip()
711 proc.stdout.close() 803 proc.stdout.close()
712 804
@@ -719,12 +811,23 @@ def _SetDefaultsTo(gitdir):
719 811
720 812
721def main(orig_args): 813def main(orig_args):
722 repo_main, rel_repo_dir = _FindRepo()
723 cmd, opt, args = _ParseArguments(orig_args) 814 cmd, opt, args = _ParseArguments(orig_args)
724 815
816 repo_main, rel_repo_dir = None, None
817 # Don't use the local repo copy, make sure to switch to the gitc client first.
818 if cmd != 'gitc-init':
819 repo_main, rel_repo_dir = _FindRepo()
820
725 wrapper_path = os.path.abspath(__file__) 821 wrapper_path = os.path.abspath(__file__)
726 my_main, my_git = _RunSelf(wrapper_path) 822 my_main, my_git = _RunSelf(wrapper_path)
727 823
824 cwd = os.getcwd()
825 if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
826 _print('error: repo cannot be used in the GITC local manifest directory.'
827 '\nIf you want to work on this GITC client please rerun this '
828 'command from the corresponding client under /gitc/',
829 file=sys.stderr)
830 sys.exit(1)
728 if not repo_main: 831 if not repo_main:
729 if opt.help: 832 if opt.help:
730 _Usage() 833 _Usage()
@@ -732,11 +835,11 @@ def main(orig_args):
732 _Help(args) 835 _Help(args)
733 if not cmd: 836 if not cmd:
734 _NotInstalled() 837 _NotInstalled()
735 if cmd == 'init': 838 if cmd == 'init' or cmd == 'gitc-init':
736 if my_git: 839 if my_git:
737 _SetDefaultsTo(my_git) 840 _SetDefaultsTo(my_git)
738 try: 841 try:
739 _Init(args) 842 _Init(args, gitc_init=(cmd == 'gitc-init'))
740 except CloneFailure: 843 except CloneFailure:
741 shutil.rmtree(os.path.join(repodir, S_repo), ignore_errors=True) 844 shutil.rmtree(os.path.join(repodir, S_repo), ignore_errors=True)
742 sys.exit(1) 845 sys.exit(1)
diff --git a/subcmds/forall.py b/subcmds/forall.py
index 96dc99d1..07ee8d58 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -120,6 +120,9 @@ without iterating through the remaining projects.
120 p.add_option('-r', '--regex', 120 p.add_option('-r', '--regex',
121 dest='regex', action='store_true', 121 dest='regex', action='store_true',
122 help="Execute the command only on projects matching regex or wildcard expression") 122 help="Execute the command only on projects matching regex or wildcard expression")
123 p.add_option('-i', '--inverse-regex',
124 dest='inverse_regex', action='store_true',
125 help="Execute the command only on projects not matching regex or wildcard expression")
123 p.add_option('-g', '--groups', 126 p.add_option('-g', '--groups',
124 dest='groups', 127 dest='groups',
125 help="Execute the command only on projects matching the specified groups") 128 help="Execute the command only on projects matching the specified groups")
@@ -215,10 +218,12 @@ without iterating through the remaining projects.
215 if os.path.isfile(smart_sync_manifest_path): 218 if os.path.isfile(smart_sync_manifest_path):
216 self.manifest.Override(smart_sync_manifest_path) 219 self.manifest.Override(smart_sync_manifest_path)
217 220
218 if not opt.regex: 221 if opt.regex:
219 projects = self.GetProjects(args, groups=opt.groups)
220 else:
221 projects = self.FindProjects(args) 222 projects = self.FindProjects(args)
223 elif opt.inverse_regex:
224 projects = self.FindProjects(args, inverse=True)
225 else:
226 projects = self.GetProjects(args, groups=opt.groups)
222 227
223 os.environ['REPO_COUNT'] = str(len(projects)) 228 os.environ['REPO_COUNT'] = str(len(projects))
224 229
@@ -240,7 +245,8 @@ without iterating through the remaining projects.
240 rc = rc or errno.EINTR 245 rc = rc or errno.EINTR
241 except Exception as e: 246 except Exception as e:
242 # Catch any other exceptions raised 247 # Catch any other exceptions raised
243 print('Got an error, terminating the pool: %r' % e, 248 print('Got an error, terminating the pool: %s: %s' %
249 (type(e).__name__, e),
244 file=sys.stderr) 250 file=sys.stderr)
245 pool.terminate() 251 pool.terminate()
246 rc = rc or getattr(e, 'errno', 1) 252 rc = rc or getattr(e, 'errno', 1)
@@ -254,7 +260,8 @@ without iterating through the remaining projects.
254 try: 260 try:
255 project = self._SerializeProject(p) 261 project = self._SerializeProject(p)
256 except Exception as e: 262 except Exception as e:
257 print('Project list error: %r' % e, 263 print('Project list error on project %s: %s: %s' %
264 (p.name, type(e).__name__, e),
258 file=sys.stderr) 265 file=sys.stderr)
259 return 266 return
260 except KeyboardInterrupt: 267 except KeyboardInterrupt:
diff --git a/subcmds/gitc_delete.py b/subcmds/gitc_delete.py
new file mode 100644
index 00000000..7380c352
--- /dev/null
+++ b/subcmds/gitc_delete.py
@@ -0,0 +1,55 @@
1#
2# Copyright (C) 2015 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16from __future__ import print_function
17import os
18import shutil
19import sys
20
21from command import Command, GitcClientCommand
22import gitc_utils
23
24from pyversion import is_python3
25if not is_python3():
26 # pylint:disable=W0622
27 input = raw_input
28 # pylint:enable=W0622
29
30class GitcDelete(Command, GitcClientCommand):
31 common = True
32 visible_everywhere = False
33 helpSummary = "Delete a GITC Client."
34 helpUsage = """
35%prog
36"""
37 helpDescription = """
38This subcommand deletes the current GITC client, deleting the GITC manifest
39and all locally downloaded sources.
40"""
41
42 def _Options(self, p):
43 p.add_option('-f', '--force',
44 dest='force', action='store_true',
45 help='Force the deletion (no prompt).')
46
47 def Execute(self, opt, args):
48 if not opt.force:
49 prompt = ('This will delete GITC client: %s\nAre you sure? (yes/no) ' %
50 self.gitc_manifest.gitc_client_name)
51 response = input(prompt).lower()
52 if not response == 'yes':
53 print('Response was not "yes"\n Exiting...')
54 sys.exit(1)
55 shutil.rmtree(self.gitc_manifest.gitc_client_dir)
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py
new file mode 100644
index 00000000..2726eaec
--- /dev/null
+++ b/subcmds/gitc_init.py
@@ -0,0 +1,82 @@
1#
2# Copyright (C) 2015 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16from __future__ import print_function
17import os
18import sys
19
20import gitc_utils
21from command import GitcAvailableCommand
22from manifest_xml import GitcManifest
23from subcmds import init
24import wrapper
25
26
27class GitcInit(init.Init, GitcAvailableCommand):
28 common = True
29 helpSummary = "Initialize a GITC Client."
30 helpUsage = """
31%prog [options] [client name]
32"""
33 helpDescription = """
34The '%prog' command is ran to initialize a new GITC client for use
35with the GITC file system.
36
37This command will setup the client directory, initialize repo, just
38like repo init does, and then downloads the manifest collection
39and installs it in the .repo/directory of the GITC client.
40
41Once this is done, a GITC manifest is generated by pulling the HEAD
42SHA for each project and generates the properly formatted XML file
43and installs it as .manifest in the GITC client directory.
44
45The -c argument is required to specify the GITC client name.
46
47The optional -f argument can be used to specify the manifest file to
48use for this GITC client.
49"""
50
51 def _Options(self, p):
52 super(GitcInit, self)._Options(p)
53 g = p.add_option_group('GITC options')
54 g.add_option('-f', '--manifest-file',
55 dest='manifest_file',
56 help='Optional manifest file to use for this GITC client.')
57 g.add_option('-c', '--gitc-client',
58 dest='gitc_client',
59 help='The name of the gitc_client instance to create or modify.')
60
61 def Execute(self, opt, args):
62 gitc_client = gitc_utils.parse_clientdir(os.getcwd())
63 if not gitc_client or (opt.gitc_client and gitc_client != opt.gitc_client):
64 print('fatal: Please update your repo command. See go/gitc for instructions.', file=sys.stderr)
65 sys.exit(1)
66 self.client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
67 gitc_client)
68 super(GitcInit, self).Execute(opt, args)
69
70 manifest_file = self.manifest.manifestFile
71 if opt.manifest_file:
72 if not os.path.exists(opt.manifest_file):
73 print('fatal: Specified manifest file %s does not exist.' %
74 opt.manifest_file)
75 sys.exit(1)
76 manifest_file = opt.manifest_file
77
78 manifest = GitcManifest(self.repodir, gitc_client)
79 manifest.Override(manifest_file)
80 gitc_utils.generate_gitc_manifest(None, manifest)
81 print('Please run `cd %s` to view your GITC client.' %
82 os.path.join(wrapper.Wrapper().GITC_FS_ROOT_DIR, gitc_client))
diff --git a/subcmds/help.py b/subcmds/help.py
index 4aa3f863..9bb4c8c7 100644
--- a/subcmds/help.py
+++ b/subcmds/help.py
@@ -19,7 +19,8 @@ import sys
19from formatter import AbstractFormatter, DumbWriter 19from formatter import AbstractFormatter, DumbWriter
20 20
21from color import Coloring 21from color import Coloring
22from command import PagedCommand, MirrorSafeCommand 22from command import PagedCommand, MirrorSafeCommand, GitcAvailableCommand, GitcClientCommand
23import gitc_utils
23 24
24class Help(PagedCommand, MirrorSafeCommand): 25class Help(PagedCommand, MirrorSafeCommand):
25 common = False 26 common = False
@@ -54,9 +55,21 @@ Displays detailed usage information about a command.
54 def _PrintCommonCommands(self): 55 def _PrintCommonCommands(self):
55 print('usage: repo COMMAND [ARGS]') 56 print('usage: repo COMMAND [ARGS]')
56 print('The most commonly used repo commands are:') 57 print('The most commonly used repo commands are:')
58
59 def gitc_supported(cmd):
60 if not isinstance(cmd, GitcAvailableCommand) and not isinstance(cmd, GitcClientCommand):
61 return True
62 if self.manifest.isGitcClient:
63 return True
64 if isinstance(cmd, GitcClientCommand):
65 return False
66 if gitc_utils.get_gitc_manifest_dir():
67 return True
68 return False
69
57 commandNames = list(sorted([name 70 commandNames = list(sorted([name
58 for name, command in self.commands.items() 71 for name, command in self.commands.items()
59 if command.common])) 72 if command.common and gitc_supported(command)]))
60 73
61 maxlen = 0 74 maxlen = 0
62 for name in commandNames: 75 for name in commandNames:
diff --git a/subcmds/init.py b/subcmds/init.py
index dbb6ddda..b8e3de5a 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -179,7 +179,7 @@ to update the working directory files.
179 r.Save() 179 r.Save()
180 180
181 groups = re.split(r'[,\s]+', opt.groups) 181 groups = re.split(r'[,\s]+', opt.groups)
182 all_platforms = ['linux', 'darwin'] 182 all_platforms = ['linux', 'darwin', 'windows']
183 platformize = lambda x: 'platform-' + x 183 platformize = lambda x: 'platform-' + x
184 if opt.platform == 'auto': 184 if opt.platform == 'auto':
185 if (not opt.mirror and 185 if (not opt.mirror and
@@ -188,7 +188,7 @@ to update the working directory files.
188 elif opt.platform == 'all': 188 elif opt.platform == 'all':
189 groups.extend(map(platformize, all_platforms)) 189 groups.extend(map(platformize, all_platforms))
190 elif opt.platform in all_platforms: 190 elif opt.platform in all_platforms:
191 groups.extend(platformize(opt.platform)) 191 groups.append(platformize(opt.platform))
192 elif opt.platform != 'none': 192 elif opt.platform != 'none':
193 print('fatal: invalid platform flag', file=sys.stderr) 193 print('fatal: invalid platform flag', file=sys.stderr)
194 sys.exit(1) 194 sys.exit(1)
diff --git a/subcmds/rebase.py b/subcmds/rebase.py
index 1bdc1f0b..74796970 100644
--- a/subcmds/rebase.py
+++ b/subcmds/rebase.py
@@ -54,6 +54,11 @@ branch but need to incorporate new upstream changes "underneath" them.
54 p.add_option('--auto-stash', 54 p.add_option('--auto-stash',
55 dest='auto_stash', action='store_true', 55 dest='auto_stash', action='store_true',
56 help='Stash local modifications before starting') 56 help='Stash local modifications before starting')
57 p.add_option('-m', '--onto-manifest',
58 dest='onto_manifest', action='store_true',
59 help='Rebase onto the manifest version instead of upstream '
60 'HEAD. This helps to make sure the local tree stays '
61 'consistent if you previously synced to a manifest.')
57 62
58 def Execute(self, opt, args): 63 def Execute(self, opt, args):
59 all_projects = self.GetProjects(args) 64 all_projects = self.GetProjects(args)
@@ -106,6 +111,10 @@ branch but need to incorporate new upstream changes "underneath" them.
106 if opt.interactive: 111 if opt.interactive:
107 args.append("-i") 112 args.append("-i")
108 113
114 if opt.onto_manifest:
115 args.append('--onto')
116 args.append(project.revisionExpr)
117
109 args.append(upbranch.LocalMerge) 118 args.append(upbranch.LocalMerge)
110 119
111 print('# %s: rebasing %s -> %s' 120 print('# %s: rebasing %s -> %s'
diff --git a/subcmds/start.py b/subcmds/start.py
index 60ad41e0..d1430a9d 100644
--- a/subcmds/start.py
+++ b/subcmds/start.py
@@ -14,11 +14,15 @@
14# limitations under the License. 14# limitations under the License.
15 15
16from __future__ import print_function 16from __future__ import print_function
17import os
17import sys 18import sys
19
18from command import Command 20from command import Command
19from git_config import IsId 21from git_config import IsId
20from git_command import git 22from git_command import git
23import gitc_utils
21from progress import Progress 24from progress import Progress
25from project import SyncBuffer
22 26
23class Start(Command): 27class Start(Command):
24 common = True 28 common = True
@@ -53,20 +57,57 @@ revision specified in the manifest.
53 print("error: at least one project must be specified", file=sys.stderr) 57 print("error: at least one project must be specified", file=sys.stderr)
54 sys.exit(1) 58 sys.exit(1)
55 59
56 all_projects = self.GetProjects(projects) 60 all_projects = self.GetProjects(projects,
61 missing_ok=bool(self.gitc_manifest))
62
63 # This must happen after we find all_projects, since GetProjects may need
64 # the local directory, which will disappear once we save the GITC manifest.
65 if self.gitc_manifest:
66 gitc_projects = self.GetProjects(projects, manifest=self.gitc_manifest,
67 missing_ok=True)
68 for project in gitc_projects:
69 if project.old_revision:
70 project.already_synced = True
71 else:
72 project.already_synced = False
73 project.old_revision = project.revisionExpr
74 project.revisionExpr = None
75 # Save the GITC manifest.
76 gitc_utils.save_manifest(self.gitc_manifest)
77
78 # Make sure we have a valid CWD
79 if not os.path.exists(os.getcwd()):
80 os.chdir(self.manifest.topdir)
57 81
58 pm = Progress('Starting %s' % nb, len(all_projects)) 82 pm = Progress('Starting %s' % nb, len(all_projects))
59 for project in all_projects: 83 for project in all_projects:
60 pm.update() 84 pm.update()
85
86 if self.gitc_manifest:
87 gitc_project = self.gitc_manifest.paths[project.relpath]
88 # Sync projects that have not been opened.
89 if not gitc_project.already_synced:
90 proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir,
91 project.relpath)
92 project.worktree = proj_localdir
93 if not os.path.exists(proj_localdir):
94 os.makedirs(proj_localdir)
95 project.Sync_NetworkHalf()
96 sync_buf = SyncBuffer(self.manifest.manifestProject.config)
97 project.Sync_LocalHalf(sync_buf)
98 project.revisionId = gitc_project.old_revision
99
61 # If the current revision is a specific SHA1 then we can't push back 100 # If the current revision is a specific SHA1 then we can't push back
62 # to it; so substitute with dest_branch if defined, or with manifest 101 # to it; so substitute with dest_branch if defined, or with manifest
63 # default revision instead. 102 # default revision instead.
103 branch_merge = ''
64 if IsId(project.revisionExpr): 104 if IsId(project.revisionExpr):
65 if project.dest_branch: 105 if project.dest_branch:
66 project.revisionExpr = project.dest_branch 106 branch_merge = project.dest_branch
67 else: 107 else:
68 project.revisionExpr = self.manifest.default.revisionExpr 108 branch_merge = self.manifest.default.revisionExpr
69 if not project.StartBranch(nb): 109
110 if not project.StartBranch(nb, branch_merge=branch_merge):
70 err.append(project) 111 err.append(project)
71 pm.end() 112 pm.end()
72 113
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 43d450be..4af411c9 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -23,18 +23,26 @@ import shutil
23import socket 23import socket
24import subprocess 24import subprocess
25import sys 25import sys
26import tempfile
26import time 27import time
27 28
28from pyversion import is_python3 29from pyversion import is_python3
29if is_python3(): 30if is_python3():
31 import http.cookiejar as cookielib
32 import urllib.error
30 import urllib.parse 33 import urllib.parse
34 import urllib.request
31 import xmlrpc.client 35 import xmlrpc.client
32else: 36else:
37 import cookielib
33 import imp 38 import imp
39 import urllib2
34 import urlparse 40 import urlparse
35 import xmlrpclib 41 import xmlrpclib
36 urllib = imp.new_module('urllib') 42 urllib = imp.new_module('urllib')
43 urllib.error = urllib2
37 urllib.parse = urlparse 44 urllib.parse = urlparse
45 urllib.request = urllib2
38 xmlrpc = imp.new_module('xmlrpc') 46 xmlrpc = imp.new_module('xmlrpc')
39 xmlrpc.client = xmlrpclib 47 xmlrpc.client = xmlrpclib
40 48
@@ -57,7 +65,9 @@ except ImportError:
57 multiprocessing = None 65 multiprocessing = None
58 66
59from git_command import GIT, git_require 67from git_command import GIT, git_require
68from git_config import GetUrlCookieFile
60from git_refs import R_HEADS, HEAD 69from git_refs import R_HEADS, HEAD
70import gitc_utils
61from project import Project 71from project import Project
62from project import RemoteSpec 72from project import RemoteSpec
63from command import Command, MirrorSafeCommand 73from command import Command, MirrorSafeCommand
@@ -65,6 +75,7 @@ from error import RepoChangedException, GitError, ManifestParseError
65from project import SyncBuffer 75from project import SyncBuffer
66from progress import Progress 76from progress import Progress
67from wrapper import Wrapper 77from wrapper import Wrapper
78from manifest_xml import GitcManifest
68 79
69_ONE_DAY_S = 24 * 60 * 60 80_ONE_DAY_S = 24 * 60 * 60
70 81
@@ -140,6 +151,9 @@ The --optimized-fetch option can be used to only fetch projects that
140are fixed to a sha1 revision if the sha1 revision does not already 151are fixed to a sha1 revision if the sha1 revision does not already
141exist locally. 152exist locally.
142 153
154The --prune option can be used to remove any refs that no longer
155exist on the remote.
156
143SSH Connections 157SSH Connections
144--------------- 158---------------
145 159
@@ -223,6 +237,8 @@ later is required to fix a server side protocol bug.
223 p.add_option('--optimized-fetch', 237 p.add_option('--optimized-fetch',
224 dest='optimized_fetch', action='store_true', 238 dest='optimized_fetch', action='store_true',
225 help='only fetch projects fixed to sha1 if revision does not exist locally') 239 help='only fetch projects fixed to sha1 if revision does not exist locally')
240 p.add_option('--prune', dest='prune', action='store_true',
241 help='delete refs that no longer exist on the remote')
226 if show_smart: 242 if show_smart:
227 p.add_option('-s', '--smart-sync', 243 p.add_option('-s', '--smart-sync',
228 dest='smart_sync', action='store_true', 244 dest='smart_sync', action='store_true',
@@ -294,7 +310,8 @@ later is required to fix a server side protocol bug.
294 force_sync=opt.force_sync, 310 force_sync=opt.force_sync,
295 clone_bundle=not opt.no_clone_bundle, 311 clone_bundle=not opt.no_clone_bundle,
296 no_tags=opt.no_tags, archive=self.manifest.IsArchive, 312 no_tags=opt.no_tags, archive=self.manifest.IsArchive,
297 optimized_fetch=opt.optimized_fetch) 313 optimized_fetch=opt.optimized_fetch,
314 prune=opt.prune)
298 self._fetch_times.Set(project, time.time() - start) 315 self._fetch_times.Set(project, time.time() - start)
299 316
300 # Lock around all the rest of the code, since printing, updating a set 317 # Lock around all the rest of the code, since printing, updating a set
@@ -303,6 +320,7 @@ later is required to fix a server side protocol bug.
303 did_lock = True 320 did_lock = True
304 321
305 if not success: 322 if not success:
323 err_event.set()
306 print('error: Cannot fetch %s' % project.name, file=sys.stderr) 324 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
307 if opt.force_broken: 325 if opt.force_broken:
308 print('warn: --force-broken, continuing to sync', 326 print('warn: --force-broken, continuing to sync',
@@ -313,7 +331,7 @@ later is required to fix a server side protocol bug.
313 fetched.add(project.gitdir) 331 fetched.add(project.gitdir)
314 pm.update() 332 pm.update()
315 except _FetchError: 333 except _FetchError:
316 err_event.set() 334 pass
317 except Exception as e: 335 except Exception as e:
318 print('error: Cannot fetch %s (%s: %s)' \ 336 print('error: Cannot fetch %s (%s: %s)' \
319 % (project.name, type(e).__name__, str(e)), file=sys.stderr) 337 % (project.name, type(e).__name__, str(e)), file=sys.stderr)
@@ -554,19 +572,18 @@ later is required to fix a server side protocol bug.
554 try: 572 try:
555 info = netrc.netrc() 573 info = netrc.netrc()
556 except IOError: 574 except IOError:
557 print('.netrc file does not exist or could not be opened', 575 # .netrc file does not exist or could not be opened
558 file=sys.stderr) 576 pass
559 else: 577 else:
560 try: 578 try:
561 parse_result = urllib.parse.urlparse(manifest_server) 579 parse_result = urllib.parse.urlparse(manifest_server)
562 if parse_result.hostname: 580 if parse_result.hostname:
563 username, _account, password = \ 581 auth = info.authenticators(parse_result.hostname)
564 info.authenticators(parse_result.hostname) 582 if auth:
565 except TypeError: 583 username, _account, password = auth
566 # TypeError is raised when the given hostname is not present 584 else:
567 # in the .netrc file. 585 print('No credentials found for %s in .netrc'
568 print('No credentials found for %s in .netrc' 586 % parse_result.hostname, file=sys.stderr)
569 % parse_result.hostname, file=sys.stderr)
570 except netrc.NetrcParseError as e: 587 except netrc.NetrcParseError as e:
571 print('Error parsing .netrc file: %s' % e, file=sys.stderr) 588 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
572 589
@@ -575,8 +592,12 @@ later is required to fix a server side protocol bug.
575 (username, password), 592 (username, password),
576 1) 593 1)
577 594
595 transport = PersistentTransport(manifest_server)
596 if manifest_server.startswith('persistent-'):
597 manifest_server = manifest_server[len('persistent-'):]
598
578 try: 599 try:
579 server = xmlrpc.client.Server(manifest_server) 600 server = xmlrpc.client.Server(manifest_server, transport=transport)
580 if opt.smart_sync: 601 if opt.smart_sync:
581 p = self.manifest.manifestProject 602 p = self.manifest.manifestProject
582 b = p.GetBranch(p.CurrentBranch) 603 b = p.GetBranch(p.CurrentBranch)
@@ -656,6 +677,42 @@ later is required to fix a server side protocol bug.
656 self._ReloadManifest(manifest_name) 677 self._ReloadManifest(manifest_name)
657 if opt.jobs is None: 678 if opt.jobs is None:
658 self.jobs = self.manifest.default.sync_j 679 self.jobs = self.manifest.default.sync_j
680
681 if self.gitc_manifest:
682 gitc_manifest_projects = self.GetProjects(args,
683 missing_ok=True)
684 gitc_projects = []
685 opened_projects = []
686 for project in gitc_manifest_projects:
687 if project.relpath in self.gitc_manifest.paths and \
688 self.gitc_manifest.paths[project.relpath].old_revision:
689 opened_projects.append(project.relpath)
690 else:
691 gitc_projects.append(project.relpath)
692
693 if not args:
694 gitc_projects = None
695
696 if gitc_projects != [] and not opt.local_only:
697 print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
698 manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
699 if manifest_name:
700 manifest.Override(manifest_name)
701 else:
702 manifest.Override(self.manifest.manifestFile)
703 gitc_utils.generate_gitc_manifest(self.gitc_manifest,
704 manifest,
705 gitc_projects)
706 print('GITC client successfully synced.')
707
708 # The opened projects need to be synced as normal, therefore we
709 # generate a new args list to represent the opened projects.
710 # TODO: make this more reliable -- if there's a project name/path overlap,
711 # this may choose the wrong project.
712 args = [os.path.relpath(self.manifest.paths[p].worktree, os.getcwd())
713 for p in opened_projects]
714 if not args:
715 return
659 all_projects = self.GetProjects(args, 716 all_projects = self.GetProjects(args,
660 missing_ok=True, 717 missing_ok=True,
661 submodules_ok=opt.fetch_submodules) 718 submodules_ok=opt.fetch_submodules)
@@ -850,3 +907,100 @@ class _FetchTimes(object):
850 os.remove(self._path) 907 os.remove(self._path)
851 except OSError: 908 except OSError:
852 pass 909 pass
910
911# This is a replacement for xmlrpc.client.Transport using urllib2
912# and supporting persistent-http[s]. It cannot change hosts from
913# request to request like the normal transport, the real url
914# is passed during initialization.
915class PersistentTransport(xmlrpc.client.Transport):
916 def __init__(self, orig_host):
917 self.orig_host = orig_host
918
919 def request(self, host, handler, request_body, verbose=False):
920 with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
921 # Python doesn't understand cookies with the #HttpOnly_ prefix
922 # Since we're only using them for HTTP, copy the file temporarily,
923 # stripping those prefixes away.
924 if cookiefile:
925 tmpcookiefile = tempfile.NamedTemporaryFile()
926 tmpcookiefile.write("# HTTP Cookie File")
927 try:
928 with open(cookiefile) as f:
929 for line in f:
930 if line.startswith("#HttpOnly_"):
931 line = line[len("#HttpOnly_"):]
932 tmpcookiefile.write(line)
933 tmpcookiefile.flush()
934
935 cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
936 try:
937 cookiejar.load()
938 except cookielib.LoadError:
939 cookiejar = cookielib.CookieJar()
940 finally:
941 tmpcookiefile.close()
942 else:
943 cookiejar = cookielib.CookieJar()
944
945 proxyhandler = urllib.request.ProxyHandler
946 if proxy:
947 proxyhandler = urllib.request.ProxyHandler({
948 "http": proxy,
949 "https": proxy })
950
951 opener = urllib.request.build_opener(
952 urllib.request.HTTPCookieProcessor(cookiejar),
953 proxyhandler)
954
955 url = urllib.parse.urljoin(self.orig_host, handler)
956 parse_results = urllib.parse.urlparse(url)
957
958 scheme = parse_results.scheme
959 if scheme == 'persistent-http':
960 scheme = 'http'
961 if scheme == 'persistent-https':
962 # If we're proxying through persistent-https, use http. The
963 # proxy itself will do the https.
964 if proxy:
965 scheme = 'http'
966 else:
967 scheme = 'https'
968
969 # Parse out any authentication information using the base class
970 host, extra_headers, _ = self.get_host_info(parse_results.netloc)
971
972 url = urllib.parse.urlunparse((
973 scheme,
974 host,
975 parse_results.path,
976 parse_results.params,
977 parse_results.query,
978 parse_results.fragment))
979
980 request = urllib.request.Request(url, request_body)
981 if extra_headers is not None:
982 for (name, header) in extra_headers:
983 request.add_header(name, header)
984 request.add_header('Content-Type', 'text/xml')
985 try:
986 response = opener.open(request)
987 except urllib.error.HTTPError as e:
988 if e.code == 501:
989 # We may have been redirected through a login process
990 # but our POST turned into a GET. Retry.
991 response = opener.open(request)
992 else:
993 raise
994
995 p, u = xmlrpc.client.getparser()
996 while 1:
997 data = response.read(1024)
998 if not data:
999 break
1000 p.feed(data)
1001 p.close()
1002 return u.close()
1003
1004 def close(self):
1005 pass
1006
diff --git a/tests/fixtures/gitc_config b/tests/fixtures/gitc_config
new file mode 100644
index 00000000..a7f3d1c9
--- /dev/null
+++ b/tests/fixtures/gitc_config
@@ -0,0 +1 @@
gitc_dir=/test/usr/local/google/gitc
diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py
new file mode 100644
index 00000000..fb32e38a
--- /dev/null
+++ b/tests/test_wrapper.py
@@ -0,0 +1,75 @@
1#
2# Copyright (C) 2015 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import os
17import unittest
18
19import wrapper
20
21def fixture(*paths):
22 """Return a path relative to tests/fixtures.
23 """
24 return os.path.join(os.path.dirname(__file__), 'fixtures', *paths)
25
26class RepoWrapperUnitTest(unittest.TestCase):
27 """Tests helper functions in the repo wrapper
28 """
29 def setUp(self):
30 """Load the wrapper module every time
31 """
32 wrapper._wrapper_module = None
33 self.wrapper = wrapper.Wrapper()
34
35 def test_get_gitc_manifest_dir_no_gitc(self):
36 """
37 Test reading a missing gitc config file
38 """
39 self.wrapper.GITC_CONFIG_FILE = fixture('missing_gitc_config')
40 val = self.wrapper.get_gitc_manifest_dir()
41 self.assertEqual(val, '')
42
43 def test_get_gitc_manifest_dir(self):
44 """
45 Test reading the gitc config file and parsing the directory
46 """
47 self.wrapper.GITC_CONFIG_FILE = fixture('gitc_config')
48 val = self.wrapper.get_gitc_manifest_dir()
49 self.assertEqual(val, '/test/usr/local/google/gitc')
50
51 def test_gitc_parse_clientdir_no_gitc(self):
52 """
53 Test parsing the gitc clientdir without gitc running
54 """
55 self.wrapper.GITC_CONFIG_FILE = fixture('missing_gitc_config')
56 self.assertEqual(self.wrapper.gitc_parse_clientdir('/something'), None)
57 self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/test'), 'test')
58
59 def test_gitc_parse_clientdir(self):
60 """
61 Test parsing the gitc clientdir
62 """
63 self.wrapper.GITC_CONFIG_FILE = fixture('gitc_config')
64 self.assertEqual(self.wrapper.gitc_parse_clientdir('/something'), None)
65 self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/test'), 'test')
66 self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/test/'), 'test')
67 self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/test/extra'), 'test')
68 self.assertEqual(self.wrapper.gitc_parse_clientdir('/test/usr/local/google/gitc/test'), 'test')
69 self.assertEqual(self.wrapper.gitc_parse_clientdir('/test/usr/local/google/gitc/test/'), 'test')
70 self.assertEqual(self.wrapper.gitc_parse_clientdir('/test/usr/local/google/gitc/test/extra'), 'test')
71 self.assertEqual(self.wrapper.gitc_parse_clientdir('/gitc/manifest-rw/'), None)
72 self.assertEqual(self.wrapper.gitc_parse_clientdir('/test/usr/local/google/gitc/'), None)
73
74if __name__ == '__main__':
75 unittest.main()