summaryrefslogtreecommitdiffstats
path: root/repo
diff options
context:
space:
mode:
Diffstat (limited to 'repo')
-rwxr-xr-xrepo1190
1 files changed, 795 insertions, 395 deletions
diff --git a/repo b/repo
index 15d2ff85..4cddbf1e 100755
--- a/repo
+++ b/repo
@@ -1,5 +1,19 @@
1#!/usr/bin/env python 1#!/usr/bin/env python
2# -*- coding:utf-8 -*- 2# -*- coding:utf-8 -*-
3#
4# Copyright (C) 2008 The Android Open Source Project
5#
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
3 17
4"""Repo launcher. 18"""Repo launcher.
5 19
@@ -10,35 +24,135 @@ copy of repo in the checkout.
10 24
11from __future__ import print_function 25from __future__ import print_function
12 26
27import datetime
28import os
29import platform
30import shlex
31import subprocess
32import sys
33
34
35# These should never be newer than the main.py version since this needs to be a
36# bit more flexible with older systems. See that file for more details on the
37# versions we select.
38MIN_PYTHON_VERSION_SOFT = (3, 6)
39MIN_PYTHON_VERSION_HARD = (3, 5)
40
41
42# Keep basic logic in sync with repo_trace.py.
43class Trace(object):
44 """Trace helper logic."""
45
46 REPO_TRACE = 'REPO_TRACE'
47
48 def __init__(self):
49 self.set(os.environ.get(self.REPO_TRACE) == '1')
50
51 def set(self, value):
52 self.enabled = bool(value)
53
54 def print(self, *args, **kwargs):
55 if self.enabled:
56 print(*args, **kwargs)
57
58
59trace = Trace()
60
61
62def exec_command(cmd):
63 """Execute |cmd| or return None on failure."""
64 trace.print(':', ' '.join(cmd))
65 try:
66 if platform.system() == 'Windows':
67 ret = subprocess.call(cmd)
68 sys.exit(ret)
69 else:
70 os.execvp(cmd[0], cmd)
71 except Exception:
72 pass
73
74
75def check_python_version():
76 """Make sure the active Python version is recent enough."""
77 def reexec(prog):
78 exec_command([prog] + sys.argv)
79
80 ver = sys.version_info
81 major = ver.major
82 minor = ver.minor
83
84 # Abort on very old Python 2 versions.
85 if (major, minor) < (2, 7):
86 print('repo: error: Your Python version is too old. '
87 'Please use Python {}.{} or newer instead.'.format(
88 *MIN_PYTHON_VERSION_SOFT), file=sys.stderr)
89 sys.exit(1)
90
91 # Try to re-exec the version specific Python 3 if needed.
92 if (major, minor) < MIN_PYTHON_VERSION_SOFT:
93 # Python makes releases ~once a year, so try our min version +10 to help
94 # bridge the gap. This is the fallback anyways so perf isn't critical.
95 min_major, min_minor = MIN_PYTHON_VERSION_SOFT
96 for inc in range(0, 10):
97 reexec('python{}.{}'.format(min_major, min_minor + inc))
98
99 # Fallback to older versions if possible.
100 for inc in range(MIN_PYTHON_VERSION_SOFT[1] - MIN_PYTHON_VERSION_HARD[1], 0, -1):
101 # Don't downgrade, and don't reexec ourselves (which would infinite loop).
102 if (min_major, min_minor - inc) <= (major, minor):
103 break
104 reexec('python{}.{}'.format(min_major, min_minor - inc))
105
106 # Try the generic Python 3 wrapper, but only if it's new enough. If it
107 # isn't, we want to just give up below and make the user resolve things.
108 try:
109 proc = subprocess.Popen(
110 ['python3', '-c', 'import sys; '
111 'print(sys.version_info.major, sys.version_info.minor)'],
112 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
113 (output, _) = proc.communicate()
114 python3_ver = tuple(int(x) for x in output.decode('utf-8').split())
115 except (OSError, subprocess.CalledProcessError):
116 python3_ver = None
117
118 # If the python3 version looks like it's new enough, give it a try.
119 if (python3_ver and python3_ver >= MIN_PYTHON_VERSION_HARD
120 and python3_ver != (major, minor)):
121 reexec('python3')
122
123 # We're still here, so diagnose things for the user.
124 if major < 3:
125 print('repo: error: Python 2 is no longer supported; '
126 'Please upgrade to Python {}.{}+.'.format(*MIN_PYTHON_VERSION_HARD),
127 file=sys.stderr)
128 sys.exit(1)
129 elif (major, minor) < MIN_PYTHON_VERSION_HARD:
130 print('repo: error: Python 3 version is too old; '
131 'Please use Python {}.{} or newer.'.format(*MIN_PYTHON_VERSION_HARD),
132 file=sys.stderr)
133 sys.exit(1)
134
135
136if __name__ == '__main__':
137 check_python_version()
138
139
13# repo default configuration 140# repo default configuration
14# 141#
15import os
16REPO_URL = os.environ.get('REPO_URL', None) 142REPO_URL = os.environ.get('REPO_URL', None)
17if not REPO_URL: 143if not REPO_URL:
18 REPO_URL = 'https://gerrit.googlesource.com/git-repo' 144 REPO_URL = 'https://gerrit.googlesource.com/git-repo'
19REPO_REV = os.environ.get('REPO_REV') 145REPO_REV = os.environ.get('REPO_REV')
20if not REPO_REV: 146if not REPO_REV:
21 REPO_REV = 'repo-1' 147 REPO_REV = 'stable'
22 148# URL to file bug reports for repo tool issues.
23# Copyright (C) 2008 Google Inc. 149BUG_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue'
24#
25# Licensed under the Apache License, Version 2.0 (the "License");
26# you may not use this file except in compliance with the License.
27# You may obtain a copy of the License at
28#
29# http://www.apache.org/licenses/LICENSE-2.0
30#
31# Unless required by applicable law or agreed to in writing, software
32# distributed under the License is distributed on an "AS IS" BASIS,
33# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
34# See the License for the specific language governing permissions and
35# limitations under the License.
36 150
37# increment this whenever we make important changes to this script 151# increment this whenever we make important changes to this script
38VERSION = (1, 27) 152VERSION = (2, 17)
39 153
40# increment this if the MAINTAINER_KEYS block is modified 154# increment this if the MAINTAINER_KEYS block is modified
41KEYRING_VERSION = (1, 2) 155KEYRING_VERSION = (2, 3)
42 156
43# Each individual key entry is created by using: 157# Each individual key entry is created by using:
44# gpg --armor --export keyid 158# gpg --armor --export keyid
@@ -46,7 +160,6 @@ MAINTAINER_KEYS = """
46 160
47 Repo Maintainer <repo@android.kernel.org> 161 Repo Maintainer <repo@android.kernel.org>
48-----BEGIN PGP PUBLIC KEY BLOCK----- 162-----BEGIN PGP PUBLIC KEY BLOCK-----
49Version: GnuPG v1.4.2.2 (GNU/Linux)
50 163
51mQGiBEj3ugERBACrLJh/ZPyVSKeClMuznFIrsQ+hpNnmJGw1a9GXKYKk8qHPhAZf 164mQGiBEj3ugERBACrLJh/ZPyVSKeClMuznFIrsQ+hpNnmJGw1a9GXKYKk8qHPhAZf
52WKtrBqAVMNRLhL85oSlekRz98u41H5si5zcuv+IXJDF5MJYcB8f22wAy15lUqPWi 165WKtrBqAVMNRLhL85oSlekRz98u41H5si5zcuv+IXJDF5MJYcB8f22wAy15lUqPWi
@@ -82,63 +195,64 @@ p3v5ILwfC7hVx4jHSnOgZ65L9s8EQdVr1ckN9243yta7rNgwfcqb60ILMFF1BRk/
825xGrFy8tfAaeBMIQ17gvFSp/suc9DYO0ICK2BISzq+F+ZiAKsjMYOBNdH/h0zobQ 1955xGrFy8tfAaeBMIQ17gvFSp/suc9DYO0ICK2BISzq+F+ZiAKsjMYOBNdH/h0zobQ
83HTHs37+/QLMomGEGKZMWi0dShU2J5mNRQu3Hhxl3hHDVbt5CeJBb26aQcQrFz69W 196HTHs37+/QLMomGEGKZMWi0dShU2J5mNRQu3Hhxl3hHDVbt5CeJBb26aQcQrFz69W
84zE3GNvmJosh6leayjtI9P2A6iEkEGBECAAkFAkj3uiACGwwACgkQFlMNXpIPXGWp 197zE3GNvmJosh6leayjtI9P2A6iEkEGBECAAkFAkj3uiACGwwACgkQFlMNXpIPXGWp
85TACbBS+Up3RpfYVfd63c1cDdlru13pQAn3NQy/SN858MkxN+zym86UBgOad2 198TACbBS+Up3RpfYVfd63c1cDdlru13pQAn3NQy/SN858MkxN+zym86UBgOad2uQIN
86=CMiZ 199BF5FqOoBEAC8aRtWEtXzeuoQhdFrLTqYs2dy6kl9y+j3DMQYAMs8je582qzUigIO
87-----END PGP PUBLIC KEY BLOCK----- 200ZZxq7T/3WQgghsdw9yPvdzlw9tKdet2TJkR1mtBfSjZQrkKwR0pQP4AD7t/90Whu
88 201R8Wlu8ysapE2hLxMH5Y2znRQX2LkUYmk0K2ik9AgZEh3AFEg3YLl2pGnSjeSp3ch
89 Conley Owens <cco3@android.com> 202cLX2n/rVZf5LXluZGRG+iov1Ka+8m+UqzohMA1DYNECJW6KPgXsNX++i8/iwZVic
90-----BEGIN PGP PUBLIC KEY BLOCK----- 203PWzhRJSQC+QiAZNsKT6HNNKs97YCUVzhjBLnRSxRBPkr0hS/VMWY2V4pbASljWyd
91Version: GnuPG v1.4.11 (GNU/Linux) 204GYmlDcxheLne0yjes0bJAdvig5rB42FOV0FCM4bDYOVwKfZ7SpzGCYXxtlwe0XNG
92 205tLW9WA6tICVqNZ/JNiRTBLrsGSkyrEhDPKnIHlHRI5Zux6IHwMVB0lQKHjSop+t6
93mQENBFHRvc8BCADFg45Xx/y6QDC+T7Y/gGc7vx0ww7qfOwIKlAZ9xG3qKunMxo+S 206oyubqWcPCGGYdz2QGQHNz7huC/Zn0wS4hsoiSwPv6HCq3jNyUkOJ7wZ3ouv60p2I
94hPCnzEl3cq+6I1Ww/ndop/HB3N3toPXRCoN8Vs4/Hc7by+SnaLFnacrm+tV5/OgT 207kPurgviVaRaPSKTYdKfkcJOtFeqOh1na5IHkXsD9rNctB7tSgfsm0G6qJIVe3ZmJ
95V37Lzt8lhay1Kl+YfpFwHYYpIEBLFV9knyfRXS/428W2qhdzYfvB15/AasRmwmor 2087QAyHBfuLrAWCq5xS8EHDlvxPdAD8EEsa9T32YxcHKIkxr1eSwrUrKb8cPhWq1pp
96py4NIzSs8UD/SPr1ihqNCdZM76+MQyN5HMYXW/ALZXUFG0pwluHFA7hrfPG74i8C 209Jiylw6G1fZ02VKixqmPC4oFMyg1PO8L2tcQTrnVmZvfFGiaekHKdhQARAQABiQKW
97zMiP7qvMWIl/r/jtzHioH1dRKgbod+LZsrDJ8mBaqsZaDmNJMhss9g76XvfMyLra 210BBgRAgAgFiEEi7mteT6OYVOvD5pEFlMNXpIPXGUFAl5FqOoCGwICQAkQFlMNXpIP
989DI9/iFuBpGzeqBv0hwOGQspLRrEoyTeR6n1ABEBAAG0H0NvbmxleSBPd2VucyA8 211XGXBdCAEGQEKAB0WIQSjShO+jna/9GoMAi2i51qCSquWJAUCXkWo6gAKCRCi51qC
99Y2NvM0BhbmRyb2lkLmNvbT6JATgEEwECACIFAlHRvc8CGwMGCwkIBwMCBhUIAgkK 212SquWJLzgD/0YEZYS7yKxhP+kk94TcTYMBMSZpU5KFClB77yu4SI1LeXq4ocBT4sp
100CwQWAgMBAh4BAheAAAoJEGe35EhpKzgsP6AIAJKJmNtn4l7hkYHKHFSo3egb6RjQ 213EPaOsQiIx//j59J67b7CBe4UeRA6D2n0pw+bCKuc731DFi5X9C1zq3a7E67SQ2yd
101zEIP3MFTcu8HFX1kF1ZFbrp7xqurLaE53kEkKuAAvjJDAgI8mcZHP1JyplubqjQA 214FbYE2fnpVnMqb62g4sTh7JmdxEtXCWBUWL0OEoWouBW1PkFDHx2kYLC7YpZt3+4t
102xvv84gK+OGP3Xk+QK1ZjUQSbjOpjEiSZpRhWcHci3dgOUH4blJfByHw25hlgHowd 215VtNhSfV8NS6PF8ep3JXHVd2wsC3DQtggeId5GM44o8N0SkwQHNjK8ZD+VZ74ZnhZ
103a/2PrNKZVcJ92YienaxxGjcXEUcd0uYEG2+rwllQigFcnMFDhr9B71MfalRHjFKE 216HeyHskomiOC61LrZWQvxD6VqtfnBQ5GvONO8QuhkiFwMMOnpPVj2k7ngSkd5o27K
104fmdoypqLrri61YBc59P88Rw2/WUpTQjgNubSqa3A2+CKdaRyaRw+2fdF4TdR0h8W 2176c53ZESOlR4bAfl0i3RZYC9B5KerGkBE3dTgTzmGjOaahl2eLz4LDPdTwMtS+sAU
105zbg+lbaPtJHsV+3mJC7fq26MiJDRJa5ZztpMn8su20gbLgi2ShBOaHAYDDi5AQ0E 2181hPPvZTQeYDdV62bOWUyteMoJu354GgZPQ9eItWYixpNCyOGNcJXl6xk3/OuoP6f
106UdG9zwEIAMoOBq+QLNozAhxOOl5GL3StTStGRgPRXINfmViTsihrqGCWBBUfXlUE 219MciFV8aMxs/7mUR8q1Ei3X9MKu+bbODYj2rC1tMkLj1OaAJkfvRuYrKsQpoUsn4q
107OytC0mYcrDUQev/8ToVoyqw+iGSwDkcSXkrEUCKFtHV/GECWtk1keyHgR10YKI1R 220VT9+aciNpU/I7M30watlWo7RfUFI3zaGdMDcMFju1cWt2Un8E3gtscGufzbz1Z5Z
108mquSXoubWGqPeG1PAI74XWaRx8UrL8uCXUtmD8Q5J7mDjKR5NpxaXrwlA0bKsf2E 221Gak+tCOWUyuYNWX3noit7Dk6+3JGHGaQettldNu2PLM9SbIXd2EaqK/eEv9BS3dd
109Gp9tu1kKauuToZhWHMRMqYSOGikQJwWSFYKT1KdNcOXLQF6+bfoJ6sjVYdwfmNQL 222ItkZwzyZXSaQ9UqAceY1AHskJJ5KVXIRLuhP5jBWWo3fnRMyMYt2nwNBAJ9B9TA8
110Ixn8QVhoTDedcqClSWB17VDEFDFa7MmqXZz2qtM3X1R/MUMHqPtegQzBGNhRdnI2 223VlBniwIl5EzCvOFOTGrtewCdHOvr3N3ieypGz1BzyCN9tJMO3G24MwReRal9Fgkr
111V45+1Nnx/uuCxDbeI4RbHzujnxDiq70AEQEAAYkBHwQYAQIACQUCUdG9zwIbDAAK 224BgEEAdpHDwEBB0BhPE/je6OuKgWzJ1mnrUmHhn4IMOHp+58+T5kHU3Oy6YjXBBgR
112CRBnt+RIaSs4LNVeB/0Y2pZ8I7gAAcEM0Xw8drr4omg2fUoK1J33ozlA/RxeA/lJ 225AgAgFiEEi7mteT6OYVOvD5pEFlMNXpIPXGUFAl5FqX0CGwIAgQkQFlMNXpIPXGV2
113I3KnyCDTpXuIeBKPGkdL8uMATC9Z8DnBBajRlftNDVZS3Hz4G09G9QpMojvJkFJV 226IAQZFggAHRYhBOH5BA16P22vrIl809O5XaJD5Io5BQJeRal9AAoJENO5XaJD5Io5
114By+01Flw/X+eeN8NpqSuLV4W+AjEO8at/VvgKr1AFvBRdZ7GkpI1o6DgPe7ZqX+1 227MEkA/3uLmiwANOcgE0zB9zga0T/KkYhYOWFx7zRyDhrTf9spAPwIfSBOAGtwxjLO
115dzQZt3e13W0rVBb/bUgx9iSLoeWP3aq/k+/GRGOR+S6F6BBSl0SQ2EF2+dIywb1x 228DCce5OaQJl/YuGHvXq2yx5h7T8pdAZ+PAJ4qfIk2LLSidsplTDXOKhOQAuOqUQCf
116JuinEP+AwLAUZ1Bsx9ISC0Agpk2VeHXPL3FGhroEmoMvBzO0kTFGyoeT7PR/BfKv 229cZ7aFsJF4PtcDrfdejyAxbtsSHI=
117+H/g3HsL2LOB9uoIm8/5p2TTU5ttYCXMHhQZ81AY 230=82Tj
118=AUp4
119-----END PGP PUBLIC KEY BLOCK----- 231-----END PGP PUBLIC KEY BLOCK-----
120""" 232"""
121 233
122GIT = 'git' # our git command 234GIT = 'git' # our git command
235# NB: The version of git that the repo launcher requires may be much older than
236# the version of git that the main repo source tree requires. Keeping this at
237# an older version also makes it easier for users to upgrade/rollback as needed.
238#
239# git-1.7 is in (EOL) Ubuntu Precise.
123MIN_GIT_VERSION = (1, 7, 2) # minimum supported git version 240MIN_GIT_VERSION = (1, 7, 2) # minimum supported git version
124repodir = '.repo' # name of repo's private directory 241repodir = '.repo' # name of repo's private directory
125S_repo = 'repo' # special repo repository 242S_repo = 'repo' # special repo repository
126S_manifests = 'manifests' # special manifest repository 243S_manifests = 'manifests' # special manifest repository
127REPO_MAIN = S_repo + '/main.py' # main script 244REPO_MAIN = S_repo + '/main.py' # main script
128MIN_PYTHON_VERSION = (2, 7) # minimum supported python version
129GITC_CONFIG_FILE = '/gitc/.config' 245GITC_CONFIG_FILE = '/gitc/.config'
130GITC_FS_ROOT_DIR = '/gitc/manifest-rw/' 246GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
131 247
132 248
133import collections 249import collections
134import errno 250import errno
251import json
135import optparse 252import optparse
136import platform
137import re 253import re
138import shutil 254import shutil
139import stat 255import stat
140import subprocess
141import sys
142 256
143if sys.version_info[0] == 3: 257if sys.version_info[0] == 3:
144 import urllib.request 258 import urllib.request
@@ -151,116 +265,215 @@ else:
151 urllib.error = urllib2 265 urllib.error = urllib2
152 266
153 267
154# Python version check
155ver = sys.version_info
156if (ver[0], ver[1]) < MIN_PYTHON_VERSION:
157 print('error: Python version {} unsupported.\n'
158 'Please use Python {}.{} instead.'.format(
159 sys.version.split(' ')[0],
160 MIN_PYTHON_VERSION[0],
161 MIN_PYTHON_VERSION[1],
162 ), file=sys.stderr)
163 sys.exit(1)
164
165home_dot_repo = os.path.expanduser('~/.repoconfig') 268home_dot_repo = os.path.expanduser('~/.repoconfig')
166gpg_dir = os.path.join(home_dot_repo, 'gnupg') 269gpg_dir = os.path.join(home_dot_repo, 'gnupg')
167 270
168extra_args = [] 271
169init_optparse = optparse.OptionParser(usage="repo init -u url [options]") 272def GetParser(gitc_init=False):
170 273 """Setup the CLI parser."""
171# Logging 274 if gitc_init:
172group = init_optparse.add_option_group('Logging options') 275 usage = 'repo gitc-init -c client [options] [-u] url'
173group.add_option('-q', '--quiet', 276 else:
174 dest="quiet", action="store_true", default=False, 277 usage = 'repo init [options] [-u] url'
175 help="be quiet") 278
176 279 parser = optparse.OptionParser(usage=usage)
177# Manifest 280 InitParser(parser, gitc_init=gitc_init)
178group = init_optparse.add_option_group('Manifest options') 281 return parser
179group.add_option('-u', '--manifest-url', 282
180 dest='manifest_url', 283
181 help='manifest repository location', metavar='URL') 284def InitParser(parser, gitc_init=False):
182group.add_option('-b', '--manifest-branch', 285 """Setup the CLI parser."""
183 dest='manifest_branch', 286 # NB: Keep in sync with command.py:_CommonOptions().
184 help='manifest branch or revision', metavar='REVISION') 287
185group.add_option('-m', '--manifest-name', 288 # Logging.
186 dest='manifest_name', 289 group = parser.add_option_group('Logging options')
187 help='initial manifest file', metavar='NAME.xml') 290 group.add_option('-v', '--verbose',
188group.add_option('--current-branch', 291 dest='output_mode', action='store_true',
189 dest='current_branch_only', action='store_true', 292 help='show all output')
190 help='fetch only current manifest branch from server') 293 group.add_option('-q', '--quiet',
191group.add_option('--mirror', 294 dest='output_mode', action='store_false',
192 dest='mirror', action='store_true', 295 help='only show errors')
193 help='create a replica of the remote repositories ' 296
194 'rather than a client working directory') 297 # Manifest.
195group.add_option('--reference', 298 group = parser.add_option_group('Manifest options')
196 dest='reference', 299 group.add_option('-u', '--manifest-url',
197 help='location of mirror directory', metavar='DIR') 300 help='manifest repository location', metavar='URL')
198group.add_option('--dissociate', 301 group.add_option('-b', '--manifest-branch', metavar='REVISION',
199 dest='dissociate', action='store_true', 302 help='manifest branch or revision (use HEAD for default)')
200 help='dissociate from reference mirrors after clone') 303 group.add_option('-m', '--manifest-name', default='default.xml',
201group.add_option('--depth', type='int', default=None, 304 help='initial manifest file', metavar='NAME.xml')
202 dest='depth', 305 group.add_option('-g', '--groups', default='default',
203 help='create a shallow clone with given depth; see git clone') 306 help='restrict manifest projects to ones with specified '
204group.add_option('--partial-clone', action='store_true', 307 'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
205 dest='partial_clone', 308 metavar='GROUP')
206 help='perform partial clone (https://git-scm.com/' 309 group.add_option('-p', '--platform', default='auto',
207 'docs/gitrepository-layout#_code_partialclone_code)') 310 help='restrict manifest projects to ones with a specified '
208group.add_option('--clone-filter', action='store', default='blob:none', 311 'platform group [auto|all|none|linux|darwin|...]',
209 dest='clone_filter', 312 metavar='PLATFORM')
210 help='filter for use with --partial-clone [default: %default]') 313 group.add_option('--submodules', action='store_true',
211group.add_option('--archive', 314 help='sync any submodules associated with the manifest repo')
212 dest='archive', action='store_true', 315 group.add_option('--standalone-manifest', action='store_true',
213 help='checkout an archive instead of a git repository for ' 316 help='download the manifest as a static file '
214 'each project. See git archive.') 317 'rather then create a git checkout of '
215group.add_option('--submodules', 318 'the manifest repo')
216 dest='submodules', action='store_true', 319
217 help='sync any submodules associated with the manifest repo') 320 # Options that only affect manifest project, and not any of the projects
218group.add_option('-g', '--groups', 321 # specified in the manifest itself.
219 dest='groups', default='default', 322 group = parser.add_option_group('Manifest (only) checkout options')
220 help='restrict manifest projects to ones with specified ' 323 cbr_opts = ['--current-branch']
221 'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]', 324 # The gitc-init subcommand allocates -c itself, but a lot of init users
222 metavar='GROUP') 325 # want -c, so try to satisfy both as best we can.
223group.add_option('-p', '--platform', 326 if not gitc_init:
224 dest='platform', default="auto", 327 cbr_opts += ['-c']
225 help='restrict manifest projects to ones with a specified ' 328 group.add_option(*cbr_opts,
226 'platform group [auto|all|none|linux|darwin|...]', 329 dest='current_branch_only', action='store_true',
227 metavar='PLATFORM') 330 help='fetch only current manifest branch from server')
228group.add_option('--no-clone-bundle', 331 group.add_option('--no-current-branch',
229 dest='no_clone_bundle', action='store_true', 332 dest='current_branch_only', action='store_false',
230 help='disable use of /clone.bundle on HTTP/HTTPS') 333 help='fetch all manifest branches from server')
231group.add_option('--no-tags', 334 group.add_option('--tags',
232 dest='no_tags', action='store_true', 335 action='store_true',
233 help="don't fetch tags in the manifest") 336 help='fetch tags in the manifest')
234 337 group.add_option('--no-tags',
235 338 dest='tags', action='store_false',
236# Tool 339 help="don't fetch tags in the manifest")
237group = init_optparse.add_option_group('repo Version options') 340
238group.add_option('--repo-url', 341 # These are fundamentally different ways of structuring the checkout.
239 dest='repo_url', 342 group = parser.add_option_group('Checkout modes')
240 help='repo repository location ($REPO_URL)', metavar='URL') 343 group.add_option('--mirror', action='store_true',
241group.add_option('--repo-branch', 344 help='create a replica of the remote repositories '
242 dest='repo_branch', 345 'rather than a client working directory')
243 help='repo branch or revision ($REPO_REV)', metavar='REVISION') 346 group.add_option('--archive', action='store_true',
244group.add_option('--no-repo-verify', 347 help='checkout an archive instead of a git repository for '
245 dest='no_repo_verify', action='store_true', 348 'each project. See git archive.')
246 help='do not verify repo source code') 349 group.add_option('--worktree', action='store_true',
247 350 help='use git-worktree to manage projects')
248# Other 351
249group = init_optparse.add_option_group('Other options') 352 # These are fundamentally different ways of structuring the checkout.
250group.add_option('--config-name', 353 group = parser.add_option_group('Project checkout optimizations')
251 dest='config_name', action="store_true", default=False, 354 group.add_option('--reference',
252 help='Always prompt for name/e-mail') 355 help='location of mirror directory', metavar='DIR')
253 356 group.add_option('--dissociate', action='store_true',
254 357 help='dissociate from reference mirrors after clone')
255def _GitcInitOptions(init_optparse_arg): 358 group.add_option('--depth', type='int', default=None,
256 init_optparse_arg.set_usage("repo gitc-init -u url -c client [options]") 359 help='create a shallow clone with given depth; '
257 g = init_optparse_arg.add_option_group('GITC options') 360 'see git clone')
258 g.add_option('-f', '--manifest-file', 361 group.add_option('--partial-clone', action='store_true',
259 dest='manifest_file', 362 help='perform partial clone (https://git-scm.com/'
260 help='Optional manifest file to use for this GITC client.') 363 'docs/gitrepository-layout#_code_partialclone_code)')
261 g.add_option('-c', '--gitc-client', 364 group.add_option('--no-partial-clone', action='store_false',
262 dest='gitc_client', 365 help='disable use of partial clone (https://git-scm.com/'
263 help='The name of the gitc_client instance to create or modify.') 366 'docs/gitrepository-layout#_code_partialclone_code)')
367 group.add_option('--partial-clone-exclude', action='store',
368 help='exclude the specified projects (a comma-delimited '
369 'project names) from partial clone (https://git-scm.com'
370 '/docs/gitrepository-layout#_code_partialclone_code)')
371 group.add_option('--clone-filter', action='store', default='blob:none',
372 help='filter for use with --partial-clone '
373 '[default: %default]')
374 group.add_option('--use-superproject', action='store_true', default=None,
375 help='use the manifest superproject to sync projects')
376 group.add_option('--no-use-superproject', action='store_false',
377 dest='use_superproject',
378 help='disable use of manifest superprojects')
379 group.add_option('--clone-bundle', action='store_true',
380 help='enable use of /clone.bundle on HTTP/HTTPS '
381 '(default if not --partial-clone)')
382 group.add_option('--no-clone-bundle',
383 dest='clone_bundle', action='store_false',
384 help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)')
385
386 # Tool.
387 group = parser.add_option_group('repo Version options')
388 group.add_option('--repo-url', metavar='URL',
389 help='repo repository location ($REPO_URL)')
390 group.add_option('--repo-rev', metavar='REV',
391 help='repo branch or revision ($REPO_REV)')
392 group.add_option('--repo-branch', dest='repo_rev',
393 help=optparse.SUPPRESS_HELP)
394 group.add_option('--no-repo-verify',
395 dest='repo_verify', default=True, action='store_false',
396 help='do not verify repo source code')
397
398 # Other.
399 group = parser.add_option_group('Other options')
400 group.add_option('--config-name',
401 action='store_true', default=False,
402 help='Always prompt for name/e-mail')
403
404 # gitc-init specific settings.
405 if gitc_init:
406 group = parser.add_option_group('GITC options')
407 group.add_option('-f', '--manifest-file',
408 help='Optional manifest file to use for this GITC client.')
409 group.add_option('-c', '--gitc-client',
410 help='Name of the gitc_client instance to create or modify.')
411
412 return parser
413
414
415# This is a poor replacement for subprocess.run until we require Python 3.6+.
416RunResult = collections.namedtuple(
417 'RunResult', ('returncode', 'stdout', 'stderr'))
418
419
420class RunError(Exception):
421 """Error when running a command failed."""
422
423
424def run_command(cmd, **kwargs):
425 """Run |cmd| and return its output."""
426 check = kwargs.pop('check', False)
427 if kwargs.pop('capture_output', False):
428 kwargs.setdefault('stdout', subprocess.PIPE)
429 kwargs.setdefault('stderr', subprocess.PIPE)
430 cmd_input = kwargs.pop('input', None)
431
432 def decode(output):
433 """Decode |output| to text."""
434 if output is None:
435 return output
436 try:
437 return output.decode('utf-8')
438 except UnicodeError:
439 print('repo: warning: Invalid UTF-8 output:\ncmd: %r\n%r' % (cmd, output),
440 file=sys.stderr)
441 # TODO(vapier): Once we require Python 3, use 'backslashreplace'.
442 return output.decode('utf-8', 'replace')
443
444 # Run & package the results.
445 proc = subprocess.Popen(cmd, **kwargs)
446 (stdout, stderr) = proc.communicate(input=cmd_input)
447 dbg = ': ' + ' '.join(cmd)
448 if cmd_input is not None:
449 dbg += ' 0<|'
450 if stdout == subprocess.PIPE:
451 dbg += ' 1>|'
452 if stderr == subprocess.PIPE:
453 dbg += ' 2>|'
454 elif stderr == subprocess.STDOUT:
455 dbg += ' 2>&1'
456 trace.print(dbg)
457 ret = RunResult(proc.returncode, decode(stdout), decode(stderr))
458
459 # If things failed, print useful debugging output.
460 if check and ret.returncode:
461 print('repo: error: "%s" failed with exit status %s' %
462 (cmd[0], ret.returncode), file=sys.stderr)
463 print(' cwd: %s\n cmd: %r' %
464 (kwargs.get('cwd', os.getcwd()), cmd), file=sys.stderr)
465
466 def _print_output(name, output):
467 if output:
468 print(' %s:\n >> %s' % (name, '\n >> '.join(output.splitlines())),
469 file=sys.stderr)
470
471 _print_output('stdout', ret.stdout)
472 _print_output('stderr', ret.stderr)
473 raise RunError(ret)
474
475 return ret
476
264 477
265_gitc_manifest_dir = None 478_gitc_manifest_dir = None
266 479
@@ -283,9 +496,11 @@ def get_gitc_manifest_dir():
283def gitc_parse_clientdir(gitc_fs_path): 496def gitc_parse_clientdir(gitc_fs_path):
284 """Parse a path in the GITC FS and return its client name. 497 """Parse a path in the GITC FS and return its client name.
285 498
286 @param gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR. 499 Args:
500 gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
287 501
288 @returns: The GITC client name 502 Returns:
503 The GITC client name.
289 """ 504 """
290 if gitc_fs_path == GITC_FS_ROOT_DIR: 505 if gitc_fs_path == GITC_FS_ROOT_DIR:
291 return None 506 return None
@@ -309,31 +524,53 @@ class CloneFailure(Exception):
309 """ 524 """
310 525
311 526
527def check_repo_verify(repo_verify, quiet=False):
528 """Check the --repo-verify state."""
529 if not repo_verify:
530 print('repo: warning: verification of repo code has been disabled;\n'
531 'repo will not be able to verify the integrity of itself.\n',
532 file=sys.stderr)
533 return False
534
535 if NeedSetupGnuPG():
536 return SetupGnuPG(quiet)
537
538 return True
539
540
541def check_repo_rev(dst, rev, repo_verify=True, quiet=False):
542 """Check that |rev| is valid."""
543 do_verify = check_repo_verify(repo_verify, quiet=quiet)
544 remote_ref, local_rev = resolve_repo_rev(dst, rev)
545 if not quiet and not remote_ref.startswith('refs/heads/'):
546 print('warning: repo is not tracking a remote branch, so it will not '
547 'receive updates', file=sys.stderr)
548 if do_verify:
549 rev = verify_rev(dst, remote_ref, local_rev, quiet)
550 else:
551 rev = local_rev
552 return (remote_ref, rev)
553
554
312def _Init(args, gitc_init=False): 555def _Init(args, gitc_init=False):
313 """Installs repo by cloning it over the network. 556 """Installs repo by cloning it over the network.
314 """ 557 """
315 if gitc_init: 558 parser = GetParser(gitc_init=gitc_init)
316 _GitcInitOptions(init_optparse) 559 opt, args = parser.parse_args(args)
317 opt, args = init_optparse.parse_args(args)
318 if args: 560 if args:
319 init_optparse.print_usage() 561 if not opt.manifest_url:
320 sys.exit(1) 562 opt.manifest_url = args.pop(0)
321 563 if args:
322 url = opt.repo_url 564 parser.print_usage()
323 if not url: 565 sys.exit(1)
324 url = REPO_URL 566 opt.quiet = opt.output_mode is False
325 extra_args.append('--repo-url=%s' % url) 567 opt.verbose = opt.output_mode is True
326 568
327 branch = opt.repo_branch 569 if opt.clone_bundle is None:
328 if not branch: 570 opt.clone_bundle = False if opt.partial_clone else True
329 branch = REPO_REV
330 extra_args.append('--repo-branch=%s' % branch)
331 571
332 if branch.startswith('refs/heads/'): 572 url = opt.repo_url or REPO_URL
333 branch = branch[len('refs/heads/'):] 573 rev = opt.repo_rev or REPO_REV
334 if branch.startswith('refs/'):
335 print("fatal: invalid branch name '%s'" % branch, file=sys.stderr)
336 raise CloneFailure()
337 574
338 try: 575 try:
339 if gitc_init: 576 if gitc_init:
@@ -368,23 +605,13 @@ def _Init(args, gitc_init=False):
368 605
369 _CheckGitVersion() 606 _CheckGitVersion()
370 try: 607 try:
371 if opt.no_repo_verify: 608 if not opt.quiet:
372 do_verify = False 609 print('Downloading Repo source from', url)
373 else:
374 if NeedSetupGnuPG():
375 do_verify = SetupGnuPG(opt.quiet)
376 else:
377 do_verify = True
378
379 dst = os.path.abspath(os.path.join(repodir, S_repo)) 610 dst = os.path.abspath(os.path.join(repodir, S_repo))
380 _Clone(url, dst, opt.quiet, not opt.no_clone_bundle) 611 _Clone(url, dst, opt.clone_bundle, opt.quiet, opt.verbose)
381
382 if do_verify:
383 rev = _Verify(dst, branch, opt.quiet)
384 else:
385 rev = 'refs/remotes/origin/%s^0' % branch
386 612
387 _Checkout(dst, branch, rev, opt.quiet) 613 remote_ref, rev = check_repo_rev(dst, rev, opt.repo_verify, quiet=opt.quiet)
614 _Checkout(dst, remote_ref, rev, opt.quiet)
388 615
389 if not os.path.isfile(os.path.join(dst, 'repo')): 616 if not os.path.isfile(os.path.join(dst, 'repo')):
390 print("warning: '%s' does not look like a git-repo repository, is " 617 print("warning: '%s' does not look like a git-repo repository, is "
@@ -397,15 +624,34 @@ def _Init(args, gitc_init=False):
397 raise 624 raise
398 625
399 626
627def run_git(*args, **kwargs):
628 """Run git and return execution details."""
629 kwargs.setdefault('capture_output', True)
630 kwargs.setdefault('check', True)
631 try:
632 return run_command([GIT] + list(args), **kwargs)
633 except OSError as e:
634 print(file=sys.stderr)
635 print('repo: error: "%s" is not available' % GIT, file=sys.stderr)
636 print('repo: error: %s' % e, file=sys.stderr)
637 print(file=sys.stderr)
638 print('Please make sure %s is installed and in your path.' % GIT,
639 file=sys.stderr)
640 sys.exit(1)
641 except RunError:
642 raise CloneFailure()
643
644
400# The git version info broken down into components for easy analysis. 645# The git version info broken down into components for easy analysis.
401# Similar to Python's sys.version_info. 646# Similar to Python's sys.version_info.
402GitVersion = collections.namedtuple( 647GitVersion = collections.namedtuple(
403 'GitVersion', ('major', 'minor', 'micro', 'full')) 648 'GitVersion', ('major', 'minor', 'micro', 'full'))
404 649
650
405def ParseGitVersion(ver_str=None): 651def ParseGitVersion(ver_str=None):
406 if ver_str is None: 652 if ver_str is None:
407 # Load the version ourselves. 653 # Load the version ourselves.
408 ver_str = _GetGitVersion() 654 ver_str = run_git('--version').stdout
409 655
410 if not ver_str.startswith('git version '): 656 if not ver_str.startswith('git version '):
411 return None 657 return None
@@ -422,41 +668,52 @@ def ParseGitVersion(ver_str=None):
422 return GitVersion(*to_tuple) 668 return GitVersion(*to_tuple)
423 669
424 670
425def _GetGitVersion():
426 cmd = [GIT, '--version']
427 try:
428 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
429 except OSError as e:
430 print(file=sys.stderr)
431 print("fatal: '%s' is not available" % GIT, file=sys.stderr)
432 print('fatal: %s' % e, file=sys.stderr)
433 print(file=sys.stderr)
434 print('Please make sure %s is installed and in your path.' % GIT,
435 file=sys.stderr)
436 raise
437
438 ver_str = proc.stdout.read().strip()
439 proc.stdout.close()
440 proc.wait()
441 return ver_str.decode('utf-8')
442
443
444def _CheckGitVersion(): 671def _CheckGitVersion():
445 try: 672 ver_act = ParseGitVersion()
446 ver_act = ParseGitVersion()
447 except OSError:
448 raise CloneFailure()
449
450 if ver_act is None: 673 if ver_act is None:
451 print('fatal: unable to detect git version', file=sys.stderr) 674 print('fatal: unable to detect git version', file=sys.stderr)
452 raise CloneFailure() 675 raise CloneFailure()
453 676
454 if ver_act < MIN_GIT_VERSION: 677 if ver_act < MIN_GIT_VERSION:
455 need = '.'.join(map(str, MIN_GIT_VERSION)) 678 need = '.'.join(map(str, MIN_GIT_VERSION))
456 print('fatal: git %s or later required' % need, file=sys.stderr) 679 print('fatal: git %s or later required; found %s' % (need, ver_act.full),
680 file=sys.stderr)
457 raise CloneFailure() 681 raise CloneFailure()
458 682
459 683
684def SetGitTrace2ParentSid(env=None):
685 """Set up GIT_TRACE2_PARENT_SID for git tracing."""
686 # We roughly follow the format git itself uses in trace2/tr2_sid.c.
687 # (1) Be unique (2) be valid filename (3) be fixed length.
688 #
689 # Since we always export this variable, we try to avoid more expensive calls.
690 # e.g. We don't attempt hostname lookups or hashing the results.
691 if env is None:
692 env = os.environ
693
694 KEY = 'GIT_TRACE2_PARENT_SID'
695
696 now = datetime.datetime.utcnow()
697 value = 'repo-%s-P%08x' % (now.strftime('%Y%m%dT%H%M%SZ'), os.getpid())
698
699 # If it's already set, then append ourselves.
700 if KEY in env:
701 value = env[KEY] + '/' + value
702
703 _setenv(KEY, value, env=env)
704
705
706def _setenv(key, value, env=None):
707 """Set |key| in the OS environment |env| to |value|."""
708 if env is None:
709 env = os.environ
710 # Environment handling across systems is messy.
711 try:
712 env[key] = value
713 except UnicodeEncodeError:
714 env[key] = value.encode()
715
716
460def NeedSetupGnuPG(): 717def NeedSetupGnuPG():
461 if not os.path.isdir(home_dot_repo): 718 if not os.path.isdir(home_dot_repo):
462 return True 719 return True
@@ -492,43 +749,54 @@ def SetupGnuPG(quiet):
492 file=sys.stderr) 749 file=sys.stderr)
493 sys.exit(1) 750 sys.exit(1)
494 751
495 env = os.environ.copy() 752 if not quiet:
496 try: 753 print('repo: Updating release signing keys to keyset ver %s' %
497 env['GNUPGHOME'] = gpg_dir 754 ('.'.join(str(x) for x in KEYRING_VERSION),))
498 except UnicodeEncodeError: 755 # NB: We use --homedir (and cwd below) because some environments (Windows) do
499 env['GNUPGHOME'] = gpg_dir.encode() 756 # not correctly handle full native paths. We avoid the issue by changing to
500 757 # the right dir with cwd=gpg_dir before executing gpg, and then telling gpg to
501 cmd = ['gpg', '--import'] 758 # use the cwd (.) as its homedir which leaves the path resolution logic to it.
759 cmd = ['gpg', '--homedir', '.', '--import']
502 try: 760 try:
503 proc = subprocess.Popen(cmd, 761 # gpg can be pretty chatty. Always capture the output and if something goes
504 env=env, 762 # wrong, the builtin check failure will dump stdout & stderr for debugging.
505 stdin=subprocess.PIPE) 763 run_command(cmd, stdin=subprocess.PIPE, capture_output=True,
506 except OSError as e: 764 cwd=gpg_dir, check=True,
765 input=MAINTAINER_KEYS.encode('utf-8'))
766 except OSError:
507 if not quiet: 767 if not quiet:
508 print('warning: gpg (GnuPG) is not available.', file=sys.stderr) 768 print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
509 print('warning: Installing it is strongly encouraged.', file=sys.stderr) 769 print('warning: Installing it is strongly encouraged.', file=sys.stderr)
510 print(file=sys.stderr) 770 print(file=sys.stderr)
511 return False 771 return False
512 772
513 proc.stdin.write(MAINTAINER_KEYS.encode('utf-8'))
514 proc.stdin.close()
515
516 if proc.wait() != 0:
517 print('fatal: registering repo maintainer keys failed', file=sys.stderr)
518 sys.exit(1)
519 print()
520
521 with open(os.path.join(home_dot_repo, 'keyring-version'), 'w') as fd: 773 with open(os.path.join(home_dot_repo, 'keyring-version'), 'w') as fd:
522 fd.write('.'.join(map(str, KEYRING_VERSION)) + '\n') 774 fd.write('.'.join(map(str, KEYRING_VERSION)) + '\n')
523 return True 775 return True
524 776
525 777
526def _SetConfig(local, name, value): 778def _SetConfig(cwd, name, value):
527 """Set a git configuration option to the specified value. 779 """Set a git configuration option to the specified value.
528 """ 780 """
529 cmd = [GIT, 'config', name, value] 781 run_git('config', name, value, cwd=cwd)
530 if subprocess.Popen(cmd, cwd=local).wait() != 0: 782
531 raise CloneFailure() 783
784def _GetRepoConfig(name):
785 """Read a repo configuration option."""
786 config = os.path.join(home_dot_repo, 'config')
787 if not os.path.exists(config):
788 return None
789
790 cmd = ['config', '--file', config, '--get', name]
791 ret = run_git(*cmd, check=False)
792 if ret.returncode == 0:
793 return ret.stdout
794 elif ret.returncode == 1:
795 return None
796 else:
797 print('repo: error: git %s failed:\n%s' % (' '.join(cmd), ret.stderr),
798 file=sys.stderr)
799 raise RunError()
532 800
533 801
534def _InitHttp(): 802def _InitHttp():
@@ -542,7 +810,7 @@ def _InitHttp():
542 p = n.hosts[host] 810 p = n.hosts[host]
543 mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2]) 811 mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2])
544 mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2]) 812 mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
545 except: 813 except Exception:
546 pass 814 pass
547 handlers.append(urllib.request.HTTPBasicAuthHandler(mgr)) 815 handlers.append(urllib.request.HTTPBasicAuthHandler(mgr))
548 handlers.append(urllib.request.HTTPDigestAuthHandler(mgr)) 816 handlers.append(urllib.request.HTTPDigestAuthHandler(mgr))
@@ -556,39 +824,29 @@ def _InitHttp():
556 urllib.request.install_opener(urllib.request.build_opener(*handlers)) 824 urllib.request.install_opener(urllib.request.build_opener(*handlers))
557 825
558 826
559def _Fetch(url, local, src, quiet): 827def _Fetch(url, cwd, src, quiet, verbose):
560 if not quiet: 828 cmd = ['fetch']
561 print('Get %s' % url, file=sys.stderr) 829 if not verbose:
562
563 cmd = [GIT, 'fetch']
564 if quiet:
565 cmd.append('--quiet') 830 cmd.append('--quiet')
831 err = None
832 if not quiet and sys.stdout.isatty():
833 cmd.append('--progress')
834 elif not verbose:
566 err = subprocess.PIPE 835 err = subprocess.PIPE
567 else:
568 err = None
569 cmd.append(src) 836 cmd.append(src)
570 cmd.append('+refs/heads/*:refs/remotes/origin/*') 837 cmd.append('+refs/heads/*:refs/remotes/origin/*')
571 cmd.append('+refs/tags/*:refs/tags/*') 838 cmd.append('+refs/tags/*:refs/tags/*')
572 839 run_git(*cmd, stderr=err, capture_output=False, cwd=cwd)
573 proc = subprocess.Popen(cmd, cwd=local, stderr=err)
574 if err:
575 proc.stderr.read()
576 proc.stderr.close()
577 if proc.wait() != 0:
578 raise CloneFailure()
579 840
580 841
581def _DownloadBundle(url, local, quiet): 842def _DownloadBundle(url, cwd, quiet, verbose):
582 if not url.endswith('/'): 843 if not url.endswith('/'):
583 url += '/' 844 url += '/'
584 url += 'clone.bundle' 845 url += 'clone.bundle'
585 846
586 proc = subprocess.Popen( 847 ret = run_git('config', '--get-regexp', 'url.*.insteadof', cwd=cwd,
587 [GIT, 'config', '--get-regexp', 'url.*.insteadof'], 848 check=False)
588 cwd=local, 849 for line in ret.stdout.splitlines():
589 stdout=subprocess.PIPE)
590 for line in proc.stdout:
591 line = line.decode('utf-8')
592 m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line) 850 m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line)
593 if m: 851 if m:
594 new_url = m.group(1) 852 new_url = m.group(1)
@@ -596,29 +854,26 @@ def _DownloadBundle(url, local, quiet):
596 if url.startswith(old_url): 854 if url.startswith(old_url):
597 url = new_url + url[len(old_url):] 855 url = new_url + url[len(old_url):]
598 break 856 break
599 proc.stdout.close()
600 proc.wait()
601 857
602 if not url.startswith('http:') and not url.startswith('https:'): 858 if not url.startswith('http:') and not url.startswith('https:'):
603 return False 859 return False
604 860
605 dest = open(os.path.join(local, '.git', 'clone.bundle'), 'w+b') 861 dest = open(os.path.join(cwd, '.git', 'clone.bundle'), 'w+b')
606 try: 862 try:
607 try: 863 try:
608 r = urllib.request.urlopen(url) 864 r = urllib.request.urlopen(url)
609 except urllib.error.HTTPError as e: 865 except urllib.error.HTTPError as e:
610 if e.code in [401, 403, 404, 501]: 866 if e.code not in [400, 401, 403, 404, 501]:
611 return False 867 print('warning: Cannot get %s' % url, file=sys.stderr)
612 print('fatal: Cannot get %s' % url, file=sys.stderr) 868 print('warning: HTTP error %s' % e.code, file=sys.stderr)
613 print('fatal: HTTP error %s' % e.code, file=sys.stderr) 869 return False
614 raise CloneFailure()
615 except urllib.error.URLError as e: 870 except urllib.error.URLError as e:
616 print('fatal: Cannot get %s' % url, file=sys.stderr) 871 print('fatal: Cannot get %s' % url, file=sys.stderr)
617 print('fatal: error %s' % e.reason, file=sys.stderr) 872 print('fatal: error %s' % e.reason, file=sys.stderr)
618 raise CloneFailure() 873 raise CloneFailure()
619 try: 874 try:
620 if not quiet: 875 if verbose:
621 print('Get %s' % url, file=sys.stderr) 876 print('Downloading clone bundle %s' % url, file=sys.stderr)
622 while True: 877 while True:
623 buf = r.read(8192) 878 buf = r.read(8192)
624 if not buf: 879 if not buf:
@@ -630,124 +885,139 @@ def _DownloadBundle(url, local, quiet):
630 dest.close() 885 dest.close()
631 886
632 887
633def _ImportBundle(local): 888def _ImportBundle(cwd):
634 path = os.path.join(local, '.git', 'clone.bundle') 889 path = os.path.join(cwd, '.git', 'clone.bundle')
635 try: 890 try:
636 _Fetch(local, local, path, True) 891 _Fetch(cwd, cwd, path, True, False)
637 finally: 892 finally:
638 os.remove(path) 893 os.remove(path)
639 894
640 895
641def _Clone(url, local, quiet, clone_bundle): 896def _Clone(url, cwd, clone_bundle, quiet, verbose):
642 """Clones a git repository to a new subdirectory of repodir 897 """Clones a git repository to a new subdirectory of repodir
643 """ 898 """
644 try: 899 if verbose:
645 os.mkdir(local) 900 print('Cloning git repository', url)
646 except OSError as e:
647 print('fatal: cannot make %s directory: %s' % (local, e.strerror),
648 file=sys.stderr)
649 raise CloneFailure()
650 901
651 cmd = [GIT, 'init', '--quiet']
652 try: 902 try:
653 proc = subprocess.Popen(cmd, cwd=local) 903 os.mkdir(cwd)
654 except OSError as e: 904 except OSError as e:
655 print(file=sys.stderr) 905 print('fatal: cannot make %s directory: %s' % (cwd, e.strerror),
656 print("fatal: '%s' is not available" % GIT, file=sys.stderr)
657 print('fatal: %s' % e, file=sys.stderr)
658 print(file=sys.stderr)
659 print('Please make sure %s is installed and in your path.' % GIT,
660 file=sys.stderr) 906 file=sys.stderr)
661 raise CloneFailure() 907 raise CloneFailure()
662 if proc.wait() != 0: 908
663 print('fatal: could not create %s' % local, file=sys.stderr) 909 run_git('init', '--quiet', cwd=cwd)
664 raise CloneFailure()
665 910
666 _InitHttp() 911 _InitHttp()
667 _SetConfig(local, 'remote.origin.url', url) 912 _SetConfig(cwd, 'remote.origin.url', url)
668 _SetConfig(local, 913 _SetConfig(cwd,
669 'remote.origin.fetch', 914 'remote.origin.fetch',
670 '+refs/heads/*:refs/remotes/origin/*') 915 '+refs/heads/*:refs/remotes/origin/*')
671 if clone_bundle and _DownloadBundle(url, local, quiet): 916 if clone_bundle and _DownloadBundle(url, cwd, quiet, verbose):
672 _ImportBundle(local) 917 _ImportBundle(cwd)
673 _Fetch(url, local, 'origin', quiet) 918 _Fetch(url, cwd, 'origin', quiet, verbose)
919
920
921def resolve_repo_rev(cwd, committish):
922 """Figure out what REPO_REV represents.
674 923
924 We support:
925 * refs/heads/xxx: Branch.
926 * refs/tags/xxx: Tag.
927 * xxx: Branch or tag or commit.
675 928
676def _Verify(cwd, branch, quiet): 929 Args:
677 """Verify the branch has been signed by a tag. 930 cwd: The git checkout to run in.
931 committish: The REPO_REV argument to resolve.
932
933 Returns:
934 A tuple of (remote ref, commit) as makes sense for the committish.
935 For branches, this will look like ('refs/heads/stable', <revision>).
936 For tags, this will look like ('refs/tags/v1.0', <revision>).
937 For commits, this will be (<revision>, <revision>).
678 """ 938 """
679 cmd = [GIT, 'describe', 'origin/%s' % branch] 939 def resolve(committish):
680 proc = subprocess.Popen(cmd, 940 ret = run_git('rev-parse', '--verify', '%s^{commit}' % (committish,),
681 stdout=subprocess.PIPE, 941 cwd=cwd, check=False)
682 stderr=subprocess.PIPE, 942 return None if ret.returncode else ret.stdout.strip()
683 cwd=cwd) 943
684 cur = proc.stdout.read().strip().decode('utf-8') 944 # An explicit branch.
685 proc.stdout.close() 945 if committish.startswith('refs/heads/'):
686 946 remote_ref = committish
687 proc.stderr.read() 947 committish = committish[len('refs/heads/'):]
688 proc.stderr.close() 948 rev = resolve('refs/remotes/origin/%s' % committish)
689 949 if rev is None:
690 if proc.wait() != 0 or not cur: 950 print('repo: error: unknown branch "%s"' % (committish,),
691 print(file=sys.stderr) 951 file=sys.stderr)
692 print("fatal: branch '%s' has not been signed" % branch, file=sys.stderr) 952 raise CloneFailure()
693 raise CloneFailure() 953 return (remote_ref, rev)
954
955 # An explicit tag.
956 if committish.startswith('refs/tags/'):
957 remote_ref = committish
958 committish = committish[len('refs/tags/'):]
959 rev = resolve(remote_ref)
960 if rev is None:
961 print('repo: error: unknown tag "%s"' % (committish,),
962 file=sys.stderr)
963 raise CloneFailure()
964 return (remote_ref, rev)
965
966 # See if it's a short branch name.
967 rev = resolve('refs/remotes/origin/%s' % committish)
968 if rev:
969 return ('refs/heads/%s' % (committish,), rev)
970
971 # See if it's a tag.
972 rev = resolve('refs/tags/%s' % committish)
973 if rev:
974 return ('refs/tags/%s' % (committish,), rev)
975
976 # See if it's a commit.
977 rev = resolve(committish)
978 if rev and rev.lower().startswith(committish.lower()):
979 return (rev, rev)
980
981 # Give up!
982 print('repo: error: unable to resolve "%s"' % (committish,), file=sys.stderr)
983 raise CloneFailure()
984
985
986def verify_rev(cwd, remote_ref, rev, quiet):
987 """Verify the commit has been signed by a tag."""
988 ret = run_git('describe', rev, cwd=cwd)
989 cur = ret.stdout.strip()
694 990
695 m = re.compile(r'^(.*)-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur) 991 m = re.compile(r'^(.*)-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur)
696 if m: 992 if m:
697 cur = m.group(1) 993 cur = m.group(1)
698 if not quiet: 994 if not quiet:
699 print(file=sys.stderr) 995 print(file=sys.stderr)
700 print("info: Ignoring branch '%s'; using tagged release '%s'" 996 print("warning: '%s' is not signed; falling back to signed release '%s'"
701 % (branch, cur), file=sys.stderr) 997 % (remote_ref, cur), file=sys.stderr)
702 print(file=sys.stderr) 998 print(file=sys.stderr)
703 999
704 env = os.environ.copy() 1000 env = os.environ.copy()
705 try: 1001 _setenv('GNUPGHOME', gpg_dir, env)
706 env['GNUPGHOME'] = gpg_dir 1002 run_git('tag', '-v', cur, cwd=cwd, env=env)
707 except UnicodeEncodeError:
708 env['GNUPGHOME'] = gpg_dir.encode()
709
710 cmd = [GIT, 'tag', '-v', cur]
711 proc = subprocess.Popen(cmd,
712 stdout=subprocess.PIPE,
713 stderr=subprocess.PIPE,
714 cwd=cwd,
715 env=env)
716 out = proc.stdout.read().decode('utf-8')
717 proc.stdout.close()
718
719 err = proc.stderr.read().decode('utf-8')
720 proc.stderr.close()
721
722 if proc.wait() != 0:
723 print(file=sys.stderr)
724 print(out, file=sys.stderr)
725 print(err, file=sys.stderr)
726 print(file=sys.stderr)
727 raise CloneFailure()
728 return '%s^0' % cur 1003 return '%s^0' % cur
729 1004
730 1005
731def _Checkout(cwd, branch, rev, quiet): 1006def _Checkout(cwd, remote_ref, rev, quiet):
732 """Checkout an upstream branch into the repository and track it. 1007 """Checkout an upstream branch into the repository and track it.
733 """ 1008 """
734 cmd = [GIT, 'update-ref', 'refs/heads/default', rev] 1009 run_git('update-ref', 'refs/heads/default', rev, cwd=cwd)
735 if subprocess.Popen(cmd, cwd=cwd).wait() != 0:
736 raise CloneFailure()
737 1010
738 _SetConfig(cwd, 'branch.default.remote', 'origin') 1011 _SetConfig(cwd, 'branch.default.remote', 'origin')
739 _SetConfig(cwd, 'branch.default.merge', 'refs/heads/%s' % branch) 1012 _SetConfig(cwd, 'branch.default.merge', remote_ref)
740 1013
741 cmd = [GIT, 'symbolic-ref', 'HEAD', 'refs/heads/default'] 1014 run_git('symbolic-ref', 'HEAD', 'refs/heads/default', cwd=cwd)
742 if subprocess.Popen(cmd, cwd=cwd).wait() != 0:
743 raise CloneFailure()
744 1015
745 cmd = [GIT, 'read-tree', '--reset', '-u'] 1016 cmd = ['read-tree', '--reset', '-u']
746 if not quiet: 1017 if not quiet:
747 cmd.append('-v') 1018 cmd.append('-v')
748 cmd.append('HEAD') 1019 cmd.append('HEAD')
749 if subprocess.Popen(cmd, cwd=cwd).wait() != 0: 1020 run_git(*cmd, cwd=cwd)
750 raise CloneFailure()
751 1021
752 1022
753def _FindRepo(): 1023def _FindRepo():
@@ -757,9 +1027,7 @@ def _FindRepo():
757 repo = None 1027 repo = None
758 1028
759 olddir = None 1029 olddir = None
760 while curdir != '/' \ 1030 while curdir != olddir and not repo:
761 and curdir != olddir \
762 and not repo:
763 repo = os.path.join(curdir, repodir, REPO_MAIN) 1031 repo = os.path.join(curdir, repodir, REPO_MAIN)
764 if not os.path.isfile(repo): 1032 if not os.path.isfile(repo):
765 repo = None 1033 repo = None
@@ -770,6 +1038,26 @@ def _FindRepo():
770 1038
771class _Options(object): 1039class _Options(object):
772 help = False 1040 help = False
1041 version = False
1042
1043
1044def _ExpandAlias(name):
1045 """Look up user registered aliases."""
1046 # We don't resolve aliases for existing subcommands. This matches git.
1047 if name in {'gitc-init', 'help', 'init'}:
1048 return name, []
1049
1050 alias = _GetRepoConfig('alias.%s' % (name,))
1051 if alias is None:
1052 return name, []
1053
1054 args = alias.strip().split(' ', 1)
1055 name = args[0]
1056 if len(args) == 2:
1057 args = shlex.split(args[1])
1058 else:
1059 args = []
1060 return name, args
773 1061
774 1062
775def _ParseArguments(args): 1063def _ParseArguments(args):
@@ -781,7 +1069,10 @@ def _ParseArguments(args):
781 a = args[i] 1069 a = args[i]
782 if a == '-h' or a == '--help': 1070 if a == '-h' or a == '--help':
783 opt.help = True 1071 opt.help = True
784 1072 elif a == '--version':
1073 opt.version = True
1074 elif a == '--trace':
1075 trace.set(True)
785 elif not a.startswith('-'): 1076 elif not a.startswith('-'):
786 cmd = a 1077 cmd = a
787 arg = args[i + 1:] 1078 arg = args[i + 1:]
@@ -789,6 +1080,90 @@ def _ParseArguments(args):
789 return cmd, opt, arg 1080 return cmd, opt, arg
790 1081
791 1082
1083class Requirements(object):
1084 """Helper for checking repo's system requirements."""
1085
1086 REQUIREMENTS_NAME = 'requirements.json'
1087
1088 def __init__(self, requirements):
1089 """Initialize.
1090
1091 Args:
1092 requirements: A dictionary of settings.
1093 """
1094 self.requirements = requirements
1095
1096 @classmethod
1097 def from_dir(cls, path):
1098 return cls.from_file(os.path.join(path, cls.REQUIREMENTS_NAME))
1099
1100 @classmethod
1101 def from_file(cls, path):
1102 try:
1103 with open(path, 'rb') as f:
1104 data = f.read()
1105 except EnvironmentError:
1106 # NB: EnvironmentError is used for Python 2 & 3 compatibility.
1107 # If we couldn't open the file, assume it's an old source tree.
1108 return None
1109
1110 return cls.from_data(data)
1111
1112 @classmethod
1113 def from_data(cls, data):
1114 comment_line = re.compile(br'^ *#')
1115 strip_data = b''.join(x for x in data.splitlines() if not comment_line.match(x))
1116 try:
1117 json_data = json.loads(strip_data)
1118 except Exception: # pylint: disable=broad-except
1119 # If we couldn't parse it, assume it's incompatible.
1120 return None
1121
1122 return cls(json_data)
1123
1124 def _get_soft_ver(self, pkg):
1125 """Return the soft version for |pkg| if it exists."""
1126 return self.requirements.get(pkg, {}).get('soft', ())
1127
1128 def _get_hard_ver(self, pkg):
1129 """Return the hard version for |pkg| if it exists."""
1130 return self.requirements.get(pkg, {}).get('hard', ())
1131
1132 @staticmethod
1133 def _format_ver(ver):
1134 """Return a dotted version from |ver|."""
1135 return '.'.join(str(x) for x in ver)
1136
1137 def assert_ver(self, pkg, curr_ver):
1138 """Verify |pkg|'s |curr_ver| is new enough."""
1139 curr_ver = tuple(curr_ver)
1140 soft_ver = tuple(self._get_soft_ver(pkg))
1141 hard_ver = tuple(self._get_hard_ver(pkg))
1142 if curr_ver < hard_ver:
1143 print('repo: error: Your version of "%s" (%s) is unsupported; '
1144 'Please upgrade to at least version %s to continue.' %
1145 (pkg, self._format_ver(curr_ver), self._format_ver(soft_ver)),
1146 file=sys.stderr)
1147 sys.exit(1)
1148
1149 if curr_ver < soft_ver:
1150 print('repo: warning: Your version of "%s" (%s) is no longer supported; '
1151 'Please upgrade to at least version %s to avoid breakage.' %
1152 (pkg, self._format_ver(curr_ver), self._format_ver(soft_ver)),
1153 file=sys.stderr)
1154
1155 def assert_all(self):
1156 """Assert all of the requirements are satisified."""
1157 # See if we need a repo launcher upgrade first.
1158 self.assert_ver('repo', VERSION)
1159
1160 # Check python before we try to import the repo code.
1161 self.assert_ver('python', sys.version_info)
1162
1163 # Check git while we're at it.
1164 self.assert_ver('git', ParseGitVersion())
1165
1166
792def _Usage(): 1167def _Usage():
793 gitc_usage = "" 1168 gitc_usage = ""
794 if get_gitc_manifest_dir(): 1169 if get_gitc_manifest_dir():
@@ -807,17 +1182,15 @@ The most commonly used repo commands are:
807 1182
808For access to the full online help, install repo ("repo init"). 1183For access to the full online help, install repo ("repo init").
809""") 1184""")
1185 print('Bug reports:', BUG_URL)
810 sys.exit(0) 1186 sys.exit(0)
811 1187
812 1188
813def _Help(args): 1189def _Help(args):
814 if args: 1190 if args:
815 if args[0] == 'init': 1191 if args[0] in {'init', 'gitc-init'}:
816 init_optparse.print_help() 1192 parser = GetParser(gitc_init=args[0] == 'gitc-init')
817 sys.exit(0) 1193 parser.print_help()
818 elif args[0] == 'gitc-init':
819 _GitcInitOptions(init_optparse)
820 init_optparse.print_help()
821 sys.exit(0) 1194 sys.exit(0)
822 else: 1195 else:
823 print("error: '%s' is not a bootstrap command.\n" 1196 print("error: '%s' is not a bootstrap command.\n"
@@ -828,6 +1201,25 @@ def _Help(args):
828 sys.exit(1) 1201 sys.exit(1)
829 1202
830 1203
1204def _Version():
1205 """Show version information."""
1206 print('<repo not installed>')
1207 print('repo launcher version %s' % ('.'.join(str(x) for x in VERSION),))
1208 print(' (from %s)' % (__file__,))
1209 print('git %s' % (ParseGitVersion().full,))
1210 print('Python %s' % sys.version)
1211 uname = platform.uname()
1212 if sys.version_info.major < 3:
1213 # Python 3 returns a named tuple, but Python 2 is simpler.
1214 print(uname)
1215 else:
1216 print('OS %s %s (%s)' % (uname.system, uname.release, uname.version))
1217 print('CPU %s (%s)' %
1218 (uname.machine, uname.processor if uname.processor else 'unknown'))
1219 print('Bug reports:', BUG_URL)
1220 sys.exit(0)
1221
1222
831def _NotInstalled(): 1223def _NotInstalled():
832 print('error: repo is not installed. Use "repo init" to install it here.', 1224 print('error: repo is not installed. Use "repo init" to install it here.',
833 file=sys.stderr) 1225 file=sys.stderr)
@@ -860,26 +1252,26 @@ def _SetDefaultsTo(gitdir):
860 global REPO_REV 1252 global REPO_REV
861 1253
862 REPO_URL = gitdir 1254 REPO_URL = gitdir
863 proc = subprocess.Popen([GIT, 1255 ret = run_git('--git-dir=%s' % gitdir, 'symbolic-ref', 'HEAD', check=False)
864 '--git-dir=%s' % gitdir, 1256 if ret.returncode:
865 'symbolic-ref', 1257 # If we're not tracking a branch (bisect/etc...), then fall back to commit.
866 'HEAD'], 1258 print('repo: warning: %s has no current branch; using HEAD' % gitdir,
867 stdout=subprocess.PIPE, 1259 file=sys.stderr)
868 stderr=subprocess.PIPE) 1260 try:
869 REPO_REV = proc.stdout.read().strip().decode('utf-8') 1261 ret = run_git('rev-parse', 'HEAD', cwd=gitdir)
870 proc.stdout.close() 1262 except CloneFailure:
871 1263 print('fatal: %s has invalid HEAD' % gitdir, file=sys.stderr)
872 proc.stderr.read() 1264 sys.exit(1)
873 proc.stderr.close() 1265
874 1266 REPO_REV = ret.stdout.strip()
875 if proc.wait() != 0:
876 print('fatal: %s has no current branch' % gitdir, file=sys.stderr)
877 sys.exit(1)
878 1267
879 1268
880def main(orig_args): 1269def main(orig_args):
881 cmd, opt, args = _ParseArguments(orig_args) 1270 cmd, opt, args = _ParseArguments(orig_args)
882 1271
1272 # We run this early as we run some git commands ourselves.
1273 SetGitTrace2ParentSid()
1274
883 repo_main, rel_repo_dir = None, None 1275 repo_main, rel_repo_dir = None, None
884 # Don't use the local repo copy, make sure to switch to the gitc client first. 1276 # Don't use the local repo copy, make sure to switch to the gitc client first.
885 if cmd != 'gitc-init': 1277 if cmd != 'gitc-init':
@@ -896,10 +1288,17 @@ def main(orig_args):
896 file=sys.stderr) 1288 file=sys.stderr)
897 sys.exit(1) 1289 sys.exit(1)
898 if not repo_main: 1290 if not repo_main:
1291 # Only expand aliases here since we'll be parsing the CLI ourselves.
1292 # If we had repo_main, alias expansion would happen in main.py.
1293 cmd, alias_args = _ExpandAlias(cmd)
1294 args = alias_args + args
1295
899 if opt.help: 1296 if opt.help:
900 _Usage() 1297 _Usage()
901 if cmd == 'help': 1298 if cmd == 'help':
902 _Help(args) 1299 _Help(args)
1300 if opt.version or cmd == 'version':
1301 _Version()
903 if not cmd: 1302 if not cmd:
904 _NotInstalled() 1303 _NotInstalled()
905 if cmd == 'init' or cmd == 'gitc-init': 1304 if cmd == 'init' or cmd == 'gitc-init':
@@ -920,6 +1319,14 @@ def main(orig_args):
920 if my_main: 1319 if my_main:
921 repo_main = my_main 1320 repo_main = my_main
922 1321
1322 if not repo_main:
1323 print("fatal: unable to find repo entry point", file=sys.stderr)
1324 sys.exit(1)
1325
1326 reqs = Requirements.from_dir(os.path.dirname(repo_main))
1327 if reqs:
1328 reqs.assert_all()
1329
923 ver_str = '.'.join(map(str, VERSION)) 1330 ver_str = '.'.join(map(str, VERSION))
924 me = [sys.executable, repo_main, 1331 me = [sys.executable, repo_main,
925 '--repo-dir=%s' % rel_repo_dir, 1332 '--repo-dir=%s' % rel_repo_dir,
@@ -927,16 +1334,9 @@ def main(orig_args):
927 '--wrapper-path=%s' % wrapper_path, 1334 '--wrapper-path=%s' % wrapper_path,
928 '--'] 1335 '--']
929 me.extend(orig_args) 1336 me.extend(orig_args)
930 me.extend(extra_args) 1337 exec_command(me)
931 try: 1338 print("fatal: unable to start %s" % repo_main, file=sys.stderr)
932 if platform.system() == "Windows": 1339 sys.exit(148)
933 sys.exit(subprocess.call(me))
934 else:
935 os.execv(sys.executable, me)
936 except OSError as e:
937 print("fatal: unable to start %s" % repo_main, file=sys.stderr)
938 print("fatal: %s" % e, file=sys.stderr)
939 sys.exit(148)
940 1340
941 1341
942if __name__ == '__main__': 1342if __name__ == '__main__':