summaryrefslogtreecommitdiffstats
path: root/project.py
diff options
context:
space:
mode:
Diffstat (limited to 'project.py')
-rw-r--r--project.py273
1 files changed, 181 insertions, 92 deletions
diff --git a/project.py b/project.py
index 142258e4..83dcf551 100644
--- a/project.py
+++ b/project.py
@@ -35,6 +35,7 @@ from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
35from error import GitError, HookError, UploadError, DownloadError 35from error import GitError, HookError, UploadError, DownloadError
36from error import ManifestInvalidRevisionError 36from error import ManifestInvalidRevisionError
37from error import NoManifestException 37from error import NoManifestException
38import platform_utils
38from trace import IsTrace, Trace 39from trace import IsTrace, Trace
39 40
40from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M 41from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
@@ -62,9 +63,9 @@ def _lwrite(path, content):
62 fd.close() 63 fd.close()
63 64
64 try: 65 try:
65 os.rename(lock, path) 66 platform_utils.rename(lock, path)
66 except OSError: 67 except OSError:
67 os.remove(lock) 68 platform_utils.remove(lock)
68 raise 69 raise
69 70
70 71
@@ -102,7 +103,7 @@ def _ProjectHooks():
102 """ 103 """
103 global _project_hook_list 104 global _project_hook_list
104 if _project_hook_list is None: 105 if _project_hook_list is None:
105 d = os.path.realpath(os.path.abspath(os.path.dirname(__file__))) 106 d = platform_utils.realpath(os.path.abspath(os.path.dirname(__file__)))
106 d = os.path.join(d, 'hooks') 107 d = os.path.join(d, 'hooks')
107 _project_hook_list = [os.path.join(d, x) for x in os.listdir(d)] 108 _project_hook_list = [os.path.join(d, x) for x in os.listdir(d)]
108 return _project_hook_list 109 return _project_hook_list
@@ -176,12 +177,20 @@ class ReviewableBranch(object):
176 def UploadForReview(self, people, 177 def UploadForReview(self, people,
177 auto_topic=False, 178 auto_topic=False,
178 draft=False, 179 draft=False,
179 dest_branch=None): 180 private=False,
181 wip=False,
182 dest_branch=None,
183 validate_certs=True,
184 push_options=None):
180 self.project.UploadForReview(self.name, 185 self.project.UploadForReview(self.name,
181 people, 186 people,
182 auto_topic=auto_topic, 187 auto_topic=auto_topic,
183 draft=draft, 188 draft=draft,
184 dest_branch=dest_branch) 189 private=private,
190 wip=wip,
191 dest_branch=dest_branch,
192 validate_certs=validate_certs,
193 push_options=push_options)
185 194
186 def GetPublishedRefs(self): 195 def GetPublishedRefs(self):
187 refs = {} 196 refs = {}
@@ -243,7 +252,7 @@ class _CopyFile(object):
243 try: 252 try:
244 # remove existing file first, since it might be read-only 253 # remove existing file first, since it might be read-only
245 if os.path.exists(dest): 254 if os.path.exists(dest):
246 os.remove(dest) 255 platform_utils.remove(dest)
247 else: 256 else:
248 dest_dir = os.path.dirname(dest) 257 dest_dir = os.path.dirname(dest)
249 if not os.path.isdir(dest_dir): 258 if not os.path.isdir(dest_dir):
@@ -268,16 +277,16 @@ class _LinkFile(object):
268 277
269 def __linkIt(self, relSrc, absDest): 278 def __linkIt(self, relSrc, absDest):
270 # link file if it does not exist or is out of date 279 # link file if it does not exist or is out of date
271 if not os.path.islink(absDest) or (os.readlink(absDest) != relSrc): 280 if not platform_utils.islink(absDest) or (platform_utils.readlink(absDest) != relSrc):
272 try: 281 try:
273 # remove existing file first, since it might be read-only 282 # remove existing file first, since it might be read-only
274 if os.path.lexists(absDest): 283 if os.path.lexists(absDest):
275 os.remove(absDest) 284 platform_utils.remove(absDest)
276 else: 285 else:
277 dest_dir = os.path.dirname(absDest) 286 dest_dir = os.path.dirname(absDest)
278 if not os.path.isdir(dest_dir): 287 if not os.path.isdir(dest_dir):
279 os.makedirs(dest_dir) 288 os.makedirs(dest_dir)
280 os.symlink(relSrc, absDest) 289 platform_utils.symlink(relSrc, absDest)
281 except IOError: 290 except IOError:
282 _error('Cannot link file %s to %s', relSrc, absDest) 291 _error('Cannot link file %s to %s', relSrc, absDest)
283 292
@@ -323,13 +332,15 @@ class RemoteSpec(object):
323 pushUrl=None, 332 pushUrl=None,
324 review=None, 333 review=None,
325 revision=None, 334 revision=None,
326 orig_name=None): 335 orig_name=None,
336 fetchUrl=None):
327 self.name = name 337 self.name = name
328 self.url = url 338 self.url = url
329 self.pushUrl = pushUrl 339 self.pushUrl = pushUrl
330 self.review = review 340 self.review = review
331 self.revision = revision 341 self.revision = revision
332 self.orig_name = orig_name 342 self.orig_name = orig_name
343 self.fetchUrl = fetchUrl
333 344
334 345
335class RepoHook(object): 346class RepoHook(object):
@@ -687,7 +698,7 @@ class Project(object):
687 self.gitdir = gitdir.replace('\\', '/') 698 self.gitdir = gitdir.replace('\\', '/')
688 self.objdir = objdir.replace('\\', '/') 699 self.objdir = objdir.replace('\\', '/')
689 if worktree: 700 if worktree:
690 self.worktree = os.path.normpath(worktree.replace('\\', '/')) 701 self.worktree = os.path.normpath(worktree).replace('\\', '/')
691 else: 702 else:
692 self.worktree = None 703 self.worktree = None
693 self.relpath = relpath 704 self.relpath = relpath
@@ -911,11 +922,13 @@ class Project(object):
911 else: 922 else:
912 return False 923 return False
913 924
914 def PrintWorkTreeStatus(self, output_redir=None): 925 def PrintWorkTreeStatus(self, output_redir=None, quiet=False):
915 """Prints the status of the repository to stdout. 926 """Prints the status of the repository to stdout.
916 927
917 Args: 928 Args:
918 output: If specified, redirect the output to this object. 929 output: If specified, redirect the output to this object.
930 quiet: If True then only print the project name. Do not print
931 the modified files, branch name, etc.
919 """ 932 """
920 if not os.path.isdir(self.worktree): 933 if not os.path.isdir(self.worktree):
921 if output_redir is None: 934 if output_redir is None:
@@ -941,6 +954,10 @@ class Project(object):
941 out.redirect(output_redir) 954 out.redirect(output_redir)
942 out.project('project %-40s', self.relpath + '/ ') 955 out.project('project %-40s', self.relpath + '/ ')
943 956
957 if quiet:
958 out.nl()
959 return 'DIRTY'
960
944 branch = self.CurrentBranch 961 branch = self.CurrentBranch
945 if branch is None: 962 if branch is None:
946 out.nobranch('(*** NO BRANCH ***)') 963 out.nobranch('(*** NO BRANCH ***)')
@@ -1099,7 +1116,11 @@ class Project(object):
1099 people=([], []), 1116 people=([], []),
1100 auto_topic=False, 1117 auto_topic=False,
1101 draft=False, 1118 draft=False,
1102 dest_branch=None): 1119 private=False,
1120 wip=False,
1121 dest_branch=None,
1122 validate_certs=True,
1123 push_options=None):
1103 """Uploads the named branch for code review. 1124 """Uploads the named branch for code review.
1104 """ 1125 """
1105 if branch is None: 1126 if branch is None:
@@ -1124,7 +1145,7 @@ class Project(object):
1124 branch.remote.projectname = self.name 1145 branch.remote.projectname = self.name
1125 branch.remote.Save() 1146 branch.remote.Save()
1126 1147
1127 url = branch.remote.ReviewUrl(self.UserEmail) 1148 url = branch.remote.ReviewUrl(self.UserEmail, validate_certs)
1128 if url is None: 1149 if url is None:
1129 raise UploadError('review not configured') 1150 raise UploadError('review not configured')
1130 cmd = ['push'] 1151 cmd = ['push']
@@ -1137,6 +1158,10 @@ class Project(object):
1137 rp.append('--cc=%s' % sq(e)) 1158 rp.append('--cc=%s' % sq(e))
1138 cmd.append('--receive-pack=%s' % " ".join(rp)) 1159 cmd.append('--receive-pack=%s' % " ".join(rp))
1139 1160
1161 for push_option in (push_options or []):
1162 cmd.append('-o')
1163 cmd.append(push_option)
1164
1140 cmd.append(url) 1165 cmd.append(url)
1141 1166
1142 if dest_branch.startswith(R_HEADS): 1167 if dest_branch.startswith(R_HEADS):
@@ -1150,9 +1175,14 @@ class Project(object):
1150 dest_branch) 1175 dest_branch)
1151 if auto_topic: 1176 if auto_topic:
1152 ref_spec = ref_spec + '/' + branch.name 1177 ref_spec = ref_spec + '/' + branch.name
1178
1153 if not url.startswith('ssh://'): 1179 if not url.startswith('ssh://'):
1154 rp = ['r=%s' % p for p in people[0]] + \ 1180 rp = ['r=%s' % p for p in people[0]] + \
1155 ['cc=%s' % p for p in people[1]] 1181 ['cc=%s' % p for p in people[1]]
1182 if private:
1183 rp = rp + ['private']
1184 if wip:
1185 rp = rp + ['wip']
1156 if rp: 1186 if rp:
1157 ref_spec = ref_spec + '%' + ','.join(rp) 1187 ref_spec = ref_spec + '%' + ','.join(rp)
1158 cmd.append(ref_spec) 1188 cmd.append(ref_spec)
@@ -1192,7 +1222,8 @@ class Project(object):
1192 no_tags=False, 1222 no_tags=False,
1193 archive=False, 1223 archive=False,
1194 optimized_fetch=False, 1224 optimized_fetch=False,
1195 prune=False): 1225 prune=False,
1226 submodules=False):
1196 """Perform only the network IO portion of the sync process. 1227 """Perform only the network IO portion of the sync process.
1197 Local working directory/branch state is not affected. 1228 Local working directory/branch state is not affected.
1198 """ 1229 """
@@ -1218,7 +1249,7 @@ class Project(object):
1218 if not self._ExtractArchive(tarpath, path=topdir): 1249 if not self._ExtractArchive(tarpath, path=topdir):
1219 return False 1250 return False
1220 try: 1251 try:
1221 os.remove(tarpath) 1252 platform_utils.remove(tarpath)
1222 except OSError as e: 1253 except OSError as e:
1223 _warn("Cannot remove archive %s: %s", tarpath, str(e)) 1254 _warn("Cannot remove archive %s: %s", tarpath, str(e))
1224 self._CopyAndLinkFiles() 1255 self._CopyAndLinkFiles()
@@ -1234,7 +1265,7 @@ class Project(object):
1234 if is_new: 1265 if is_new:
1235 alt = os.path.join(self.gitdir, 'objects/info/alternates') 1266 alt = os.path.join(self.gitdir, 'objects/info/alternates')
1236 try: 1267 try:
1237 fd = open(alt, 'rb') 1268 fd = open(alt)
1238 try: 1269 try:
1239 alt_dir = fd.readline().rstrip() 1270 alt_dir = fd.readline().rstrip()
1240 finally: 1271 finally:
@@ -1258,13 +1289,19 @@ class Project(object):
1258 elif self.manifest.default.sync_c: 1289 elif self.manifest.default.sync_c:
1259 current_branch_only = True 1290 current_branch_only = True
1260 1291
1292 if self.clone_depth:
1293 depth = self.clone_depth
1294 else:
1295 depth = self.manifest.manifestProject.config.GetString('repo.depth')
1296
1261 need_to_fetch = not (optimized_fetch and 1297 need_to_fetch = not (optimized_fetch and
1262 (ID_RE.match(self.revisionExpr) and 1298 (ID_RE.match(self.revisionExpr) and
1263 self._CheckForSha1())) 1299 self._CheckForImmutableRevision()))
1264 if (need_to_fetch and 1300 if (need_to_fetch and
1265 not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir, 1301 not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
1266 current_branch_only=current_branch_only, 1302 current_branch_only=current_branch_only,
1267 no_tags=no_tags, prune=prune)): 1303 no_tags=no_tags, prune=prune, depth=depth,
1304 submodules=submodules)):
1268 return False 1305 return False
1269 1306
1270 if self.worktree: 1307 if self.worktree:
@@ -1272,7 +1309,7 @@ class Project(object):
1272 else: 1309 else:
1273 self._InitMirrorHead() 1310 self._InitMirrorHead()
1274 try: 1311 try:
1275 os.remove(os.path.join(self.gitdir, 'FETCH_HEAD')) 1312 platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'))
1276 except OSError: 1313 except OSError:
1277 pass 1314 pass
1278 return True 1315 return True
@@ -1320,11 +1357,11 @@ class Project(object):
1320 raise ManifestInvalidRevisionError('revision %s in %s not found' % 1357 raise ManifestInvalidRevisionError('revision %s in %s not found' %
1321 (self.revisionExpr, self.name)) 1358 (self.revisionExpr, self.name))
1322 1359
1323 def Sync_LocalHalf(self, syncbuf, force_sync=False): 1360 def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False):
1324 """Perform only the local IO portion of the sync process. 1361 """Perform only the local IO portion of the sync process.
1325 Network access is not required. 1362 Network access is not required.
1326 """ 1363 """
1327 self._InitWorkTree(force_sync=force_sync) 1364 self._InitWorkTree(force_sync=force_sync, submodules=submodules)
1328 all_refs = self.bare_ref.all 1365 all_refs = self.bare_ref.all
1329 self.CleanPublishedCache(all_refs) 1366 self.CleanPublishedCache(all_refs)
1330 revid = self.GetRevisionId(all_refs) 1367 revid = self.GetRevisionId(all_refs)
@@ -1333,6 +1370,9 @@ class Project(object):
1333 self._FastForward(revid) 1370 self._FastForward(revid)
1334 self._CopyAndLinkFiles() 1371 self._CopyAndLinkFiles()
1335 1372
1373 def _dosubmodules():
1374 self._SyncSubmodules(quiet=True)
1375
1336 head = self.work_git.GetHead() 1376 head = self.work_git.GetHead()
1337 if head.startswith(R_HEADS): 1377 if head.startswith(R_HEADS):
1338 branch = head[len(R_HEADS):] 1378 branch = head[len(R_HEADS):]
@@ -1366,6 +1406,8 @@ class Project(object):
1366 1406
1367 try: 1407 try:
1368 self._Checkout(revid, quiet=True) 1408 self._Checkout(revid, quiet=True)
1409 if submodules:
1410 self._SyncSubmodules(quiet=True)
1369 except GitError as e: 1411 except GitError as e:
1370 syncbuf.fail(self, e) 1412 syncbuf.fail(self, e)
1371 return 1413 return
@@ -1390,6 +1432,8 @@ class Project(object):
1390 branch.name) 1432 branch.name)
1391 try: 1433 try:
1392 self._Checkout(revid, quiet=True) 1434 self._Checkout(revid, quiet=True)
1435 if submodules:
1436 self._SyncSubmodules(quiet=True)
1393 except GitError as e: 1437 except GitError as e:
1394 syncbuf.fail(self, e) 1438 syncbuf.fail(self, e)
1395 return 1439 return
@@ -1415,6 +1459,8 @@ class Project(object):
1415 # strict subset. We can fast-forward safely. 1459 # strict subset. We can fast-forward safely.
1416 # 1460 #
1417 syncbuf.later1(self, _doff) 1461 syncbuf.later1(self, _doff)
1462 if submodules:
1463 syncbuf.later1(self, _dosubmodules)
1418 return 1464 return
1419 1465
1420 # Examine the local commits not in the remote. Find the 1466 # Examine the local commits not in the remote. Find the
@@ -1466,19 +1512,28 @@ class Project(object):
1466 branch.Save() 1512 branch.Save()
1467 1513
1468 if cnt_mine > 0 and self.rebase: 1514 if cnt_mine > 0 and self.rebase:
1515 def _docopyandlink():
1516 self._CopyAndLinkFiles()
1517
1469 def _dorebase(): 1518 def _dorebase():
1470 self._Rebase(upstream='%s^1' % last_mine, onto=revid) 1519 self._Rebase(upstream='%s^1' % last_mine, onto=revid)
1471 self._CopyAndLinkFiles()
1472 syncbuf.later2(self, _dorebase) 1520 syncbuf.later2(self, _dorebase)
1521 if submodules:
1522 syncbuf.later2(self, _dosubmodules)
1523 syncbuf.later2(self, _docopyandlink)
1473 elif local_changes: 1524 elif local_changes:
1474 try: 1525 try:
1475 self._ResetHard(revid) 1526 self._ResetHard(revid)
1527 if submodules:
1528 self._SyncSubmodules(quiet=True)
1476 self._CopyAndLinkFiles() 1529 self._CopyAndLinkFiles()
1477 except GitError as e: 1530 except GitError as e:
1478 syncbuf.fail(self, e) 1531 syncbuf.fail(self, e)
1479 return 1532 return
1480 else: 1533 else:
1481 syncbuf.later1(self, _doff) 1534 syncbuf.later1(self, _doff)
1535 if submodules:
1536 syncbuf.later1(self, _dosubmodules)
1482 1537
1483 def AddCopyFile(self, src, dest, absdest): 1538 def AddCopyFile(self, src, dest, absdest):
1484 # dest should already be an absolute path, but src is project relative 1539 # dest should already be an absolute path, but src is project relative
@@ -1764,7 +1819,7 @@ class Project(object):
1764 except GitError: 1819 except GitError:
1765 return [], [] 1820 return [], []
1766 finally: 1821 finally:
1767 os.remove(temp_gitmodules_path) 1822 platform_utils.remove(temp_gitmodules_path)
1768 1823
1769 names = set() 1824 names = set()
1770 paths = {} 1825 paths = {}
@@ -1851,7 +1906,7 @@ class Project(object):
1851 1906
1852 1907
1853# Direct Git Commands ## 1908# Direct Git Commands ##
1854 def _CheckForSha1(self): 1909 def _CheckForImmutableRevision(self):
1855 try: 1910 try:
1856 # if revision (sha or tag) is not present then following function 1911 # if revision (sha or tag) is not present then following function
1857 # throws an error. 1912 # throws an error.
@@ -1880,23 +1935,18 @@ class Project(object):
1880 quiet=False, 1935 quiet=False,
1881 alt_dir=None, 1936 alt_dir=None,
1882 no_tags=False, 1937 no_tags=False,
1883 prune=False): 1938 prune=False,
1939 depth=None,
1940 submodules=False):
1884 1941
1885 is_sha1 = False 1942 is_sha1 = False
1886 tag_name = None 1943 tag_name = None
1887 depth = None
1888
1889 # The depth should not be used when fetching to a mirror because 1944 # The depth should not be used when fetching to a mirror because
1890 # it will result in a shallow repository that cannot be cloned or 1945 # it will result in a shallow repository that cannot be cloned or
1891 # fetched from. 1946 # fetched from.
1892 if not self.manifest.IsMirror: 1947 # The repo project should also never be synced with partial depth.
1893 if self.clone_depth: 1948 if self.manifest.IsMirror or self.relpath == '.repo/repo':
1894 depth = self.clone_depth 1949 depth = None
1895 else:
1896 depth = self.manifest.manifestProject.config.GetString('repo.depth')
1897 # The repo project should never be synced with partial depth
1898 if self.relpath == '.repo/repo':
1899 depth = None
1900 1950
1901 if depth: 1951 if depth:
1902 current_branch_only = True 1952 current_branch_only = True
@@ -1910,7 +1960,9 @@ class Project(object):
1910 tag_name = self.revisionExpr[len(R_TAGS):] 1960 tag_name = self.revisionExpr[len(R_TAGS):]
1911 1961
1912 if is_sha1 or tag_name is not None: 1962 if is_sha1 or tag_name is not None:
1913 if self._CheckForSha1(): 1963 if self._CheckForImmutableRevision():
1964 print('Skipped fetching project %s (already have persistent ref)'
1965 % self.name)
1914 return True 1966 return True
1915 if is_sha1 and not depth: 1967 if is_sha1 and not depth:
1916 # When syncing a specific commit and --depth is not set: 1968 # When syncing a specific commit and --depth is not set:
@@ -1958,15 +2010,17 @@ class Project(object):
1958 ids.add(ref_id) 2010 ids.add(ref_id)
1959 tmp.add(r) 2011 tmp.add(r)
1960 2012
1961 tmp_packed = '' 2013 tmp_packed_lines = []
1962 old_packed = '' 2014 old_packed_lines = []
1963 2015
1964 for r in sorted(all_refs): 2016 for r in sorted(all_refs):
1965 line = '%s %s\n' % (all_refs[r], r) 2017 line = '%s %s\n' % (all_refs[r], r)
1966 tmp_packed += line 2018 tmp_packed_lines.append(line)
1967 if r not in tmp: 2019 if r not in tmp:
1968 old_packed += line 2020 old_packed_lines.append(line)
1969 2021
2022 tmp_packed = ''.join(tmp_packed_lines)
2023 old_packed = ''.join(old_packed_lines)
1970 _lwrite(packed_refs, tmp_packed) 2024 _lwrite(packed_refs, tmp_packed)
1971 else: 2025 else:
1972 alt_dir = None 2026 alt_dir = None
@@ -1999,6 +2053,9 @@ class Project(object):
1999 if prune: 2053 if prune:
2000 cmd.append('--prune') 2054 cmd.append('--prune')
2001 2055
2056 if submodules:
2057 cmd.append('--recurse-submodules=on-demand')
2058
2002 spec = [] 2059 spec = []
2003 if not current_branch_only: 2060 if not current_branch_only:
2004 # Fetch whole repo 2061 # Fetch whole repo
@@ -2054,24 +2111,25 @@ class Project(object):
2054 if old_packed != '': 2111 if old_packed != '':
2055 _lwrite(packed_refs, old_packed) 2112 _lwrite(packed_refs, old_packed)
2056 else: 2113 else:
2057 os.remove(packed_refs) 2114 platform_utils.remove(packed_refs)
2058 self.bare_git.pack_refs('--all', '--prune') 2115 self.bare_git.pack_refs('--all', '--prune')
2059 2116
2060 if is_sha1 and current_branch_only and self.upstream: 2117 if is_sha1 and current_branch_only:
2061 # We just synced the upstream given branch; verify we 2118 # We just synced the upstream given branch; verify we
2062 # got what we wanted, else trigger a second run of all 2119 # got what we wanted, else trigger a second run of all
2063 # refs. 2120 # refs.
2064 if not self._CheckForSha1(): 2121 if not self._CheckForImmutableRevision():
2065 if not depth: 2122 if current_branch_only and depth:
2066 # Avoid infinite recursion when depth is True (since depth implies 2123 # Sync the current branch only with depth set to None
2067 # current_branch_only)
2068 return self._RemoteFetch(name=name, current_branch_only=False,
2069 initial=False, quiet=quiet, alt_dir=alt_dir)
2070 if self.clone_depth:
2071 self.clone_depth = None
2072 return self._RemoteFetch(name=name, 2124 return self._RemoteFetch(name=name,
2073 current_branch_only=current_branch_only, 2125 current_branch_only=current_branch_only,
2074 initial=False, quiet=quiet, alt_dir=alt_dir) 2126 initial=False, quiet=quiet, alt_dir=alt_dir,
2127 depth=None)
2128 else:
2129 # Avoid infinite recursion: sync all branches with depth set to None
2130 return self._RemoteFetch(name=name, current_branch_only=False,
2131 initial=False, quiet=quiet, alt_dir=alt_dir,
2132 depth=None)
2075 2133
2076 return ok 2134 return ok
2077 2135
@@ -2115,14 +2173,14 @@ class Project(object):
2115 2173
2116 ok = GitCommand(self, cmd, bare=True).Wait() == 0 2174 ok = GitCommand(self, cmd, bare=True).Wait() == 0
2117 if os.path.exists(bundle_dst): 2175 if os.path.exists(bundle_dst):
2118 os.remove(bundle_dst) 2176 platform_utils.remove(bundle_dst)
2119 if os.path.exists(bundle_tmp): 2177 if os.path.exists(bundle_tmp):
2120 os.remove(bundle_tmp) 2178 platform_utils.remove(bundle_tmp)
2121 return ok 2179 return ok
2122 2180
2123 def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet): 2181 def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
2124 if os.path.exists(dstPath): 2182 if os.path.exists(dstPath):
2125 os.remove(dstPath) 2183 platform_utils.remove(dstPath)
2126 2184
2127 cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location'] 2185 cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
2128 if quiet: 2186 if quiet:
@@ -2132,7 +2190,7 @@ class Project(object):
2132 if size >= 1024: 2190 if size >= 1024:
2133 cmd += ['--continue-at', '%d' % (size,)] 2191 cmd += ['--continue-at', '%d' % (size,)]
2134 else: 2192 else:
2135 os.remove(tmpPath) 2193 platform_utils.remove(tmpPath)
2136 if 'http_proxy' in os.environ and 'darwin' == sys.platform: 2194 if 'http_proxy' in os.environ and 'darwin' == sys.platform:
2137 cmd += ['--proxy', os.environ['http_proxy']] 2195 cmd += ['--proxy', os.environ['http_proxy']]
2138 with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, _proxy): 2196 with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, _proxy):
@@ -2163,10 +2221,10 @@ class Project(object):
2163 2221
2164 if os.path.exists(tmpPath): 2222 if os.path.exists(tmpPath):
2165 if curlret == 0 and self._IsValidBundle(tmpPath, quiet): 2223 if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
2166 os.rename(tmpPath, dstPath) 2224 platform_utils.rename(tmpPath, dstPath)
2167 return True 2225 return True
2168 else: 2226 else:
2169 os.remove(tmpPath) 2227 platform_utils.remove(tmpPath)
2170 return False 2228 return False
2171 else: 2229 else:
2172 return False 2230 return False
@@ -2218,6 +2276,13 @@ class Project(object):
2218 if GitCommand(self, cmd).Wait() != 0: 2276 if GitCommand(self, cmd).Wait() != 0:
2219 raise GitError('%s reset --hard %s ' % (self.name, rev)) 2277 raise GitError('%s reset --hard %s ' % (self.name, rev))
2220 2278
2279 def _SyncSubmodules(self, quiet=True):
2280 cmd = ['submodule', 'update', '--init', '--recursive']
2281 if quiet:
2282 cmd.append('-q')
2283 if GitCommand(self, cmd).Wait() != 0:
2284 raise GitError('%s submodule update --init --recursive %s ' % self.name)
2285
2221 def _Rebase(self, upstream, onto=None): 2286 def _Rebase(self, upstream, onto=None):
2222 cmd = ['rebase'] 2287 cmd = ['rebase']
2223 if onto is not None: 2288 if onto is not None:
@@ -2257,10 +2322,10 @@ class Project(object):
2257 print("Retrying clone after deleting %s" % 2322 print("Retrying clone after deleting %s" %
2258 self.gitdir, file=sys.stderr) 2323 self.gitdir, file=sys.stderr)
2259 try: 2324 try:
2260 shutil.rmtree(os.path.realpath(self.gitdir)) 2325 platform_utils.rmtree(platform_utils.realpath(self.gitdir))
2261 if self.worktree and os.path.exists(os.path.realpath 2326 if self.worktree and os.path.exists(platform_utils.realpath
2262 (self.worktree)): 2327 (self.worktree)):
2263 shutil.rmtree(os.path.realpath(self.worktree)) 2328 platform_utils.rmtree(platform_utils.realpath(self.worktree))
2264 return self._InitGitDir(mirror_git=mirror_git, force_sync=False) 2329 return self._InitGitDir(mirror_git=mirror_git, force_sync=False)
2265 except: 2330 except:
2266 raise e 2331 raise e
@@ -2302,9 +2367,9 @@ class Project(object):
2302 self.config.SetString('core.bare', None) 2367 self.config.SetString('core.bare', None)
2303 except Exception: 2368 except Exception:
2304 if init_obj_dir and os.path.exists(self.objdir): 2369 if init_obj_dir and os.path.exists(self.objdir):
2305 shutil.rmtree(self.objdir) 2370 platform_utils.rmtree(self.objdir)
2306 if init_git_dir and os.path.exists(self.gitdir): 2371 if init_git_dir and os.path.exists(self.gitdir):
2307 shutil.rmtree(self.gitdir) 2372 platform_utils.rmtree(self.gitdir)
2308 raise 2373 raise
2309 2374
2310 def _UpdateHooks(self): 2375 def _UpdateHooks(self):
@@ -2312,7 +2377,7 @@ class Project(object):
2312 self._InitHooks() 2377 self._InitHooks()
2313 2378
2314 def _InitHooks(self): 2379 def _InitHooks(self):
2315 hooks = os.path.realpath(self._gitdir_path('hooks')) 2380 hooks = platform_utils.realpath(self._gitdir_path('hooks'))
2316 if not os.path.exists(hooks): 2381 if not os.path.exists(hooks):
2317 os.makedirs(hooks) 2382 os.makedirs(hooks)
2318 for stock_hook in _ProjectHooks(): 2383 for stock_hook in _ProjectHooks():
@@ -2328,20 +2393,21 @@ class Project(object):
2328 continue 2393 continue
2329 2394
2330 dst = os.path.join(hooks, name) 2395 dst = os.path.join(hooks, name)
2331 if os.path.islink(dst): 2396 if platform_utils.islink(dst):
2332 continue 2397 continue
2333 if os.path.exists(dst): 2398 if os.path.exists(dst):
2334 if filecmp.cmp(stock_hook, dst, shallow=False): 2399 if filecmp.cmp(stock_hook, dst, shallow=False):
2335 os.remove(dst) 2400 platform_utils.remove(dst)
2336 else: 2401 else:
2337 _warn("%s: Not replacing locally modified %s hook", 2402 _warn("%s: Not replacing locally modified %s hook",
2338 self.relpath, name) 2403 self.relpath, name)
2339 continue 2404 continue
2340 try: 2405 try:
2341 os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst) 2406 platform_utils.symlink(
2407 os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
2342 except OSError as e: 2408 except OSError as e:
2343 if e.errno == errno.EPERM: 2409 if e.errno == errno.EPERM:
2344 raise GitError('filesystem must support symlinks') 2410 raise GitError(self._get_symlink_error_message())
2345 else: 2411 else:
2346 raise 2412 raise
2347 2413
@@ -2389,11 +2455,12 @@ class Project(object):
2389 symlink_dirs += self.working_tree_dirs 2455 symlink_dirs += self.working_tree_dirs
2390 to_symlink = symlink_files + symlink_dirs 2456 to_symlink = symlink_files + symlink_dirs
2391 for name in set(to_symlink): 2457 for name in set(to_symlink):
2392 dst = os.path.realpath(os.path.join(destdir, name)) 2458 dst = platform_utils.realpath(os.path.join(destdir, name))
2393 if os.path.lexists(dst): 2459 if os.path.lexists(dst):
2394 src = os.path.realpath(os.path.join(srcdir, name)) 2460 src = platform_utils.realpath(os.path.join(srcdir, name))
2395 # Fail if the links are pointing to the wrong place 2461 # Fail if the links are pointing to the wrong place
2396 if src != dst: 2462 if src != dst:
2463 _error('%s is different in %s vs %s', name, destdir, srcdir)
2397 raise GitError('--force-sync not enabled; cannot overwrite a local ' 2464 raise GitError('--force-sync not enabled; cannot overwrite a local '
2398 'work tree. If you\'re comfortable with the ' 2465 'work tree. If you\'re comfortable with the '
2399 'possibility of losing the work tree\'s git metadata,' 2466 'possibility of losing the work tree\'s git metadata,'
@@ -2422,10 +2489,10 @@ class Project(object):
2422 if copy_all: 2489 if copy_all:
2423 to_copy = os.listdir(gitdir) 2490 to_copy = os.listdir(gitdir)
2424 2491
2425 dotgit = os.path.realpath(dotgit) 2492 dotgit = platform_utils.realpath(dotgit)
2426 for name in set(to_copy).union(to_symlink): 2493 for name in set(to_copy).union(to_symlink):
2427 try: 2494 try:
2428 src = os.path.realpath(os.path.join(gitdir, name)) 2495 src = platform_utils.realpath(os.path.join(gitdir, name))
2429 dst = os.path.join(dotgit, name) 2496 dst = os.path.join(dotgit, name)
2430 2497
2431 if os.path.lexists(dst): 2498 if os.path.lexists(dst):
@@ -2435,28 +2502,30 @@ class Project(object):
2435 if name in symlink_dirs and not os.path.lexists(src): 2502 if name in symlink_dirs and not os.path.lexists(src):
2436 os.makedirs(src) 2503 os.makedirs(src)
2437 2504
2505 if name in to_symlink:
2506 platform_utils.symlink(
2507 os.path.relpath(src, os.path.dirname(dst)), dst)
2508 elif copy_all and not platform_utils.islink(dst):
2509 if os.path.isdir(src):
2510 shutil.copytree(src, dst)
2511 elif os.path.isfile(src):
2512 shutil.copy(src, dst)
2513
2438 # If the source file doesn't exist, ensure the destination 2514 # If the source file doesn't exist, ensure the destination
2439 # file doesn't either. 2515 # file doesn't either.
2440 if name in symlink_files and not os.path.lexists(src): 2516 if name in symlink_files and not os.path.lexists(src):
2441 try: 2517 try:
2442 os.remove(dst) 2518 platform_utils.remove(dst)
2443 except OSError: 2519 except OSError:
2444 pass 2520 pass
2445 2521
2446 if name in to_symlink:
2447 os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
2448 elif copy_all and not os.path.islink(dst):
2449 if os.path.isdir(src):
2450 shutil.copytree(src, dst)
2451 elif os.path.isfile(src):
2452 shutil.copy(src, dst)
2453 except OSError as e: 2522 except OSError as e:
2454 if e.errno == errno.EPERM: 2523 if e.errno == errno.EPERM:
2455 raise DownloadError('filesystem must support symlinks') 2524 raise DownloadError(self._get_symlink_error_message())
2456 else: 2525 else:
2457 raise 2526 raise
2458 2527
2459 def _InitWorkTree(self, force_sync=False): 2528 def _InitWorkTree(self, force_sync=False, submodules=False):
2460 dotgit = os.path.join(self.worktree, '.git') 2529 dotgit = os.path.join(self.worktree, '.git')
2461 init_dotgit = not os.path.exists(dotgit) 2530 init_dotgit = not os.path.exists(dotgit)
2462 try: 2531 try:
@@ -2470,8 +2539,8 @@ class Project(object):
2470 except GitError as e: 2539 except GitError as e:
2471 if force_sync: 2540 if force_sync:
2472 try: 2541 try:
2473 shutil.rmtree(dotgit) 2542 platform_utils.rmtree(dotgit)
2474 return self._InitWorkTree(force_sync=False) 2543 return self._InitWorkTree(force_sync=False, submodules=submodules)
2475 except: 2544 except:
2476 raise e 2545 raise e
2477 raise e 2546 raise e
@@ -2485,14 +2554,24 @@ class Project(object):
2485 if GitCommand(self, cmd).Wait() != 0: 2554 if GitCommand(self, cmd).Wait() != 0:
2486 raise GitError("cannot initialize work tree") 2555 raise GitError("cannot initialize work tree")
2487 2556
2557 if submodules:
2558 self._SyncSubmodules(quiet=True)
2488 self._CopyAndLinkFiles() 2559 self._CopyAndLinkFiles()
2489 except Exception: 2560 except Exception:
2490 if init_dotgit: 2561 if init_dotgit:
2491 shutil.rmtree(dotgit) 2562 platform_utils.rmtree(dotgit)
2492 raise 2563 raise
2493 2564
2565 def _get_symlink_error_message(self):
2566 if platform_utils.isWindows():
2567 return ('Unable to create symbolic link. Please re-run the command as '
2568 'Administrator, or see '
2569 'https://github.com/git-for-windows/git/wiki/Symbolic-Links '
2570 'for other options.')
2571 return 'filesystem must support symlinks'
2572
2494 def _gitdir_path(self, path): 2573 def _gitdir_path(self, path):
2495 return os.path.realpath(os.path.join(self.gitdir, path)) 2574 return platform_utils.realpath(os.path.join(self.gitdir, path))
2496 2575
2497 def _revlist(self, *args, **kw): 2576 def _revlist(self, *args, **kw):
2498 a = [] 2577 a = []
@@ -2627,11 +2706,11 @@ class Project(object):
2627 else: 2706 else:
2628 path = os.path.join(self._project.worktree, '.git', HEAD) 2707 path = os.path.join(self._project.worktree, '.git', HEAD)
2629 try: 2708 try:
2630 fd = open(path, 'rb') 2709 fd = open(path)
2631 except IOError as e: 2710 except IOError as e:
2632 raise NoManifestException(path, str(e)) 2711 raise NoManifestException(path, str(e))
2633 try: 2712 try:
2634 line = fd.read() 2713 line = fd.readline()
2635 finally: 2714 finally:
2636 fd.close() 2715 fd.close()
2637 try: 2716 try:
@@ -2833,13 +2912,14 @@ class SyncBuffer(object):
2833 2912
2834 self.detach_head = detach_head 2913 self.detach_head = detach_head
2835 self.clean = True 2914 self.clean = True
2915 self.recent_clean = True
2836 2916
2837 def info(self, project, fmt, *args): 2917 def info(self, project, fmt, *args):
2838 self._messages.append(_InfoMessage(project, fmt % args)) 2918 self._messages.append(_InfoMessage(project, fmt % args))
2839 2919
2840 def fail(self, project, err=None): 2920 def fail(self, project, err=None):
2841 self._failures.append(_Failure(project, err)) 2921 self._failures.append(_Failure(project, err))
2842 self.clean = False 2922 self._MarkUnclean()
2843 2923
2844 def later1(self, project, what): 2924 def later1(self, project, what):
2845 self._later_queue1.append(_Later(project, what)) 2925 self._later_queue1.append(_Later(project, what))
@@ -2853,6 +2933,15 @@ class SyncBuffer(object):
2853 self._PrintMessages() 2933 self._PrintMessages()
2854 return self.clean 2934 return self.clean
2855 2935
2936 def Recently(self):
2937 recent_clean = self.recent_clean
2938 self.recent_clean = True
2939 return recent_clean
2940
2941 def _MarkUnclean(self):
2942 self.clean = False
2943 self.recent_clean = False
2944
2856 def _RunLater(self): 2945 def _RunLater(self):
2857 for q in ['_later_queue1', '_later_queue2']: 2946 for q in ['_later_queue1', '_later_queue2']:
2858 if not self._RunQueue(q): 2947 if not self._RunQueue(q):
@@ -2861,7 +2950,7 @@ class SyncBuffer(object):
2861 def _RunQueue(self, queue): 2950 def _RunQueue(self, queue):
2862 for m in getattr(self, queue): 2951 for m in getattr(self, queue):
2863 if not m.Run(self): 2952 if not m.Run(self):
2864 self.clean = False 2953 self._MarkUnclean()
2865 return False 2954 return False
2866 setattr(self, queue, []) 2955 setattr(self, queue, [])
2867 return True 2956 return True
@@ -2903,14 +2992,14 @@ class MetaProject(Project):
2903 self.revisionExpr = base 2992 self.revisionExpr = base
2904 self.revisionId = None 2993 self.revisionId = None
2905 2994
2906 def MetaBranchSwitch(self): 2995 def MetaBranchSwitch(self, submodules=False):
2907 """ Prepare MetaProject for manifest branch switch 2996 """ Prepare MetaProject for manifest branch switch
2908 """ 2997 """
2909 2998
2910 # detach and delete manifest branch, allowing a new 2999 # detach and delete manifest branch, allowing a new
2911 # branch to take over 3000 # branch to take over
2912 syncbuf = SyncBuffer(self.config, detach_head=True) 3001 syncbuf = SyncBuffer(self.config, detach_head=True)
2913 self.Sync_LocalHalf(syncbuf) 3002 self.Sync_LocalHalf(syncbuf, submodules=submodules)
2914 syncbuf.Finish() 3003 syncbuf.Finish()
2915 3004
2916 return GitCommand(self, 3005 return GitCommand(self,