diff options
Diffstat (limited to 'project.py')
-rw-r--r-- | project.py | 273 |
1 files changed, 181 insertions, 92 deletions
@@ -35,6 +35,7 @@ from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \ | |||
35 | from error import GitError, HookError, UploadError, DownloadError | 35 | from error import GitError, HookError, UploadError, DownloadError |
36 | from error import ManifestInvalidRevisionError | 36 | from error import ManifestInvalidRevisionError |
37 | from error import NoManifestException | 37 | from error import NoManifestException |
38 | import platform_utils | ||
38 | from trace import IsTrace, Trace | 39 | from trace import IsTrace, Trace |
39 | 40 | ||
40 | from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M | 41 | from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M |
@@ -62,9 +63,9 @@ def _lwrite(path, content): | |||
62 | fd.close() | 63 | fd.close() |
63 | 64 | ||
64 | try: | 65 | try: |
65 | os.rename(lock, path) | 66 | platform_utils.rename(lock, path) |
66 | except OSError: | 67 | except OSError: |
67 | os.remove(lock) | 68 | platform_utils.remove(lock) |
68 | raise | 69 | raise |
69 | 70 | ||
70 | 71 | ||
@@ -102,7 +103,7 @@ def _ProjectHooks(): | |||
102 | """ | 103 | """ |
103 | global _project_hook_list | 104 | global _project_hook_list |
104 | if _project_hook_list is None: | 105 | if _project_hook_list is None: |
105 | d = os.path.realpath(os.path.abspath(os.path.dirname(__file__))) | 106 | d = platform_utils.realpath(os.path.abspath(os.path.dirname(__file__))) |
106 | d = os.path.join(d, 'hooks') | 107 | d = os.path.join(d, 'hooks') |
107 | _project_hook_list = [os.path.join(d, x) for x in os.listdir(d)] | 108 | _project_hook_list = [os.path.join(d, x) for x in os.listdir(d)] |
108 | return _project_hook_list | 109 | return _project_hook_list |
@@ -176,12 +177,20 @@ class ReviewableBranch(object): | |||
176 | def UploadForReview(self, people, | 177 | def UploadForReview(self, people, |
177 | auto_topic=False, | 178 | auto_topic=False, |
178 | draft=False, | 179 | draft=False, |
179 | dest_branch=None): | 180 | private=False, |
181 | wip=False, | ||
182 | dest_branch=None, | ||
183 | validate_certs=True, | ||
184 | push_options=None): | ||
180 | self.project.UploadForReview(self.name, | 185 | self.project.UploadForReview(self.name, |
181 | people, | 186 | people, |
182 | auto_topic=auto_topic, | 187 | auto_topic=auto_topic, |
183 | draft=draft, | 188 | draft=draft, |
184 | dest_branch=dest_branch) | 189 | private=private, |
190 | wip=wip, | ||
191 | dest_branch=dest_branch, | ||
192 | validate_certs=validate_certs, | ||
193 | push_options=push_options) | ||
185 | 194 | ||
186 | def GetPublishedRefs(self): | 195 | def GetPublishedRefs(self): |
187 | refs = {} | 196 | refs = {} |
@@ -243,7 +252,7 @@ class _CopyFile(object): | |||
243 | try: | 252 | try: |
244 | # remove existing file first, since it might be read-only | 253 | # remove existing file first, since it might be read-only |
245 | if os.path.exists(dest): | 254 | if os.path.exists(dest): |
246 | os.remove(dest) | 255 | platform_utils.remove(dest) |
247 | else: | 256 | else: |
248 | dest_dir = os.path.dirname(dest) | 257 | dest_dir = os.path.dirname(dest) |
249 | if not os.path.isdir(dest_dir): | 258 | if not os.path.isdir(dest_dir): |
@@ -268,16 +277,16 @@ class _LinkFile(object): | |||
268 | 277 | ||
269 | def __linkIt(self, relSrc, absDest): | 278 | def __linkIt(self, relSrc, absDest): |
270 | # link file if it does not exist or is out of date | 279 | # link file if it does not exist or is out of date |
271 | if not os.path.islink(absDest) or (os.readlink(absDest) != relSrc): | 280 | if not platform_utils.islink(absDest) or (platform_utils.readlink(absDest) != relSrc): |
272 | try: | 281 | try: |
273 | # remove existing file first, since it might be read-only | 282 | # remove existing file first, since it might be read-only |
274 | if os.path.lexists(absDest): | 283 | if os.path.lexists(absDest): |
275 | os.remove(absDest) | 284 | platform_utils.remove(absDest) |
276 | else: | 285 | else: |
277 | dest_dir = os.path.dirname(absDest) | 286 | dest_dir = os.path.dirname(absDest) |
278 | if not os.path.isdir(dest_dir): | 287 | if not os.path.isdir(dest_dir): |
279 | os.makedirs(dest_dir) | 288 | os.makedirs(dest_dir) |
280 | os.symlink(relSrc, absDest) | 289 | platform_utils.symlink(relSrc, absDest) |
281 | except IOError: | 290 | except IOError: |
282 | _error('Cannot link file %s to %s', relSrc, absDest) | 291 | _error('Cannot link file %s to %s', relSrc, absDest) |
283 | 292 | ||
@@ -323,13 +332,15 @@ class RemoteSpec(object): | |||
323 | pushUrl=None, | 332 | pushUrl=None, |
324 | review=None, | 333 | review=None, |
325 | revision=None, | 334 | revision=None, |
326 | orig_name=None): | 335 | orig_name=None, |
336 | fetchUrl=None): | ||
327 | self.name = name | 337 | self.name = name |
328 | self.url = url | 338 | self.url = url |
329 | self.pushUrl = pushUrl | 339 | self.pushUrl = pushUrl |
330 | self.review = review | 340 | self.review = review |
331 | self.revision = revision | 341 | self.revision = revision |
332 | self.orig_name = orig_name | 342 | self.orig_name = orig_name |
343 | self.fetchUrl = fetchUrl | ||
333 | 344 | ||
334 | 345 | ||
335 | class RepoHook(object): | 346 | class RepoHook(object): |
@@ -687,7 +698,7 @@ class Project(object): | |||
687 | self.gitdir = gitdir.replace('\\', '/') | 698 | self.gitdir = gitdir.replace('\\', '/') |
688 | self.objdir = objdir.replace('\\', '/') | 699 | self.objdir = objdir.replace('\\', '/') |
689 | if worktree: | 700 | if worktree: |
690 | self.worktree = os.path.normpath(worktree.replace('\\', '/')) | 701 | self.worktree = os.path.normpath(worktree).replace('\\', '/') |
691 | else: | 702 | else: |
692 | self.worktree = None | 703 | self.worktree = None |
693 | self.relpath = relpath | 704 | self.relpath = relpath |
@@ -911,11 +922,13 @@ class Project(object): | |||
911 | else: | 922 | else: |
912 | return False | 923 | return False |
913 | 924 | ||
914 | def PrintWorkTreeStatus(self, output_redir=None): | 925 | def PrintWorkTreeStatus(self, output_redir=None, quiet=False): |
915 | """Prints the status of the repository to stdout. | 926 | """Prints the status of the repository to stdout. |
916 | 927 | ||
917 | Args: | 928 | Args: |
918 | output: If specified, redirect the output to this object. | 929 | output: If specified, redirect the output to this object. |
930 | quiet: If True then only print the project name. Do not print | ||
931 | the modified files, branch name, etc. | ||
919 | """ | 932 | """ |
920 | if not os.path.isdir(self.worktree): | 933 | if not os.path.isdir(self.worktree): |
921 | if output_redir is None: | 934 | if output_redir is None: |
@@ -941,6 +954,10 @@ class Project(object): | |||
941 | out.redirect(output_redir) | 954 | out.redirect(output_redir) |
942 | out.project('project %-40s', self.relpath + '/ ') | 955 | out.project('project %-40s', self.relpath + '/ ') |
943 | 956 | ||
957 | if quiet: | ||
958 | out.nl() | ||
959 | return 'DIRTY' | ||
960 | |||
944 | branch = self.CurrentBranch | 961 | branch = self.CurrentBranch |
945 | if branch is None: | 962 | if branch is None: |
946 | out.nobranch('(*** NO BRANCH ***)') | 963 | out.nobranch('(*** NO BRANCH ***)') |
@@ -1099,7 +1116,11 @@ class Project(object): | |||
1099 | people=([], []), | 1116 | people=([], []), |
1100 | auto_topic=False, | 1117 | auto_topic=False, |
1101 | draft=False, | 1118 | draft=False, |
1102 | dest_branch=None): | 1119 | private=False, |
1120 | wip=False, | ||
1121 | dest_branch=None, | ||
1122 | validate_certs=True, | ||
1123 | push_options=None): | ||
1103 | """Uploads the named branch for code review. | 1124 | """Uploads the named branch for code review. |
1104 | """ | 1125 | """ |
1105 | if branch is None: | 1126 | if branch is None: |
@@ -1124,7 +1145,7 @@ class Project(object): | |||
1124 | branch.remote.projectname = self.name | 1145 | branch.remote.projectname = self.name |
1125 | branch.remote.Save() | 1146 | branch.remote.Save() |
1126 | 1147 | ||
1127 | url = branch.remote.ReviewUrl(self.UserEmail) | 1148 | url = branch.remote.ReviewUrl(self.UserEmail, validate_certs) |
1128 | if url is None: | 1149 | if url is None: |
1129 | raise UploadError('review not configured') | 1150 | raise UploadError('review not configured') |
1130 | cmd = ['push'] | 1151 | cmd = ['push'] |
@@ -1137,6 +1158,10 @@ class Project(object): | |||
1137 | rp.append('--cc=%s' % sq(e)) | 1158 | rp.append('--cc=%s' % sq(e)) |
1138 | cmd.append('--receive-pack=%s' % " ".join(rp)) | 1159 | cmd.append('--receive-pack=%s' % " ".join(rp)) |
1139 | 1160 | ||
1161 | for push_option in (push_options or []): | ||
1162 | cmd.append('-o') | ||
1163 | cmd.append(push_option) | ||
1164 | |||
1140 | cmd.append(url) | 1165 | cmd.append(url) |
1141 | 1166 | ||
1142 | if dest_branch.startswith(R_HEADS): | 1167 | if dest_branch.startswith(R_HEADS): |
@@ -1150,9 +1175,14 @@ class Project(object): | |||
1150 | dest_branch) | 1175 | dest_branch) |
1151 | if auto_topic: | 1176 | if auto_topic: |
1152 | ref_spec = ref_spec + '/' + branch.name | 1177 | ref_spec = ref_spec + '/' + branch.name |
1178 | |||
1153 | if not url.startswith('ssh://'): | 1179 | if not url.startswith('ssh://'): |
1154 | rp = ['r=%s' % p for p in people[0]] + \ | 1180 | rp = ['r=%s' % p for p in people[0]] + \ |
1155 | ['cc=%s' % p for p in people[1]] | 1181 | ['cc=%s' % p for p in people[1]] |
1182 | if private: | ||
1183 | rp = rp + ['private'] | ||
1184 | if wip: | ||
1185 | rp = rp + ['wip'] | ||
1156 | if rp: | 1186 | if rp: |
1157 | ref_spec = ref_spec + '%' + ','.join(rp) | 1187 | ref_spec = ref_spec + '%' + ','.join(rp) |
1158 | cmd.append(ref_spec) | 1188 | cmd.append(ref_spec) |
@@ -1192,7 +1222,8 @@ class Project(object): | |||
1192 | no_tags=False, | 1222 | no_tags=False, |
1193 | archive=False, | 1223 | archive=False, |
1194 | optimized_fetch=False, | 1224 | optimized_fetch=False, |
1195 | prune=False): | 1225 | prune=False, |
1226 | submodules=False): | ||
1196 | """Perform only the network IO portion of the sync process. | 1227 | """Perform only the network IO portion of the sync process. |
1197 | Local working directory/branch state is not affected. | 1228 | Local working directory/branch state is not affected. |
1198 | """ | 1229 | """ |
@@ -1218,7 +1249,7 @@ class Project(object): | |||
1218 | if not self._ExtractArchive(tarpath, path=topdir): | 1249 | if not self._ExtractArchive(tarpath, path=topdir): |
1219 | return False | 1250 | return False |
1220 | try: | 1251 | try: |
1221 | os.remove(tarpath) | 1252 | platform_utils.remove(tarpath) |
1222 | except OSError as e: | 1253 | except OSError as e: |
1223 | _warn("Cannot remove archive %s: %s", tarpath, str(e)) | 1254 | _warn("Cannot remove archive %s: %s", tarpath, str(e)) |
1224 | self._CopyAndLinkFiles() | 1255 | self._CopyAndLinkFiles() |
@@ -1234,7 +1265,7 @@ class Project(object): | |||
1234 | if is_new: | 1265 | if is_new: |
1235 | alt = os.path.join(self.gitdir, 'objects/info/alternates') | 1266 | alt = os.path.join(self.gitdir, 'objects/info/alternates') |
1236 | try: | 1267 | try: |
1237 | fd = open(alt, 'rb') | 1268 | fd = open(alt) |
1238 | try: | 1269 | try: |
1239 | alt_dir = fd.readline().rstrip() | 1270 | alt_dir = fd.readline().rstrip() |
1240 | finally: | 1271 | finally: |
@@ -1258,13 +1289,19 @@ class Project(object): | |||
1258 | elif self.manifest.default.sync_c: | 1289 | elif self.manifest.default.sync_c: |
1259 | current_branch_only = True | 1290 | current_branch_only = True |
1260 | 1291 | ||
1292 | if self.clone_depth: | ||
1293 | depth = self.clone_depth | ||
1294 | else: | ||
1295 | depth = self.manifest.manifestProject.config.GetString('repo.depth') | ||
1296 | |||
1261 | need_to_fetch = not (optimized_fetch and | 1297 | need_to_fetch = not (optimized_fetch and |
1262 | (ID_RE.match(self.revisionExpr) and | 1298 | (ID_RE.match(self.revisionExpr) and |
1263 | self._CheckForSha1())) | 1299 | self._CheckForImmutableRevision())) |
1264 | if (need_to_fetch and | 1300 | if (need_to_fetch and |
1265 | not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir, | 1301 | not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir, |
1266 | current_branch_only=current_branch_only, | 1302 | current_branch_only=current_branch_only, |
1267 | no_tags=no_tags, prune=prune)): | 1303 | no_tags=no_tags, prune=prune, depth=depth, |
1304 | submodules=submodules)): | ||
1268 | return False | 1305 | return False |
1269 | 1306 | ||
1270 | if self.worktree: | 1307 | if self.worktree: |
@@ -1272,7 +1309,7 @@ class Project(object): | |||
1272 | else: | 1309 | else: |
1273 | self._InitMirrorHead() | 1310 | self._InitMirrorHead() |
1274 | try: | 1311 | try: |
1275 | os.remove(os.path.join(self.gitdir, 'FETCH_HEAD')) | 1312 | platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD')) |
1276 | except OSError: | 1313 | except OSError: |
1277 | pass | 1314 | pass |
1278 | return True | 1315 | return True |
@@ -1320,11 +1357,11 @@ class Project(object): | |||
1320 | raise ManifestInvalidRevisionError('revision %s in %s not found' % | 1357 | raise ManifestInvalidRevisionError('revision %s in %s not found' % |
1321 | (self.revisionExpr, self.name)) | 1358 | (self.revisionExpr, self.name)) |
1322 | 1359 | ||
1323 | def Sync_LocalHalf(self, syncbuf, force_sync=False): | 1360 | def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False): |
1324 | """Perform only the local IO portion of the sync process. | 1361 | """Perform only the local IO portion of the sync process. |
1325 | Network access is not required. | 1362 | Network access is not required. |
1326 | """ | 1363 | """ |
1327 | self._InitWorkTree(force_sync=force_sync) | 1364 | self._InitWorkTree(force_sync=force_sync, submodules=submodules) |
1328 | all_refs = self.bare_ref.all | 1365 | all_refs = self.bare_ref.all |
1329 | self.CleanPublishedCache(all_refs) | 1366 | self.CleanPublishedCache(all_refs) |
1330 | revid = self.GetRevisionId(all_refs) | 1367 | revid = self.GetRevisionId(all_refs) |
@@ -1333,6 +1370,9 @@ class Project(object): | |||
1333 | self._FastForward(revid) | 1370 | self._FastForward(revid) |
1334 | self._CopyAndLinkFiles() | 1371 | self._CopyAndLinkFiles() |
1335 | 1372 | ||
1373 | def _dosubmodules(): | ||
1374 | self._SyncSubmodules(quiet=True) | ||
1375 | |||
1336 | head = self.work_git.GetHead() | 1376 | head = self.work_git.GetHead() |
1337 | if head.startswith(R_HEADS): | 1377 | if head.startswith(R_HEADS): |
1338 | branch = head[len(R_HEADS):] | 1378 | branch = head[len(R_HEADS):] |
@@ -1366,6 +1406,8 @@ class Project(object): | |||
1366 | 1406 | ||
1367 | try: | 1407 | try: |
1368 | self._Checkout(revid, quiet=True) | 1408 | self._Checkout(revid, quiet=True) |
1409 | if submodules: | ||
1410 | self._SyncSubmodules(quiet=True) | ||
1369 | except GitError as e: | 1411 | except GitError as e: |
1370 | syncbuf.fail(self, e) | 1412 | syncbuf.fail(self, e) |
1371 | return | 1413 | return |
@@ -1390,6 +1432,8 @@ class Project(object): | |||
1390 | branch.name) | 1432 | branch.name) |
1391 | try: | 1433 | try: |
1392 | self._Checkout(revid, quiet=True) | 1434 | self._Checkout(revid, quiet=True) |
1435 | if submodules: | ||
1436 | self._SyncSubmodules(quiet=True) | ||
1393 | except GitError as e: | 1437 | except GitError as e: |
1394 | syncbuf.fail(self, e) | 1438 | syncbuf.fail(self, e) |
1395 | return | 1439 | return |
@@ -1415,6 +1459,8 @@ class Project(object): | |||
1415 | # strict subset. We can fast-forward safely. | 1459 | # strict subset. We can fast-forward safely. |
1416 | # | 1460 | # |
1417 | syncbuf.later1(self, _doff) | 1461 | syncbuf.later1(self, _doff) |
1462 | if submodules: | ||
1463 | syncbuf.later1(self, _dosubmodules) | ||
1418 | return | 1464 | return |
1419 | 1465 | ||
1420 | # Examine the local commits not in the remote. Find the | 1466 | # Examine the local commits not in the remote. Find the |
@@ -1466,19 +1512,28 @@ class Project(object): | |||
1466 | branch.Save() | 1512 | branch.Save() |
1467 | 1513 | ||
1468 | if cnt_mine > 0 and self.rebase: | 1514 | if cnt_mine > 0 and self.rebase: |
1515 | def _docopyandlink(): | ||
1516 | self._CopyAndLinkFiles() | ||
1517 | |||
1469 | def _dorebase(): | 1518 | def _dorebase(): |
1470 | self._Rebase(upstream='%s^1' % last_mine, onto=revid) | 1519 | self._Rebase(upstream='%s^1' % last_mine, onto=revid) |
1471 | self._CopyAndLinkFiles() | ||
1472 | syncbuf.later2(self, _dorebase) | 1520 | syncbuf.later2(self, _dorebase) |
1521 | if submodules: | ||
1522 | syncbuf.later2(self, _dosubmodules) | ||
1523 | syncbuf.later2(self, _docopyandlink) | ||
1473 | elif local_changes: | 1524 | elif local_changes: |
1474 | try: | 1525 | try: |
1475 | self._ResetHard(revid) | 1526 | self._ResetHard(revid) |
1527 | if submodules: | ||
1528 | self._SyncSubmodules(quiet=True) | ||
1476 | self._CopyAndLinkFiles() | 1529 | self._CopyAndLinkFiles() |
1477 | except GitError as e: | 1530 | except GitError as e: |
1478 | syncbuf.fail(self, e) | 1531 | syncbuf.fail(self, e) |
1479 | return | 1532 | return |
1480 | else: | 1533 | else: |
1481 | syncbuf.later1(self, _doff) | 1534 | syncbuf.later1(self, _doff) |
1535 | if submodules: | ||
1536 | syncbuf.later1(self, _dosubmodules) | ||
1482 | 1537 | ||
1483 | def AddCopyFile(self, src, dest, absdest): | 1538 | def AddCopyFile(self, src, dest, absdest): |
1484 | # dest should already be an absolute path, but src is project relative | 1539 | # dest should already be an absolute path, but src is project relative |
@@ -1764,7 +1819,7 @@ class Project(object): | |||
1764 | except GitError: | 1819 | except GitError: |
1765 | return [], [] | 1820 | return [], [] |
1766 | finally: | 1821 | finally: |
1767 | os.remove(temp_gitmodules_path) | 1822 | platform_utils.remove(temp_gitmodules_path) |
1768 | 1823 | ||
1769 | names = set() | 1824 | names = set() |
1770 | paths = {} | 1825 | paths = {} |
@@ -1851,7 +1906,7 @@ class Project(object): | |||
1851 | 1906 | ||
1852 | 1907 | ||
1853 | # Direct Git Commands ## | 1908 | # Direct Git Commands ## |
1854 | def _CheckForSha1(self): | 1909 | def _CheckForImmutableRevision(self): |
1855 | try: | 1910 | try: |
1856 | # if revision (sha or tag) is not present then following function | 1911 | # if revision (sha or tag) is not present then following function |
1857 | # throws an error. | 1912 | # throws an error. |
@@ -1880,23 +1935,18 @@ class Project(object): | |||
1880 | quiet=False, | 1935 | quiet=False, |
1881 | alt_dir=None, | 1936 | alt_dir=None, |
1882 | no_tags=False, | 1937 | no_tags=False, |
1883 | prune=False): | 1938 | prune=False, |
1939 | depth=None, | ||
1940 | submodules=False): | ||
1884 | 1941 | ||
1885 | is_sha1 = False | 1942 | is_sha1 = False |
1886 | tag_name = None | 1943 | tag_name = None |
1887 | depth = None | ||
1888 | |||
1889 | # The depth should not be used when fetching to a mirror because | 1944 | # The depth should not be used when fetching to a mirror because |
1890 | # it will result in a shallow repository that cannot be cloned or | 1945 | # it will result in a shallow repository that cannot be cloned or |
1891 | # fetched from. | 1946 | # fetched from. |
1892 | if not self.manifest.IsMirror: | 1947 | # The repo project should also never be synced with partial depth. |
1893 | if self.clone_depth: | 1948 | if self.manifest.IsMirror or self.relpath == '.repo/repo': |
1894 | depth = self.clone_depth | 1949 | depth = None |
1895 | else: | ||
1896 | depth = self.manifest.manifestProject.config.GetString('repo.depth') | ||
1897 | # The repo project should never be synced with partial depth | ||
1898 | if self.relpath == '.repo/repo': | ||
1899 | depth = None | ||
1900 | 1950 | ||
1901 | if depth: | 1951 | if depth: |
1902 | current_branch_only = True | 1952 | current_branch_only = True |
@@ -1910,7 +1960,9 @@ class Project(object): | |||
1910 | tag_name = self.revisionExpr[len(R_TAGS):] | 1960 | tag_name = self.revisionExpr[len(R_TAGS):] |
1911 | 1961 | ||
1912 | if is_sha1 or tag_name is not None: | 1962 | if is_sha1 or tag_name is not None: |
1913 | if self._CheckForSha1(): | 1963 | if self._CheckForImmutableRevision(): |
1964 | print('Skipped fetching project %s (already have persistent ref)' | ||
1965 | % self.name) | ||
1914 | return True | 1966 | return True |
1915 | if is_sha1 and not depth: | 1967 | if is_sha1 and not depth: |
1916 | # When syncing a specific commit and --depth is not set: | 1968 | # When syncing a specific commit and --depth is not set: |
@@ -1958,15 +2010,17 @@ class Project(object): | |||
1958 | ids.add(ref_id) | 2010 | ids.add(ref_id) |
1959 | tmp.add(r) | 2011 | tmp.add(r) |
1960 | 2012 | ||
1961 | tmp_packed = '' | 2013 | tmp_packed_lines = [] |
1962 | old_packed = '' | 2014 | old_packed_lines = [] |
1963 | 2015 | ||
1964 | for r in sorted(all_refs): | 2016 | for r in sorted(all_refs): |
1965 | line = '%s %s\n' % (all_refs[r], r) | 2017 | line = '%s %s\n' % (all_refs[r], r) |
1966 | tmp_packed += line | 2018 | tmp_packed_lines.append(line) |
1967 | if r not in tmp: | 2019 | if r not in tmp: |
1968 | old_packed += line | 2020 | old_packed_lines.append(line) |
1969 | 2021 | ||
2022 | tmp_packed = ''.join(tmp_packed_lines) | ||
2023 | old_packed = ''.join(old_packed_lines) | ||
1970 | _lwrite(packed_refs, tmp_packed) | 2024 | _lwrite(packed_refs, tmp_packed) |
1971 | else: | 2025 | else: |
1972 | alt_dir = None | 2026 | alt_dir = None |
@@ -1999,6 +2053,9 @@ class Project(object): | |||
1999 | if prune: | 2053 | if prune: |
2000 | cmd.append('--prune') | 2054 | cmd.append('--prune') |
2001 | 2055 | ||
2056 | if submodules: | ||
2057 | cmd.append('--recurse-submodules=on-demand') | ||
2058 | |||
2002 | spec = [] | 2059 | spec = [] |
2003 | if not current_branch_only: | 2060 | if not current_branch_only: |
2004 | # Fetch whole repo | 2061 | # Fetch whole repo |
@@ -2054,24 +2111,25 @@ class Project(object): | |||
2054 | if old_packed != '': | 2111 | if old_packed != '': |
2055 | _lwrite(packed_refs, old_packed) | 2112 | _lwrite(packed_refs, old_packed) |
2056 | else: | 2113 | else: |
2057 | os.remove(packed_refs) | 2114 | platform_utils.remove(packed_refs) |
2058 | self.bare_git.pack_refs('--all', '--prune') | 2115 | self.bare_git.pack_refs('--all', '--prune') |
2059 | 2116 | ||
2060 | if is_sha1 and current_branch_only and self.upstream: | 2117 | if is_sha1 and current_branch_only: |
2061 | # We just synced the upstream given branch; verify we | 2118 | # We just synced the upstream given branch; verify we |
2062 | # got what we wanted, else trigger a second run of all | 2119 | # got what we wanted, else trigger a second run of all |
2063 | # refs. | 2120 | # refs. |
2064 | if not self._CheckForSha1(): | 2121 | if not self._CheckForImmutableRevision(): |
2065 | if not depth: | 2122 | if current_branch_only and depth: |
2066 | # Avoid infinite recursion when depth is True (since depth implies | 2123 | # Sync the current branch only with depth set to None |
2067 | # current_branch_only) | ||
2068 | return self._RemoteFetch(name=name, current_branch_only=False, | ||
2069 | initial=False, quiet=quiet, alt_dir=alt_dir) | ||
2070 | if self.clone_depth: | ||
2071 | self.clone_depth = None | ||
2072 | return self._RemoteFetch(name=name, | 2124 | return self._RemoteFetch(name=name, |
2073 | current_branch_only=current_branch_only, | 2125 | current_branch_only=current_branch_only, |
2074 | initial=False, quiet=quiet, alt_dir=alt_dir) | 2126 | initial=False, quiet=quiet, alt_dir=alt_dir, |
2127 | depth=None) | ||
2128 | else: | ||
2129 | # Avoid infinite recursion: sync all branches with depth set to None | ||
2130 | return self._RemoteFetch(name=name, current_branch_only=False, | ||
2131 | initial=False, quiet=quiet, alt_dir=alt_dir, | ||
2132 | depth=None) | ||
2075 | 2133 | ||
2076 | return ok | 2134 | return ok |
2077 | 2135 | ||
@@ -2115,14 +2173,14 @@ class Project(object): | |||
2115 | 2173 | ||
2116 | ok = GitCommand(self, cmd, bare=True).Wait() == 0 | 2174 | ok = GitCommand(self, cmd, bare=True).Wait() == 0 |
2117 | if os.path.exists(bundle_dst): | 2175 | if os.path.exists(bundle_dst): |
2118 | os.remove(bundle_dst) | 2176 | platform_utils.remove(bundle_dst) |
2119 | if os.path.exists(bundle_tmp): | 2177 | if os.path.exists(bundle_tmp): |
2120 | os.remove(bundle_tmp) | 2178 | platform_utils.remove(bundle_tmp) |
2121 | return ok | 2179 | return ok |
2122 | 2180 | ||
2123 | def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet): | 2181 | def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet): |
2124 | if os.path.exists(dstPath): | 2182 | if os.path.exists(dstPath): |
2125 | os.remove(dstPath) | 2183 | platform_utils.remove(dstPath) |
2126 | 2184 | ||
2127 | cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location'] | 2185 | cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location'] |
2128 | if quiet: | 2186 | if quiet: |
@@ -2132,7 +2190,7 @@ class Project(object): | |||
2132 | if size >= 1024: | 2190 | if size >= 1024: |
2133 | cmd += ['--continue-at', '%d' % (size,)] | 2191 | cmd += ['--continue-at', '%d' % (size,)] |
2134 | else: | 2192 | else: |
2135 | os.remove(tmpPath) | 2193 | platform_utils.remove(tmpPath) |
2136 | if 'http_proxy' in os.environ and 'darwin' == sys.platform: | 2194 | if 'http_proxy' in os.environ and 'darwin' == sys.platform: |
2137 | cmd += ['--proxy', os.environ['http_proxy']] | 2195 | cmd += ['--proxy', os.environ['http_proxy']] |
2138 | with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, _proxy): | 2196 | with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, _proxy): |
@@ -2163,10 +2221,10 @@ class Project(object): | |||
2163 | 2221 | ||
2164 | if os.path.exists(tmpPath): | 2222 | if os.path.exists(tmpPath): |
2165 | if curlret == 0 and self._IsValidBundle(tmpPath, quiet): | 2223 | if curlret == 0 and self._IsValidBundle(tmpPath, quiet): |
2166 | os.rename(tmpPath, dstPath) | 2224 | platform_utils.rename(tmpPath, dstPath) |
2167 | return True | 2225 | return True |
2168 | else: | 2226 | else: |
2169 | os.remove(tmpPath) | 2227 | platform_utils.remove(tmpPath) |
2170 | return False | 2228 | return False |
2171 | else: | 2229 | else: |
2172 | return False | 2230 | return False |
@@ -2218,6 +2276,13 @@ class Project(object): | |||
2218 | if GitCommand(self, cmd).Wait() != 0: | 2276 | if GitCommand(self, cmd).Wait() != 0: |
2219 | raise GitError('%s reset --hard %s ' % (self.name, rev)) | 2277 | raise GitError('%s reset --hard %s ' % (self.name, rev)) |
2220 | 2278 | ||
2279 | def _SyncSubmodules(self, quiet=True): | ||
2280 | cmd = ['submodule', 'update', '--init', '--recursive'] | ||
2281 | if quiet: | ||
2282 | cmd.append('-q') | ||
2283 | if GitCommand(self, cmd).Wait() != 0: | ||
2284 | raise GitError('%s submodule update --init --recursive %s ' % self.name) | ||
2285 | |||
2221 | def _Rebase(self, upstream, onto=None): | 2286 | def _Rebase(self, upstream, onto=None): |
2222 | cmd = ['rebase'] | 2287 | cmd = ['rebase'] |
2223 | if onto is not None: | 2288 | if onto is not None: |
@@ -2257,10 +2322,10 @@ class Project(object): | |||
2257 | print("Retrying clone after deleting %s" % | 2322 | print("Retrying clone after deleting %s" % |
2258 | self.gitdir, file=sys.stderr) | 2323 | self.gitdir, file=sys.stderr) |
2259 | try: | 2324 | try: |
2260 | shutil.rmtree(os.path.realpath(self.gitdir)) | 2325 | platform_utils.rmtree(platform_utils.realpath(self.gitdir)) |
2261 | if self.worktree and os.path.exists(os.path.realpath | 2326 | if self.worktree and os.path.exists(platform_utils.realpath |
2262 | (self.worktree)): | 2327 | (self.worktree)): |
2263 | shutil.rmtree(os.path.realpath(self.worktree)) | 2328 | platform_utils.rmtree(platform_utils.realpath(self.worktree)) |
2264 | return self._InitGitDir(mirror_git=mirror_git, force_sync=False) | 2329 | return self._InitGitDir(mirror_git=mirror_git, force_sync=False) |
2265 | except: | 2330 | except: |
2266 | raise e | 2331 | raise e |
@@ -2302,9 +2367,9 @@ class Project(object): | |||
2302 | self.config.SetString('core.bare', None) | 2367 | self.config.SetString('core.bare', None) |
2303 | except Exception: | 2368 | except Exception: |
2304 | if init_obj_dir and os.path.exists(self.objdir): | 2369 | if init_obj_dir and os.path.exists(self.objdir): |
2305 | shutil.rmtree(self.objdir) | 2370 | platform_utils.rmtree(self.objdir) |
2306 | if init_git_dir and os.path.exists(self.gitdir): | 2371 | if init_git_dir and os.path.exists(self.gitdir): |
2307 | shutil.rmtree(self.gitdir) | 2372 | platform_utils.rmtree(self.gitdir) |
2308 | raise | 2373 | raise |
2309 | 2374 | ||
2310 | def _UpdateHooks(self): | 2375 | def _UpdateHooks(self): |
@@ -2312,7 +2377,7 @@ class Project(object): | |||
2312 | self._InitHooks() | 2377 | self._InitHooks() |
2313 | 2378 | ||
2314 | def _InitHooks(self): | 2379 | def _InitHooks(self): |
2315 | hooks = os.path.realpath(self._gitdir_path('hooks')) | 2380 | hooks = platform_utils.realpath(self._gitdir_path('hooks')) |
2316 | if not os.path.exists(hooks): | 2381 | if not os.path.exists(hooks): |
2317 | os.makedirs(hooks) | 2382 | os.makedirs(hooks) |
2318 | for stock_hook in _ProjectHooks(): | 2383 | for stock_hook in _ProjectHooks(): |
@@ -2328,20 +2393,21 @@ class Project(object): | |||
2328 | continue | 2393 | continue |
2329 | 2394 | ||
2330 | dst = os.path.join(hooks, name) | 2395 | dst = os.path.join(hooks, name) |
2331 | if os.path.islink(dst): | 2396 | if platform_utils.islink(dst): |
2332 | continue | 2397 | continue |
2333 | if os.path.exists(dst): | 2398 | if os.path.exists(dst): |
2334 | if filecmp.cmp(stock_hook, dst, shallow=False): | 2399 | if filecmp.cmp(stock_hook, dst, shallow=False): |
2335 | os.remove(dst) | 2400 | platform_utils.remove(dst) |
2336 | else: | 2401 | else: |
2337 | _warn("%s: Not replacing locally modified %s hook", | 2402 | _warn("%s: Not replacing locally modified %s hook", |
2338 | self.relpath, name) | 2403 | self.relpath, name) |
2339 | continue | 2404 | continue |
2340 | try: | 2405 | try: |
2341 | os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst) | 2406 | platform_utils.symlink( |
2407 | os.path.relpath(stock_hook, os.path.dirname(dst)), dst) | ||
2342 | except OSError as e: | 2408 | except OSError as e: |
2343 | if e.errno == errno.EPERM: | 2409 | if e.errno == errno.EPERM: |
2344 | raise GitError('filesystem must support symlinks') | 2410 | raise GitError(self._get_symlink_error_message()) |
2345 | else: | 2411 | else: |
2346 | raise | 2412 | raise |
2347 | 2413 | ||
@@ -2389,11 +2455,12 @@ class Project(object): | |||
2389 | symlink_dirs += self.working_tree_dirs | 2455 | symlink_dirs += self.working_tree_dirs |
2390 | to_symlink = symlink_files + symlink_dirs | 2456 | to_symlink = symlink_files + symlink_dirs |
2391 | for name in set(to_symlink): | 2457 | for name in set(to_symlink): |
2392 | dst = os.path.realpath(os.path.join(destdir, name)) | 2458 | dst = platform_utils.realpath(os.path.join(destdir, name)) |
2393 | if os.path.lexists(dst): | 2459 | if os.path.lexists(dst): |
2394 | src = os.path.realpath(os.path.join(srcdir, name)) | 2460 | src = platform_utils.realpath(os.path.join(srcdir, name)) |
2395 | # Fail if the links are pointing to the wrong place | 2461 | # Fail if the links are pointing to the wrong place |
2396 | if src != dst: | 2462 | if src != dst: |
2463 | _error('%s is different in %s vs %s', name, destdir, srcdir) | ||
2397 | raise GitError('--force-sync not enabled; cannot overwrite a local ' | 2464 | raise GitError('--force-sync not enabled; cannot overwrite a local ' |
2398 | 'work tree. If you\'re comfortable with the ' | 2465 | 'work tree. If you\'re comfortable with the ' |
2399 | 'possibility of losing the work tree\'s git metadata,' | 2466 | 'possibility of losing the work tree\'s git metadata,' |
@@ -2422,10 +2489,10 @@ class Project(object): | |||
2422 | if copy_all: | 2489 | if copy_all: |
2423 | to_copy = os.listdir(gitdir) | 2490 | to_copy = os.listdir(gitdir) |
2424 | 2491 | ||
2425 | dotgit = os.path.realpath(dotgit) | 2492 | dotgit = platform_utils.realpath(dotgit) |
2426 | for name in set(to_copy).union(to_symlink): | 2493 | for name in set(to_copy).union(to_symlink): |
2427 | try: | 2494 | try: |
2428 | src = os.path.realpath(os.path.join(gitdir, name)) | 2495 | src = platform_utils.realpath(os.path.join(gitdir, name)) |
2429 | dst = os.path.join(dotgit, name) | 2496 | dst = os.path.join(dotgit, name) |
2430 | 2497 | ||
2431 | if os.path.lexists(dst): | 2498 | if os.path.lexists(dst): |
@@ -2435,28 +2502,30 @@ class Project(object): | |||
2435 | if name in symlink_dirs and not os.path.lexists(src): | 2502 | if name in symlink_dirs and not os.path.lexists(src): |
2436 | os.makedirs(src) | 2503 | os.makedirs(src) |
2437 | 2504 | ||
2505 | if name in to_symlink: | ||
2506 | platform_utils.symlink( | ||
2507 | os.path.relpath(src, os.path.dirname(dst)), dst) | ||
2508 | elif copy_all and not platform_utils.islink(dst): | ||
2509 | if os.path.isdir(src): | ||
2510 | shutil.copytree(src, dst) | ||
2511 | elif os.path.isfile(src): | ||
2512 | shutil.copy(src, dst) | ||
2513 | |||
2438 | # If the source file doesn't exist, ensure the destination | 2514 | # If the source file doesn't exist, ensure the destination |
2439 | # file doesn't either. | 2515 | # file doesn't either. |
2440 | if name in symlink_files and not os.path.lexists(src): | 2516 | if name in symlink_files and not os.path.lexists(src): |
2441 | try: | 2517 | try: |
2442 | os.remove(dst) | 2518 | platform_utils.remove(dst) |
2443 | except OSError: | 2519 | except OSError: |
2444 | pass | 2520 | pass |
2445 | 2521 | ||
2446 | if name in to_symlink: | ||
2447 | os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst) | ||
2448 | elif copy_all and not os.path.islink(dst): | ||
2449 | if os.path.isdir(src): | ||
2450 | shutil.copytree(src, dst) | ||
2451 | elif os.path.isfile(src): | ||
2452 | shutil.copy(src, dst) | ||
2453 | except OSError as e: | 2522 | except OSError as e: |
2454 | if e.errno == errno.EPERM: | 2523 | if e.errno == errno.EPERM: |
2455 | raise DownloadError('filesystem must support symlinks') | 2524 | raise DownloadError(self._get_symlink_error_message()) |
2456 | else: | 2525 | else: |
2457 | raise | 2526 | raise |
2458 | 2527 | ||
2459 | def _InitWorkTree(self, force_sync=False): | 2528 | def _InitWorkTree(self, force_sync=False, submodules=False): |
2460 | dotgit = os.path.join(self.worktree, '.git') | 2529 | dotgit = os.path.join(self.worktree, '.git') |
2461 | init_dotgit = not os.path.exists(dotgit) | 2530 | init_dotgit = not os.path.exists(dotgit) |
2462 | try: | 2531 | try: |
@@ -2470,8 +2539,8 @@ class Project(object): | |||
2470 | except GitError as e: | 2539 | except GitError as e: |
2471 | if force_sync: | 2540 | if force_sync: |
2472 | try: | 2541 | try: |
2473 | shutil.rmtree(dotgit) | 2542 | platform_utils.rmtree(dotgit) |
2474 | return self._InitWorkTree(force_sync=False) | 2543 | return self._InitWorkTree(force_sync=False, submodules=submodules) |
2475 | except: | 2544 | except: |
2476 | raise e | 2545 | raise e |
2477 | raise e | 2546 | raise e |
@@ -2485,14 +2554,24 @@ class Project(object): | |||
2485 | if GitCommand(self, cmd).Wait() != 0: | 2554 | if GitCommand(self, cmd).Wait() != 0: |
2486 | raise GitError("cannot initialize work tree") | 2555 | raise GitError("cannot initialize work tree") |
2487 | 2556 | ||
2557 | if submodules: | ||
2558 | self._SyncSubmodules(quiet=True) | ||
2488 | self._CopyAndLinkFiles() | 2559 | self._CopyAndLinkFiles() |
2489 | except Exception: | 2560 | except Exception: |
2490 | if init_dotgit: | 2561 | if init_dotgit: |
2491 | shutil.rmtree(dotgit) | 2562 | platform_utils.rmtree(dotgit) |
2492 | raise | 2563 | raise |
2493 | 2564 | ||
2565 | def _get_symlink_error_message(self): | ||
2566 | if platform_utils.isWindows(): | ||
2567 | return ('Unable to create symbolic link. Please re-run the command as ' | ||
2568 | 'Administrator, or see ' | ||
2569 | 'https://github.com/git-for-windows/git/wiki/Symbolic-Links ' | ||
2570 | 'for other options.') | ||
2571 | return 'filesystem must support symlinks' | ||
2572 | |||
2494 | def _gitdir_path(self, path): | 2573 | def _gitdir_path(self, path): |
2495 | return os.path.realpath(os.path.join(self.gitdir, path)) | 2574 | return platform_utils.realpath(os.path.join(self.gitdir, path)) |
2496 | 2575 | ||
2497 | def _revlist(self, *args, **kw): | 2576 | def _revlist(self, *args, **kw): |
2498 | a = [] | 2577 | a = [] |
@@ -2627,11 +2706,11 @@ class Project(object): | |||
2627 | else: | 2706 | else: |
2628 | path = os.path.join(self._project.worktree, '.git', HEAD) | 2707 | path = os.path.join(self._project.worktree, '.git', HEAD) |
2629 | try: | 2708 | try: |
2630 | fd = open(path, 'rb') | 2709 | fd = open(path) |
2631 | except IOError as e: | 2710 | except IOError as e: |
2632 | raise NoManifestException(path, str(e)) | 2711 | raise NoManifestException(path, str(e)) |
2633 | try: | 2712 | try: |
2634 | line = fd.read() | 2713 | line = fd.readline() |
2635 | finally: | 2714 | finally: |
2636 | fd.close() | 2715 | fd.close() |
2637 | try: | 2716 | try: |
@@ -2833,13 +2912,14 @@ class SyncBuffer(object): | |||
2833 | 2912 | ||
2834 | self.detach_head = detach_head | 2913 | self.detach_head = detach_head |
2835 | self.clean = True | 2914 | self.clean = True |
2915 | self.recent_clean = True | ||
2836 | 2916 | ||
2837 | def info(self, project, fmt, *args): | 2917 | def info(self, project, fmt, *args): |
2838 | self._messages.append(_InfoMessage(project, fmt % args)) | 2918 | self._messages.append(_InfoMessage(project, fmt % args)) |
2839 | 2919 | ||
2840 | def fail(self, project, err=None): | 2920 | def fail(self, project, err=None): |
2841 | self._failures.append(_Failure(project, err)) | 2921 | self._failures.append(_Failure(project, err)) |
2842 | self.clean = False | 2922 | self._MarkUnclean() |
2843 | 2923 | ||
2844 | def later1(self, project, what): | 2924 | def later1(self, project, what): |
2845 | self._later_queue1.append(_Later(project, what)) | 2925 | self._later_queue1.append(_Later(project, what)) |
@@ -2853,6 +2933,15 @@ class SyncBuffer(object): | |||
2853 | self._PrintMessages() | 2933 | self._PrintMessages() |
2854 | return self.clean | 2934 | return self.clean |
2855 | 2935 | ||
2936 | def Recently(self): | ||
2937 | recent_clean = self.recent_clean | ||
2938 | self.recent_clean = True | ||
2939 | return recent_clean | ||
2940 | |||
2941 | def _MarkUnclean(self): | ||
2942 | self.clean = False | ||
2943 | self.recent_clean = False | ||
2944 | |||
2856 | def _RunLater(self): | 2945 | def _RunLater(self): |
2857 | for q in ['_later_queue1', '_later_queue2']: | 2946 | for q in ['_later_queue1', '_later_queue2']: |
2858 | if not self._RunQueue(q): | 2947 | if not self._RunQueue(q): |
@@ -2861,7 +2950,7 @@ class SyncBuffer(object): | |||
2861 | def _RunQueue(self, queue): | 2950 | def _RunQueue(self, queue): |
2862 | for m in getattr(self, queue): | 2951 | for m in getattr(self, queue): |
2863 | if not m.Run(self): | 2952 | if not m.Run(self): |
2864 | self.clean = False | 2953 | self._MarkUnclean() |
2865 | return False | 2954 | return False |
2866 | setattr(self, queue, []) | 2955 | setattr(self, queue, []) |
2867 | return True | 2956 | return True |
@@ -2903,14 +2992,14 @@ class MetaProject(Project): | |||
2903 | self.revisionExpr = base | 2992 | self.revisionExpr = base |
2904 | self.revisionId = None | 2993 | self.revisionId = None |
2905 | 2994 | ||
2906 | def MetaBranchSwitch(self): | 2995 | def MetaBranchSwitch(self, submodules=False): |
2907 | """ Prepare MetaProject for manifest branch switch | 2996 | """ Prepare MetaProject for manifest branch switch |
2908 | """ | 2997 | """ |
2909 | 2998 | ||
2910 | # detach and delete manifest branch, allowing a new | 2999 | # detach and delete manifest branch, allowing a new |
2911 | # branch to take over | 3000 | # branch to take over |
2912 | syncbuf = SyncBuffer(self.config, detach_head=True) | 3001 | syncbuf = SyncBuffer(self.config, detach_head=True) |
2913 | self.Sync_LocalHalf(syncbuf) | 3002 | self.Sync_LocalHalf(syncbuf, submodules=submodules) |
2914 | syncbuf.Finish() | 3003 | syncbuf.Finish() |
2915 | 3004 | ||
2916 | return GitCommand(self, | 3005 | return GitCommand(self, |