summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--git_command.py29
-rw-r--r--git_config.py12
-rw-r--r--git_refs.py11
-rw-r--r--manifest_xml.py3
-rw-r--r--platform_utils.py244
-rw-r--r--platform_utils_win32.py63
-rw-r--r--project.py35
-rw-r--r--subcmds/forall.py29
-rw-r--r--subcmds/gitc_delete.py4
-rw-r--r--subcmds/init.py4
-rw-r--r--subcmds/start.py10
-rw-r--r--subcmds/sync.py4
12 files changed, 373 insertions, 75 deletions
diff --git a/git_command.py b/git_command.py
index 9f7d2930..dfa6a924 100644
--- a/git_command.py
+++ b/git_command.py
@@ -14,14 +14,14 @@
14# limitations under the License. 14# limitations under the License.
15 15
16from __future__ import print_function 16from __future__ import print_function
17import fcntl
18import os 17import os
19import select
20import sys 18import sys
21import subprocess 19import subprocess
22import tempfile 20import tempfile
23from signal import SIGTERM 21from signal import SIGTERM
22
24from error import GitError 23from error import GitError
24import platform_utils
25from trace import REPO_TRACE, IsTrace, Trace 25from trace import REPO_TRACE, IsTrace, Trace
26from wrapper import Wrapper 26from wrapper import Wrapper
27 27
@@ -78,16 +78,6 @@ def terminate_ssh_clients():
78 78
79_git_version = None 79_git_version = None
80 80
81class _sfd(object):
82 """select file descriptor class"""
83 def __init__(self, fd, dest, std_name):
84 assert std_name in ('stdout', 'stderr')
85 self.fd = fd
86 self.dest = dest
87 self.std_name = std_name
88 def fileno(self):
89 return self.fd.fileno()
90
91class _GitCall(object): 81class _GitCall(object):
92 def version(self): 82 def version(self):
93 p = GitCommand(None, ['--version'], capture_stdout=True) 83 p = GitCommand(None, ['--version'], capture_stdout=True)
@@ -253,19 +243,16 @@ class GitCommand(object):
253 243
254 def _CaptureOutput(self): 244 def _CaptureOutput(self):
255 p = self.process 245 p = self.process
256 s_in = [_sfd(p.stdout, sys.stdout, 'stdout'), 246 s_in = platform_utils.FileDescriptorStreams.create()
257 _sfd(p.stderr, sys.stderr, 'stderr')] 247 s_in.add(p.stdout, sys.stdout, 'stdout')
248 s_in.add(p.stderr, sys.stderr, 'stderr')
258 self.stdout = '' 249 self.stdout = ''
259 self.stderr = '' 250 self.stderr = ''
260 251
261 for s in s_in: 252 while not s_in.is_done:
262 flags = fcntl.fcntl(s.fd, fcntl.F_GETFL) 253 in_ready = s_in.select()
263 fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
264
265 while s_in:
266 in_ready, _, _ = select.select(s_in, [], [])
267 for s in in_ready: 254 for s in in_ready:
268 buf = s.fd.read(4096) 255 buf = s.read()
269 if not buf: 256 if not buf:
270 s_in.remove(s) 257 s_in.remove(s)
271 continue 258 continue
diff --git a/git_config.py b/git_config.py
index fb4377cf..8f666e6d 100644
--- a/git_config.py
+++ b/git_config.py
@@ -51,16 +51,24 @@ else:
51from git_command import GitCommand 51from git_command import GitCommand
52from git_command import ssh_sock 52from git_command import ssh_sock
53from git_command import terminate_ssh_clients 53from git_command import terminate_ssh_clients
54from git_refs import R_CHANGES, R_HEADS, R_TAGS
54 55
55R_HEADS = 'refs/heads/'
56R_TAGS = 'refs/tags/'
57ID_RE = re.compile(r'^[0-9a-f]{40}$') 56ID_RE = re.compile(r'^[0-9a-f]{40}$')
58 57
59REVIEW_CACHE = dict() 58REVIEW_CACHE = dict()
60 59
60def IsChange(rev):
61 return rev.startswith(R_CHANGES)
62
61def IsId(rev): 63def IsId(rev):
62 return ID_RE.match(rev) 64 return ID_RE.match(rev)
63 65
66def IsTag(rev):
67 return rev.startswith(R_TAGS)
68
69def IsImmutable(rev):
70 return IsChange(rev) or IsId(rev) or IsTag(rev)
71
64def _key(name): 72def _key(name):
65 parts = name.split('.') 73 parts = name.split('.')
66 if len(parts) < 2: 74 if len(parts) < 2:
diff --git a/git_refs.py b/git_refs.py
index 3c266061..58c838a6 100644
--- a/git_refs.py
+++ b/git_refs.py
@@ -16,11 +16,12 @@
16import os 16import os
17from trace import Trace 17from trace import Trace
18 18
19HEAD = 'HEAD' 19HEAD = 'HEAD'
20R_HEADS = 'refs/heads/' 20R_CHANGES = 'refs/changes/'
21R_TAGS = 'refs/tags/' 21R_HEADS = 'refs/heads/'
22R_PUB = 'refs/published/' 22R_TAGS = 'refs/tags/'
23R_M = 'refs/remotes/m/' 23R_PUB = 'refs/published/'
24R_M = 'refs/remotes/m/'
24 25
25 26
26class GitRefs(object): 27class GitRefs(object):
diff --git a/manifest_xml.py b/manifest_xml.py
index 55d25a79..05651c6c 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -32,6 +32,7 @@ else:
32import gitc_utils 32import gitc_utils
33from git_config import GitConfig 33from git_config import GitConfig
34from git_refs import R_HEADS, HEAD 34from git_refs import R_HEADS, HEAD
35import platform_utils
35from project import RemoteSpec, Project, MetaProject 36from project import RemoteSpec, Project, MetaProject
36from error import ManifestParseError, ManifestInvalidRevisionError 37from error import ManifestParseError, ManifestInvalidRevisionError
37 38
@@ -166,7 +167,7 @@ class XmlManifest(object):
166 try: 167 try:
167 if os.path.lexists(self.manifestFile): 168 if os.path.lexists(self.manifestFile):
168 os.remove(self.manifestFile) 169 os.remove(self.manifestFile)
169 os.symlink(os.path.join('manifests', name), self.manifestFile) 170 platform_utils.symlink(os.path.join('manifests', name), self.manifestFile)
170 except OSError as e: 171 except OSError as e:
171 raise ManifestParseError('cannot link manifest %s: %s' % (name, str(e))) 172 raise ManifestParseError('cannot link manifest %s: %s' % (name, str(e)))
172 173
diff --git a/platform_utils.py b/platform_utils.py
new file mode 100644
index 00000000..e0fa9dcc
--- /dev/null
+++ b/platform_utils.py
@@ -0,0 +1,244 @@
1#
2# Copyright (C) 2016 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import errno
17import os
18import platform
19import select
20import shutil
21import stat
22
23from Queue import Queue
24from threading import Thread
25
26
27def isWindows():
28 """ Returns True when running with the native port of Python for Windows,
29 False when running on any other platform (including the Cygwin port of
30 Python).
31 """
32 # Note: The cygwin port of Python returns "CYGWIN_NT_xxx"
33 return platform.system() == "Windows"
34
35
36class FileDescriptorStreams(object):
37 """ Platform agnostic abstraction enabling non-blocking I/O over a
38 collection of file descriptors. This abstraction is required because
39 fctnl(os.O_NONBLOCK) is not supported on Windows.
40 """
41 @classmethod
42 def create(cls):
43 """ Factory method: instantiates the concrete class according to the
44 current platform.
45 """
46 if isWindows():
47 return _FileDescriptorStreamsThreads()
48 else:
49 return _FileDescriptorStreamsNonBlocking()
50
51 def __init__(self):
52 self.streams = []
53
54 def add(self, fd, dest, std_name):
55 """ Wraps an existing file descriptor as a stream.
56 """
57 self.streams.append(self._create_stream(fd, dest, std_name))
58
59 def remove(self, stream):
60 """ Removes a stream, when done with it.
61 """
62 self.streams.remove(stream)
63
64 @property
65 def is_done(self):
66 """ Returns True when all streams have been processed.
67 """
68 return len(self.streams) == 0
69
70 def select(self):
71 """ Returns the set of streams that have data available to read.
72 The returned streams each expose a read() and a close() method.
73 When done with a stream, call the remove(stream) method.
74 """
75 raise NotImplementedError
76
77 def _create_stream(fd, dest, std_name):
78 """ Creates a new stream wrapping an existing file descriptor.
79 """
80 raise NotImplementedError
81
82
83class _FileDescriptorStreamsNonBlocking(FileDescriptorStreams):
84 """ Implementation of FileDescriptorStreams for platforms that support
85 non blocking I/O.
86 """
87 class Stream(object):
88 """ Encapsulates a file descriptor """
89 def __init__(self, fd, dest, std_name):
90 self.fd = fd
91 self.dest = dest
92 self.std_name = std_name
93 self.set_non_blocking()
94
95 def set_non_blocking(self):
96 import fcntl
97 flags = fcntl.fcntl(self.fd, fcntl.F_GETFL)
98 fcntl.fcntl(self.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
99
100 def fileno(self):
101 return self.fd.fileno()
102
103 def read(self):
104 return self.fd.read(4096)
105
106 def close(self):
107 self.fd.close()
108
109 def _create_stream(self, fd, dest, std_name):
110 return self.Stream(fd, dest, std_name)
111
112 def select(self):
113 ready_streams, _, _ = select.select(self.streams, [], [])
114 return ready_streams
115
116
117class _FileDescriptorStreamsThreads(FileDescriptorStreams):
118 """ Implementation of FileDescriptorStreams for platforms that don't support
119 non blocking I/O. This implementation requires creating threads issuing
120 blocking read operations on file descriptors.
121 """
122 def __init__(self):
123 super(_FileDescriptorStreamsThreads, self).__init__()
124 # The queue is shared accross all threads so we can simulate the
125 # behavior of the select() function
126 self.queue = Queue(10) # Limit incoming data from streams
127
128 def _create_stream(self, fd, dest, std_name):
129 return self.Stream(fd, dest, std_name, self.queue)
130
131 def select(self):
132 # Return only one stream at a time, as it is the most straighforward
133 # thing to do and it is compatible with the select() function.
134 item = self.queue.get()
135 stream = item.stream
136 stream.data = item.data
137 return [stream]
138
139 class QueueItem(object):
140 """ Item put in the shared queue """
141 def __init__(self, stream, data):
142 self.stream = stream
143 self.data = data
144
145 class Stream(object):
146 """ Encapsulates a file descriptor """
147 def __init__(self, fd, dest, std_name, queue):
148 self.fd = fd
149 self.dest = dest
150 self.std_name = std_name
151 self.queue = queue
152 self.data = None
153 self.thread = Thread(target=self.read_to_queue)
154 self.thread.daemon = True
155 self.thread.start()
156
157 def close(self):
158 self.fd.close()
159
160 def read(self):
161 data = self.data
162 self.data = None
163 return data
164
165 def read_to_queue(self):
166 """ The thread function: reads everything from the file descriptor into
167 the shared queue and terminates when reaching EOF.
168 """
169 for line in iter(self.fd.readline, b''):
170 self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, line))
171 self.fd.close()
172 self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, None))
173
174
175def symlink(source, link_name):
176 """Creates a symbolic link pointing to source named link_name.
177 Note: On Windows, source must exist on disk, as the implementation needs
178 to know whether to create a "File" or a "Directory" symbolic link.
179 """
180 if isWindows():
181 import platform_utils_win32
182 source = _validate_winpath(source)
183 link_name = _validate_winpath(link_name)
184 target = os.path.join(os.path.dirname(link_name), source)
185 if os.path.isdir(target):
186 platform_utils_win32.create_dirsymlink(source, link_name)
187 else:
188 platform_utils_win32.create_filesymlink(source, link_name)
189 else:
190 return os.symlink(source, link_name)
191
192
193def _validate_winpath(path):
194 path = os.path.normpath(path)
195 if _winpath_is_valid(path):
196 return path
197 raise ValueError("Path \"%s\" must be a relative path or an absolute "
198 "path starting with a drive letter".format(path))
199
200
201def _winpath_is_valid(path):
202 """Windows only: returns True if path is relative (e.g. ".\\foo") or is
203 absolute including a drive letter (e.g. "c:\\foo"). Returns False if path
204 is ambiguous (e.g. "x:foo" or "\\foo").
205 """
206 assert isWindows()
207 path = os.path.normpath(path)
208 drive, tail = os.path.splitdrive(path)
209 if tail:
210 if not drive:
211 return tail[0] != os.sep # "\\foo" is invalid
212 else:
213 return tail[0] == os.sep # "x:foo" is invalid
214 else:
215 return not drive # "x:" is invalid
216
217
218def rmtree(path):
219 if isWindows():
220 shutil.rmtree(path, onerror=handle_rmtree_error)
221 else:
222 shutil.rmtree(path)
223
224
225def handle_rmtree_error(function, path, excinfo):
226 # Allow deleting read-only files
227 os.chmod(path, stat.S_IWRITE)
228 function(path)
229
230
231def rename(src, dst):
232 if isWindows():
233 # On Windows, rename fails if destination exists, see
234 # https://docs.python.org/2/library/os.html#os.rename
235 try:
236 os.rename(src, dst)
237 except OSError as e:
238 if e.errno == errno.EEXIST:
239 os.remove(dst)
240 os.rename(src, dst)
241 else:
242 raise
243 else:
244 os.rename(src, dst)
diff --git a/platform_utils_win32.py b/platform_utils_win32.py
new file mode 100644
index 00000000..02fb013a
--- /dev/null
+++ b/platform_utils_win32.py
@@ -0,0 +1,63 @@
1#
2# Copyright (C) 2016 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import errno
17
18from ctypes import WinDLL, get_last_error, FormatError, WinError
19from ctypes.wintypes import BOOL, LPCWSTR, DWORD
20
21kernel32 = WinDLL('kernel32', use_last_error=True)
22
23# Win32 error codes
24ERROR_SUCCESS = 0
25ERROR_PRIVILEGE_NOT_HELD = 1314
26
27# Win32 API entry points
28CreateSymbolicLinkW = kernel32.CreateSymbolicLinkW
29CreateSymbolicLinkW.restype = BOOL
30CreateSymbolicLinkW.argtypes = (LPCWSTR, # lpSymlinkFileName In
31 LPCWSTR, # lpTargetFileName In
32 DWORD) # dwFlags In
33
34# Symbolic link creation flags
35SYMBOLIC_LINK_FLAG_FILE = 0x00
36SYMBOLIC_LINK_FLAG_DIRECTORY = 0x01
37
38
39def create_filesymlink(source, link_name):
40 """Creates a Windows file symbolic link source pointing to link_name."""
41 _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_FILE)
42
43
44def create_dirsymlink(source, link_name):
45 """Creates a Windows directory symbolic link source pointing to link_name.
46 """
47 _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_DIRECTORY)
48
49
50def _create_symlink(source, link_name, dwFlags):
51 # Note: Win32 documentation for CreateSymbolicLink is incorrect.
52 # On success, the function returns "1".
53 # On error, the function returns some random value (e.g. 1280).
54 # The best bet seems to use "GetLastError" and check for error/success.
55 CreateSymbolicLinkW(link_name, source, dwFlags)
56 code = get_last_error()
57 if code != ERROR_SUCCESS:
58 error_desc = FormatError(code).strip()
59 if code == ERROR_PRIVILEGE_NOT_HELD:
60 raise OSError(errno.EPERM, error_desc, link_name)
61 error_desc = 'Error creating symbolic link %s: %s'.format(
62 link_name, error_desc)
63 raise WinError(code, error_desc)
diff --git a/project.py b/project.py
index c2cccb4f..4eca9b67 100644
--- a/project.py
+++ b/project.py
@@ -35,6 +35,7 @@ from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
35from error import GitError, HookError, UploadError, DownloadError 35from error import GitError, HookError, UploadError, DownloadError
36from error import ManifestInvalidRevisionError 36from error import ManifestInvalidRevisionError
37from error import NoManifestException 37from error import NoManifestException
38import platform_utils
38from trace import IsTrace, Trace 39from trace import IsTrace, Trace
39 40
40from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M 41from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
@@ -62,7 +63,7 @@ def _lwrite(path, content):
62 fd.close() 63 fd.close()
63 64
64 try: 65 try:
65 os.rename(lock, path) 66 platform_utils.rename(lock, path)
66 except OSError: 67 except OSError:
67 os.remove(lock) 68 os.remove(lock)
68 raise 69 raise
@@ -283,7 +284,7 @@ class _LinkFile(object):
283 dest_dir = os.path.dirname(absDest) 284 dest_dir = os.path.dirname(absDest)
284 if not os.path.isdir(dest_dir): 285 if not os.path.isdir(dest_dir):
285 os.makedirs(dest_dir) 286 os.makedirs(dest_dir)
286 os.symlink(relSrc, absDest) 287 platform_utils.symlink(relSrc, absDest)
287 except IOError: 288 except IOError:
288 _error('Cannot link file %s to %s', relSrc, absDest) 289 _error('Cannot link file %s to %s', relSrc, absDest)
289 290
@@ -1288,7 +1289,7 @@ class Project(object):
1288 1289
1289 need_to_fetch = not (optimized_fetch and 1290 need_to_fetch = not (optimized_fetch and
1290 (ID_RE.match(self.revisionExpr) and 1291 (ID_RE.match(self.revisionExpr) and
1291 self._CheckForSha1())) 1292 self._CheckForImmutableRevision()))
1292 if (need_to_fetch and 1293 if (need_to_fetch and
1293 not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir, 1294 not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
1294 current_branch_only=current_branch_only, 1295 current_branch_only=current_branch_only,
@@ -1898,7 +1899,7 @@ class Project(object):
1898 1899
1899 1900
1900# Direct Git Commands ## 1901# Direct Git Commands ##
1901 def _CheckForSha1(self): 1902 def _CheckForImmutableRevision(self):
1902 try: 1903 try:
1903 # if revision (sha or tag) is not present then following function 1904 # if revision (sha or tag) is not present then following function
1904 # throws an error. 1905 # throws an error.
@@ -1952,7 +1953,9 @@ class Project(object):
1952 tag_name = self.revisionExpr[len(R_TAGS):] 1953 tag_name = self.revisionExpr[len(R_TAGS):]
1953 1954
1954 if is_sha1 or tag_name is not None: 1955 if is_sha1 or tag_name is not None:
1955 if self._CheckForSha1(): 1956 if self._CheckForImmutableRevision():
1957 print('Skipped fetching project %s (already have persistent ref)'
1958 % self.name)
1956 return True 1959 return True
1957 if is_sha1 and not depth: 1960 if is_sha1 and not depth:
1958 # When syncing a specific commit and --depth is not set: 1961 # When syncing a specific commit and --depth is not set:
@@ -2108,7 +2111,7 @@ class Project(object):
2108 # We just synced the upstream given branch; verify we 2111 # We just synced the upstream given branch; verify we
2109 # got what we wanted, else trigger a second run of all 2112 # got what we wanted, else trigger a second run of all
2110 # refs. 2113 # refs.
2111 if not self._CheckForSha1(): 2114 if not self._CheckForImmutableRevision():
2112 if current_branch_only and depth: 2115 if current_branch_only and depth:
2113 # Sync the current branch only with depth set to None 2116 # Sync the current branch only with depth set to None
2114 return self._RemoteFetch(name=name, 2117 return self._RemoteFetch(name=name,
@@ -2211,7 +2214,7 @@ class Project(object):
2211 2214
2212 if os.path.exists(tmpPath): 2215 if os.path.exists(tmpPath):
2213 if curlret == 0 and self._IsValidBundle(tmpPath, quiet): 2216 if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
2214 os.rename(tmpPath, dstPath) 2217 platform_utils.rename(tmpPath, dstPath)
2215 return True 2218 return True
2216 else: 2219 else:
2217 os.remove(tmpPath) 2220 os.remove(tmpPath)
@@ -2312,10 +2315,10 @@ class Project(object):
2312 print("Retrying clone after deleting %s" % 2315 print("Retrying clone after deleting %s" %
2313 self.gitdir, file=sys.stderr) 2316 self.gitdir, file=sys.stderr)
2314 try: 2317 try:
2315 shutil.rmtree(os.path.realpath(self.gitdir)) 2318 platform_utils.rmtree(os.path.realpath(self.gitdir))
2316 if self.worktree and os.path.exists(os.path.realpath 2319 if self.worktree and os.path.exists(os.path.realpath
2317 (self.worktree)): 2320 (self.worktree)):
2318 shutil.rmtree(os.path.realpath(self.worktree)) 2321 platform_utils.rmtree(os.path.realpath(self.worktree))
2319 return self._InitGitDir(mirror_git=mirror_git, force_sync=False) 2322 return self._InitGitDir(mirror_git=mirror_git, force_sync=False)
2320 except: 2323 except:
2321 raise e 2324 raise e
@@ -2357,9 +2360,9 @@ class Project(object):
2357 self.config.SetString('core.bare', None) 2360 self.config.SetString('core.bare', None)
2358 except Exception: 2361 except Exception:
2359 if init_obj_dir and os.path.exists(self.objdir): 2362 if init_obj_dir and os.path.exists(self.objdir):
2360 shutil.rmtree(self.objdir) 2363 platform_utils.rmtree(self.objdir)
2361 if init_git_dir and os.path.exists(self.gitdir): 2364 if init_git_dir and os.path.exists(self.gitdir):
2362 shutil.rmtree(self.gitdir) 2365 platform_utils.rmtree(self.gitdir)
2363 raise 2366 raise
2364 2367
2365 def _UpdateHooks(self): 2368 def _UpdateHooks(self):
@@ -2393,7 +2396,8 @@ class Project(object):
2393 self.relpath, name) 2396 self.relpath, name)
2394 continue 2397 continue
2395 try: 2398 try:
2396 os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst) 2399 platform_utils.symlink(
2400 os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
2397 except OSError as e: 2401 except OSError as e:
2398 if e.errno == errno.EPERM: 2402 if e.errno == errno.EPERM:
2399 raise GitError('filesystem must support symlinks') 2403 raise GitError('filesystem must support symlinks')
@@ -2492,7 +2496,8 @@ class Project(object):
2492 os.makedirs(src) 2496 os.makedirs(src)
2493 2497
2494 if name in to_symlink: 2498 if name in to_symlink:
2495 os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst) 2499 platform_utils.symlink(
2500 os.path.relpath(src, os.path.dirname(dst)), dst)
2496 elif copy_all and not os.path.islink(dst): 2501 elif copy_all and not os.path.islink(dst):
2497 if os.path.isdir(src): 2502 if os.path.isdir(src):
2498 shutil.copytree(src, dst) 2503 shutil.copytree(src, dst)
@@ -2527,7 +2532,7 @@ class Project(object):
2527 except GitError as e: 2532 except GitError as e:
2528 if force_sync: 2533 if force_sync:
2529 try: 2534 try:
2530 shutil.rmtree(dotgit) 2535 platform_utils.rmtree(dotgit)
2531 return self._InitWorkTree(force_sync=False, submodules=submodules) 2536 return self._InitWorkTree(force_sync=False, submodules=submodules)
2532 except: 2537 except:
2533 raise e 2538 raise e
@@ -2547,7 +2552,7 @@ class Project(object):
2547 self._CopyAndLinkFiles() 2552 self._CopyAndLinkFiles()
2548 except Exception: 2553 except Exception:
2549 if init_dotgit: 2554 if init_dotgit:
2550 shutil.rmtree(dotgit) 2555 platform_utils.rmtree(dotgit)
2551 raise 2556 raise
2552 2557
2553 def _gitdir_path(self, path): 2558 def _gitdir_path(self, path):
diff --git a/subcmds/forall.py b/subcmds/forall.py
index 07ee8d58..2c12c55f 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -15,17 +15,16 @@
15 15
16from __future__ import print_function 16from __future__ import print_function
17import errno 17import errno
18import fcntl
19import multiprocessing 18import multiprocessing
20import re 19import re
21import os 20import os
22import select
23import signal 21import signal
24import sys 22import sys
25import subprocess 23import subprocess
26 24
27from color import Coloring 25from color import Coloring
28from command import Command, MirrorSafeCommand 26from command import Command, MirrorSafeCommand
27import platform_utils
29 28
30_CAN_COLOR = [ 29_CAN_COLOR = [
31 'branch', 30 'branch',
@@ -344,35 +343,25 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
344 if opt.project_header: 343 if opt.project_header:
345 out = ForallColoring(config) 344 out = ForallColoring(config)
346 out.redirect(sys.stdout) 345 out.redirect(sys.stdout)
347 class sfd(object):
348 def __init__(self, fd, dest):
349 self.fd = fd
350 self.dest = dest
351 def fileno(self):
352 return self.fd.fileno()
353
354 empty = True 346 empty = True
355 errbuf = '' 347 errbuf = ''
356 348
357 p.stdin.close() 349 p.stdin.close()
358 s_in = [sfd(p.stdout, sys.stdout), 350 s_in = platform_utils.FileDescriptorStreams.create()
359 sfd(p.stderr, sys.stderr)] 351 s_in.add(p.stdout, sys.stdout, 'stdout')
360 352 s_in.add(p.stderr, sys.stderr, 'stderr')
361 for s in s_in:
362 flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
363 fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
364 353
365 while s_in: 354 while not s_in.is_done:
366 in_ready, _out_ready, _err_ready = select.select(s_in, [], []) 355 in_ready = s_in.select()
367 for s in in_ready: 356 for s in in_ready:
368 buf = s.fd.read(4096) 357 buf = s.read()
369 if not buf: 358 if not buf:
370 s.fd.close() 359 s.close()
371 s_in.remove(s) 360 s_in.remove(s)
372 continue 361 continue
373 362
374 if not opt.verbose: 363 if not opt.verbose:
375 if s.fd != p.stdout: 364 if s.std_name == 'stderr':
376 errbuf += buf 365 errbuf += buf
377 continue 366 continue
378 367
diff --git a/subcmds/gitc_delete.py b/subcmds/gitc_delete.py
index 19caac5a..54f62f46 100644
--- a/subcmds/gitc_delete.py
+++ b/subcmds/gitc_delete.py
@@ -14,10 +14,10 @@
14# limitations under the License. 14# limitations under the License.
15 15
16from __future__ import print_function 16from __future__ import print_function
17import shutil
18import sys 17import sys
19 18
20from command import Command, GitcClientCommand 19from command import Command, GitcClientCommand
20import platform_utils
21 21
22from pyversion import is_python3 22from pyversion import is_python3
23if not is_python3(): 23if not is_python3():
@@ -50,4 +50,4 @@ and all locally downloaded sources.
50 if not response == 'yes': 50 if not response == 'yes':
51 print('Response was not "yes"\n Exiting...') 51 print('Response was not "yes"\n Exiting...')
52 sys.exit(1) 52 sys.exit(1)
53 shutil.rmtree(self.gitc_manifest.gitc_client_dir) 53 platform_utils.rmtree(self.gitc_manifest.gitc_client_dir)
diff --git a/subcmds/init.py b/subcmds/init.py
index 46cdd23a..eeddca06 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -17,7 +17,6 @@ from __future__ import print_function
17import os 17import os
18import platform 18import platform
19import re 19import re
20import shutil
21import sys 20import sys
22 21
23from pyversion import is_python3 22from pyversion import is_python3
@@ -35,6 +34,7 @@ from error import ManifestParseError
35from project import SyncBuffer 34from project import SyncBuffer
36from git_config import GitConfig 35from git_config import GitConfig
37from git_command import git_require, MIN_GIT_VERSION 36from git_command import git_require, MIN_GIT_VERSION
37import platform_utils
38 38
39class Init(InteractiveCommand, MirrorSafeCommand): 39class Init(InteractiveCommand, MirrorSafeCommand):
40 common = True 40 common = True
@@ -252,7 +252,7 @@ to update the working directory files.
252 # Better delete the manifest git dir if we created it; otherwise next 252 # Better delete the manifest git dir if we created it; otherwise next
253 # time (when user fixes problems) we won't go through the "is_new" logic. 253 # time (when user fixes problems) we won't go through the "is_new" logic.
254 if is_new: 254 if is_new:
255 shutil.rmtree(m.gitdir) 255 platform_utils.rmtree(m.gitdir)
256 sys.exit(1) 256 sys.exit(1)
257 257
258 if opt.manifest_branch: 258 if opt.manifest_branch:
diff --git a/subcmds/start.py b/subcmds/start.py
index 290b6897..c3ec303e 100644
--- a/subcmds/start.py
+++ b/subcmds/start.py
@@ -18,7 +18,7 @@ import os
18import sys 18import sys
19 19
20from command import Command 20from command import Command
21from git_config import IsId 21from git_config import IsImmutable
22from git_command import git 22from git_command import git
23import gitc_utils 23import gitc_utils
24from progress import Progress 24from progress import Progress
@@ -96,11 +96,11 @@ revision specified in the manifest.
96 project.Sync_LocalHalf(sync_buf) 96 project.Sync_LocalHalf(sync_buf)
97 project.revisionId = gitc_project.old_revision 97 project.revisionId = gitc_project.old_revision
98 98
99 # If the current revision is a specific SHA1 then we can't push back 99 # If the current revision is immutable, such as a SHA1, a tag or
100 # to it; so substitute with dest_branch if defined, or with manifest 100 # a change, then we can't push back to it. Substitute with
101 # default revision instead. 101 # dest_branch, if defined; or with manifest default revision instead.
102 branch_merge = '' 102 branch_merge = ''
103 if IsId(project.revisionExpr): 103 if IsImmutable(project.revisionExpr):
104 if project.dest_branch: 104 if project.dest_branch:
105 branch_merge = project.dest_branch 105 branch_merge = project.dest_branch
106 else: 106 else:
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 8de730bc..b88c596d 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -19,7 +19,6 @@ import netrc
19from optparse import SUPPRESS_HELP 19from optparse import SUPPRESS_HELP
20import os 20import os
21import re 21import re
22import shutil
23import socket 22import socket
24import subprocess 23import subprocess
25import sys 24import sys
@@ -73,6 +72,7 @@ from project import Project
73from project import RemoteSpec 72from project import RemoteSpec
74from command import Command, MirrorSafeCommand 73from command import Command, MirrorSafeCommand
75from error import RepoChangedException, GitError, ManifestParseError 74from error import RepoChangedException, GitError, ManifestParseError
75import platform_utils
76from project import SyncBuffer 76from project import SyncBuffer
77from progress import Progress 77from progress import Progress
78from wrapper import Wrapper 78from wrapper import Wrapper
@@ -475,7 +475,7 @@ later is required to fix a server side protocol bug.
475 # working git repository around. There shouldn't be any git projects here, 475 # working git repository around. There shouldn't be any git projects here,
476 # so rmtree works. 476 # so rmtree works.
477 try: 477 try:
478 shutil.rmtree(os.path.join(path, '.git')) 478 platform_utils.rmtree(os.path.join(path, '.git'))
479 except OSError: 479 except OSError:
480 print('Failed to remove %s' % os.path.join(path, '.git'), file=sys.stderr) 480 print('Failed to remove %s' % os.path.join(path, '.git'), file=sys.stderr)
481 print('error: Failed to delete obsolete path %s' % path, file=sys.stderr) 481 print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)