From 2e7029116204cf2d6f516e4514091f0b492bc689 Mon Sep 17 00:00:00 2001 From: Renaud Paquay Date: Tue, 1 Nov 2016 11:23:38 -0700 Subject: Make "git command" and "forall" work on Windows Python on Windows does not support non blocking file operations. To workaround this issue, we instead use Threads and a Queue to simulate non-blocking calls. This is happens only when running with the native Windows version of Python, meaning Linux and Cygwin are not affected by this change. Change-Id: I4ce23827b096c5138f67a85c721f58a12279bb6f --- subcmds/forall.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) (limited to 'subcmds') diff --git a/subcmds/forall.py b/subcmds/forall.py index 07ee8d58..2c12c55f 100644 --- a/subcmds/forall.py +++ b/subcmds/forall.py @@ -15,17 +15,16 @@ from __future__ import print_function import errno -import fcntl import multiprocessing import re import os -import select import signal import sys import subprocess from color import Coloring from command import Command, MirrorSafeCommand +import platform_utils _CAN_COLOR = [ 'branch', @@ -344,35 +343,25 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config): if opt.project_header: out = ForallColoring(config) out.redirect(sys.stdout) - class sfd(object): - def __init__(self, fd, dest): - self.fd = fd - self.dest = dest - def fileno(self): - return self.fd.fileno() - empty = True errbuf = '' p.stdin.close() - s_in = [sfd(p.stdout, sys.stdout), - sfd(p.stderr, sys.stderr)] - - for s in s_in: - flags = fcntl.fcntl(s.fd, fcntl.F_GETFL) - fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + s_in = platform_utils.FileDescriptorStreams.create() + s_in.add(p.stdout, sys.stdout, 'stdout') + s_in.add(p.stderr, sys.stderr, 'stderr') - while s_in: - in_ready, _out_ready, _err_ready = select.select(s_in, [], []) + while not s_in.is_done: + in_ready = s_in.select() for s in in_ready: - buf = s.fd.read(4096) + buf = s.read() if not buf: - s.fd.close() + s.close() s_in.remove(s) continue if not opt.verbose: - if s.fd != p.stdout: + if s.std_name == 'stderr': errbuf += buf continue -- cgit v1.2.3-54-g00ecf From a65adf74f990eeac0d90011476376c7239cb7af5 Mon Sep 17 00:00:00 2001 From: Renaud Paquay Date: Thu, 3 Nov 2016 10:37:53 -0700 Subject: Workaround shutil.rmtree limitation on Windows By default, shutil.rmtree raises an exception when deleting readonly files on Windows. Replace all shutil.rmtree with platform_utils.rmtree, which adds an error handler to make files read-write when they can't be deleted. Change-Id: I9cfea9a7b3703fb16a82cf69331540c2c179ed53 --- platform_utils.py | 15 +++++++++++++++ project.py | 12 ++++++------ subcmds/gitc_delete.py | 4 ++-- subcmds/init.py | 4 ++-- subcmds/sync.py | 4 ++-- 5 files changed, 27 insertions(+), 12 deletions(-) (limited to 'subcmds') diff --git a/platform_utils.py b/platform_utils.py index f4dfa0b1..4417c5a3 100644 --- a/platform_utils.py +++ b/platform_utils.py @@ -16,6 +16,8 @@ import os import platform import select +import shutil +import stat from Queue import Queue from threading import Thread @@ -210,3 +212,16 @@ def _winpath_is_valid(path): return tail[0] == os.sep # "x:foo" is invalid else: return not drive # "x:" is invalid + + +def rmtree(path): + if isWindows(): + shutil.rmtree(path, onerror=handle_rmtree_error) + else: + shutil.rmtree(path) + + +def handle_rmtree_error(function, path, excinfo): + # Allow deleting read-only files + os.chmod(path, stat.S_IWRITE) + function(path) diff --git a/project.py b/project.py index de5c7915..ba18337b 100644 --- a/project.py +++ b/project.py @@ -2299,10 +2299,10 @@ class Project(object): print("Retrying clone after deleting %s" % self.gitdir, file=sys.stderr) try: - shutil.rmtree(os.path.realpath(self.gitdir)) + platform_utils.rmtree(os.path.realpath(self.gitdir)) if self.worktree and os.path.exists(os.path.realpath (self.worktree)): - shutil.rmtree(os.path.realpath(self.worktree)) + platform_utils.rmtree(os.path.realpath(self.worktree)) return self._InitGitDir(mirror_git=mirror_git, force_sync=False) except: raise e @@ -2344,9 +2344,9 @@ class Project(object): self.config.SetString('core.bare', None) except Exception: if init_obj_dir and os.path.exists(self.objdir): - shutil.rmtree(self.objdir) + platform_utils.rmtree(self.objdir) if init_git_dir and os.path.exists(self.gitdir): - shutil.rmtree(self.gitdir) + platform_utils.rmtree(self.gitdir) raise def _UpdateHooks(self): @@ -2516,7 +2516,7 @@ class Project(object): except GitError as e: if force_sync: try: - shutil.rmtree(dotgit) + platform_utils.rmtree(dotgit) return self._InitWorkTree(force_sync=False, submodules=submodules) except: raise e @@ -2536,7 +2536,7 @@ class Project(object): self._CopyAndLinkFiles() except Exception: if init_dotgit: - shutil.rmtree(dotgit) + platform_utils.rmtree(dotgit) raise def _gitdir_path(self, path): diff --git a/subcmds/gitc_delete.py b/subcmds/gitc_delete.py index 19caac5a..54f62f46 100644 --- a/subcmds/gitc_delete.py +++ b/subcmds/gitc_delete.py @@ -14,10 +14,10 @@ # limitations under the License. from __future__ import print_function -import shutil import sys from command import Command, GitcClientCommand +import platform_utils from pyversion import is_python3 if not is_python3(): @@ -50,4 +50,4 @@ and all locally downloaded sources. if not response == 'yes': print('Response was not "yes"\n Exiting...') sys.exit(1) - shutil.rmtree(self.gitc_manifest.gitc_client_dir) + platform_utils.rmtree(self.gitc_manifest.gitc_client_dir) diff --git a/subcmds/init.py b/subcmds/init.py index 65dfd1fd..e6470916 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -17,7 +17,6 @@ from __future__ import print_function import os import platform import re -import shutil import sys from pyversion import is_python3 @@ -35,6 +34,7 @@ from error import ManifestParseError from project import SyncBuffer from git_config import GitConfig from git_command import git_require, MIN_GIT_VERSION +import platform_utils class Init(InteractiveCommand, MirrorSafeCommand): common = True @@ -252,7 +252,7 @@ to update the working directory files. # Better delete the manifest git dir if we created it; otherwise next # time (when user fixes problems) we won't go through the "is_new" logic. if is_new: - shutil.rmtree(m.gitdir) + platform_utils.rmtree(m.gitdir) sys.exit(1) if opt.manifest_branch: diff --git a/subcmds/sync.py b/subcmds/sync.py index ef023274..797fc403 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -19,7 +19,6 @@ import netrc from optparse import SUPPRESS_HELP import os import re -import shutil import socket import subprocess import sys @@ -73,6 +72,7 @@ from project import Project from project import RemoteSpec from command import Command, MirrorSafeCommand from error import RepoChangedException, GitError, ManifestParseError +import platform_utils from project import SyncBuffer from progress import Progress from wrapper import Wrapper @@ -473,7 +473,7 @@ later is required to fix a server side protocol bug. # working git repository around. There shouldn't be any git projects here, # so rmtree works. try: - shutil.rmtree(os.path.join(path, '.git')) + platform_utils.rmtree(os.path.join(path, '.git')) except OSError: print('Failed to remove %s' % os.path.join(path, '.git'), file=sys.stderr) print('error: Failed to delete obsolete path %s' % path, file=sys.stderr) -- cgit v1.2.3-54-g00ecf