From bed8b62345e484b27e048e8f21280c5611f795df Mon Sep 17 00:00:00 2001 From: Renaud Paquay Date: Thu, 27 Sep 2018 10:46:58 -0700 Subject: Add support for long paths * Add more file i/o wrappers in platform_utils to allow using long paths (length > MAX_PATH) on Windows. * Paths using the long path syntax ("\\?\" prefix) should never escape the platform_utils API surface area, so that this specific syntax is not visible to the rest of the repo code base. * Forward many calls from os.xxx to platform_utils.xxx in various place to ensure long paths support, specifically when repo decides to delete obsolete directories. * There are more places that need to be converted to support long paths, this commit is an initial effort to unblock a few common use cases. * Also, fix remove function to handle directory symlinks Change-Id: If82ccc408e516e96ff7260be25f8fd2fe3f9571a --- platform_utils.py | 126 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 110 insertions(+), 16 deletions(-) (limited to 'platform_utils.py') diff --git a/platform_utils.py b/platform_utils.py index a3e96531..b2cc2459 100644 --- a/platform_utils.py +++ b/platform_utils.py @@ -187,10 +187,10 @@ def symlink(source, link_name): source = _validate_winpath(source) link_name = _validate_winpath(link_name) target = os.path.join(os.path.dirname(link_name), source) - if os.path.isdir(target): - platform_utils_win32.create_dirsymlink(source, link_name) + if isdir(target): + platform_utils_win32.create_dirsymlink(_makelongpath(source), link_name) else: - platform_utils_win32.create_filesymlink(source, link_name) + platform_utils_win32.create_filesymlink(_makelongpath(source), link_name) else: return os.symlink(source, link_name) @@ -220,9 +220,32 @@ def _winpath_is_valid(path): return not drive # "x:" is invalid +def _makelongpath(path): + """Return the input path normalized to support the Windows long path syntax + ("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the + MAX_PATH limit. + """ + if isWindows(): + # Note: MAX_PATH is 260, but, for directories, the maximum value is actually 246. + if len(path) < 246: + return path + if path.startswith(u"\\\\?\\"): + return path + if not os.path.isabs(path): + return path + # Append prefix and ensure unicode so that the special longpath syntax + # is supported by underlying Win32 API calls + return u"\\\\?\\" + os.path.normpath(path) + else: + return path + + def rmtree(path): + """shutil.rmtree(path) wrapper with support for long paths on Windows. + + Availability: Unix, Windows.""" if isWindows(): - shutil.rmtree(path, onerror=handle_rmtree_error) + shutil.rmtree(_makelongpath(path), onerror=handle_rmtree_error) else: shutil.rmtree(path) @@ -234,15 +257,18 @@ def handle_rmtree_error(function, path, excinfo): def rename(src, dst): + """os.rename(src, dst) wrapper with support for long paths on Windows. + + Availability: Unix, Windows.""" if isWindows(): # On Windows, rename fails if destination exists, see # https://docs.python.org/2/library/os.html#os.rename try: - os.rename(src, dst) + os.rename(_makelongpath(src), _makelongpath(dst)) except OSError as e: if e.errno == errno.EEXIST: - os.remove(dst) - os.rename(src, dst) + os.remove(_makelongpath(dst)) + os.rename(_makelongpath(src), _makelongpath(dst)) else: raise else: @@ -250,30 +276,98 @@ def rename(src, dst): def remove(path): - """Remove (delete) the file path. This is a replacement for os.remove, but - allows deleting read-only files on Windows. - """ + """Remove (delete) the file path. This is a replacement for os.remove that + allows deleting read-only files on Windows, with support for long paths and + for deleting directory symbolic links. + + Availability: Unix, Windows.""" if isWindows(): + longpath = _makelongpath(path) try: - os.remove(path) + os.remove(longpath) except OSError as e: if e.errno == errno.EACCES: - os.chmod(path, stat.S_IWRITE) - os.remove(path) + os.chmod(longpath, stat.S_IWRITE) + # Directory symbolic links must be deleted with 'rmdir'. + if islink(longpath) and isdir(longpath): + os.rmdir(longpath) + else: + os.remove(longpath) else: raise else: os.remove(path) +def walk(top, topdown=True, onerror=None, followlinks=False): + """os.walk(path) wrapper with support for long paths on Windows. + + Availability: Windows, Unix. + """ + if isWindows(): + return _walk_windows_impl(top, topdown, onerror, followlinks) + else: + return os.walk(top, topdown, onerror, followlinks) + + +def _walk_windows_impl(top, topdown, onerror, followlinks): + try: + names = listdir(top) + except error, err: + if onerror is not None: + onerror(err) + return + + dirs, nondirs = [], [] + for name in names: + if isdir(os.path.join(top, name)): + dirs.append(name) + else: + nondirs.append(name) + + if topdown: + yield top, dirs, nondirs + for name in dirs: + new_path = os.path.join(top, name) + if followlinks or not islink(new_path): + for x in _walk_windows_impl(new_path, topdown, onerror, followlinks): + yield x + if not topdown: + yield top, dirs, nondirs + + +def listdir(path): + """os.listdir(path) wrapper with support for long paths on Windows. + + Availability: Windows, Unix. + """ + return os.listdir(_makelongpath(path)) + + +def rmdir(path): + """os.rmdir(path) wrapper with support for long paths on Windows. + + Availability: Windows, Unix. + """ + os.rmdir(_makelongpath(path)) + + +def isdir(path): + """os.path.isdir(path) wrapper with support for long paths on Windows. + + Availability: Windows, Unix. + """ + return os.path.isdir(_makelongpath(path)) + + def islink(path): - """Test whether a path is a symbolic link. + """os.path.islink(path) wrapper with support for long paths on Windows. Availability: Windows, Unix. """ if isWindows(): import platform_utils_win32 - return platform_utils_win32.islink(path) + return platform_utils_win32.islink(_makelongpath(path)) else: return os.path.islink(path) @@ -288,7 +382,7 @@ def readlink(path): """ if isWindows(): import platform_utils_win32 - return platform_utils_win32.readlink(path) + return platform_utils_win32.readlink(_makelongpath(path)) else: return os.readlink(path) -- cgit v1.2.3-54-g00ecf