Add support for long paths

* Add more file i/o wrappers in platform_utils to allow using
  long paths (length > MAX_PATH) on Windows.

* Paths using the long path syntax ("\\?\" prefix) should never
  escape the platform_utils API surface area, so that this
  specific syntax is not visible to the rest of the repo code base.

* Forward many calls from os.xxx to platform_utils.xxx in various place
  to ensure long paths support, specifically when repo decides to delete
  obsolete directories.

* There are more places that need to be converted to support long paths,
  this commit is an initial effort to unblock a few common use cases.

* Also, fix remove function to handle directory symlinks

Change-Id: If82ccc408e516e96ff7260be25f8fd2fe3f9571a

1 files changed, 110 insertions, 16 deletions

diff --git a/platform_utils.py b/platform_utils.py
index a3e96531..b2cc2459 100644
--- a/platform_utils.py
+++ b/platform_utils.py
@@ -187,10 +187,10 @@ def symlink(source, link_name):
     source = _validate_winpath(source)
     link_name = _validate_winpath(link_name)
     target = os.path.join(os.path.dirname(link_name), source)
-    if os.path.isdir(target):
-      platform_utils_win32.create_dirsymlink(source, link_name)
+    if isdir(target):
+      platform_utils_win32.create_dirsymlink(_makelongpath(source), link_name)
     else:
-      platform_utils_win32.create_filesymlink(source, link_name)
+      platform_utils_win32.create_filesymlink(_makelongpath(source), link_name)
   else:
     return os.symlink(source, link_name)
 
@@ -220,9 +220,32 @@ def _winpath_is_valid(path):
     return not drive  # "x:" is invalid
 
 
+def _makelongpath(path):
+  """Return the input path normalized to support the Windows long path syntax
+  ("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the
+  MAX_PATH limit.
+  """
+  if isWindows():
+    # Note: MAX_PATH is 260, but, for directories, the maximum value is actually 246.
+    if len(path) < 246:
+      return path
+    if path.startswith(u"\\\\?\\"):
+      return path
+    if not os.path.isabs(path):
+      return path
+    # Append prefix and ensure unicode so that the special longpath syntax
+    # is supported by underlying Win32 API calls
+    return u"\\\\?\\" + os.path.normpath(path)
+  else:
+    return path
+
+
 def rmtree(path):
+  """shutil.rmtree(path) wrapper with support for long paths on Windows.
+
+  Availability: Unix, Windows."""
   if isWindows():
-    shutil.rmtree(path, onerror=handle_rmtree_error)
+    shutil.rmtree(_makelongpath(path), onerror=handle_rmtree_error)
   else:
     shutil.rmtree(path)
 
@@ -234,15 +257,18 @@ def handle_rmtree_error(function, path, excinfo):
 
 
 def rename(src, dst):
+  """os.rename(src, dst) wrapper with support for long paths on Windows.
+
+  Availability: Unix, Windows."""
   if isWindows():
     # On Windows, rename fails if destination exists, see
     # https://docs.python.org/2/library/os.html#os.rename
     try:
-      os.rename(src, dst)
+      os.rename(_makelongpath(src), _makelongpath(dst))
     except OSError as e:
       if e.errno == errno.EEXIST:
-        os.remove(dst)
-        os.rename(src, dst)
+        os.remove(_makelongpath(dst))
+        os.rename(_makelongpath(src), _makelongpath(dst))
       else:
         raise
   else:
@@ -250,30 +276,98 @@ def rename(src, dst):
 
 
 def remove(path):
-  """Remove (delete) the file path. This is a replacement for os.remove, but
-  allows deleting read-only files on Windows.
-  """
+  """Remove (delete) the file path. This is a replacement for os.remove that
+  allows deleting read-only files on Windows, with support for long paths and
+  for deleting directory symbolic links.
+
+  Availability: Unix, Windows."""
   if isWindows():
+    longpath = _makelongpath(path)
     try:
-      os.remove(path)
+      os.remove(longpath)
     except OSError as e:
       if e.errno == errno.EACCES:
-        os.chmod(path, stat.S_IWRITE)
-        os.remove(path)
+        os.chmod(longpath, stat.S_IWRITE)
+        # Directory symbolic links must be deleted with 'rmdir'.
+        if islink(longpath) and isdir(longpath):
+          os.rmdir(longpath)
+        else:
+          os.remove(longpath)
       else:
         raise
   else:
     os.remove(path)
 
 
+def walk(top, topdown=True, onerror=None, followlinks=False):
+  """os.walk(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  if isWindows():
+    return _walk_windows_impl(top, topdown, onerror, followlinks)
+  else:
+    return os.walk(top, topdown, onerror, followlinks)
+
+
+def _walk_windows_impl(top, topdown, onerror, followlinks):
+  try:
+    names = listdir(top)
+  except error, err:
+    if onerror is not None:
+      onerror(err)
+    return
+
+  dirs, nondirs = [], []
+  for name in names:
+    if isdir(os.path.join(top, name)):
+      dirs.append(name)
+    else:
+      nondirs.append(name)
+
+  if topdown:
+    yield top, dirs, nondirs
+  for name in dirs:
+    new_path = os.path.join(top, name)
+    if followlinks or not islink(new_path):
+      for x in _walk_windows_impl(new_path, topdown, onerror, followlinks):
+        yield x
+  if not topdown:
+    yield top, dirs, nondirs
+
+
+def listdir(path):
+  """os.listdir(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  return os.listdir(_makelongpath(path))
+
+
+def rmdir(path):
+  """os.rmdir(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  os.rmdir(_makelongpath(path))
+
+
+def isdir(path):
+  """os.path.isdir(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  return os.path.isdir(_makelongpath(path))
+
+
 def islink(path):
-  """Test whether a path is a symbolic link.
+  """os.path.islink(path) wrapper with support for long paths on Windows.
 
   Availability: Windows, Unix.
   """
   if isWindows():
     import platform_utils_win32
-    return platform_utils_win32.islink(path)
+    return platform_utils_win32.islink(_makelongpath(path))
   else:
     return os.path.islink(path)
 
@@ -288,7 +382,7 @@ def readlink(path):
   """
   if isWindows():
     import platform_utils_win32
-    return platform_utils_win32.readlink(path)
+    return platform_utils_win32.readlink(_makelongpath(path))
   else:
     return os.readlink(path)