10 files changed, 160 insertions, 47 deletions

diff --git a/error.py b/error.py
index ff948f9c..f2a7c4e4 100644
--- a/error.py
+++ b/error.py
@@ -80,7 +80,7 @@ class NoSuchProjectError(Exception):
     self.name = name
 
   def __str__(self):
-    if self.Name is None:
+    if self.name is None:
       return 'in current directory'
     return self.name
 
@@ -93,7 +93,7 @@ class InvalidProjectGroupsError(Exception):
     self.name = name
 
   def __str__(self):
-    if self.Name is None:
+    if self.name is None:
       return 'in current directory'
     return self.name
 
diff --git a/git_command.py b/git_command.py
index 259fb02c..0893bff7 100644
--- a/git_command.py
+++ b/git_command.py
@@ -92,7 +92,10 @@ class _GitCall(object):
   def version(self):
     p = GitCommand(None, ['--version'], capture_stdout=True)
     if p.Wait() == 0:
-      return p.stdout.decode('utf-8')
+      if hasattr(p.stdout, 'decode'):
+        return p.stdout.decode('utf-8')
+      else:
+        return p.stdout
     return None
 
   def version_tuple(self):
@@ -263,6 +266,8 @@ class GitCommand(object):
         if not buf:
           s_in.remove(s)
           continue
+        if not hasattr(buf, 'encode'):
+          buf = buf.decode()
         if s.std_name == 'stdout':
           self.stdout += buf
         else:
diff --git a/git_config.py b/git_config.py
index c4c31e18..8ded7c25 100644
--- a/git_config.py
+++ b/git_config.py
@@ -280,7 +280,7 @@ class GitConfig(object):
       finally:
         fd.close()
     except (IOError, TypeError):
-      if os.path.exists(self.json):
+      if os.path.exists(self._json):
         os.remove(self._json)
 
   def _ReadGit(self):
diff --git a/main.py b/main.py
index 47f083df..6736abc9 100755
--- a/main.py
+++ b/main.py
@@ -45,6 +45,7 @@ from command import MirrorSafeCommand
 from subcmds.version import Version
 from editor import Editor
 from error import DownloadError
+from error import InvalidProjectGroupsError
 from error import ManifestInvalidRevisionError
 from error import ManifestParseError
 from error import NoManifestException
@@ -173,6 +174,12 @@ class _Repo(object):
       else:
         print('error: no project in current directory', file=sys.stderr)
       result = 1
+    except InvalidProjectGroupsError as e:
+      if e.name:
+        print('error: project group must be enabled for project %s' % e.name, file=sys.stderr)
+      else:
+        print('error: project group must be enabled for the project in the current directory', file=sys.stderr)
+      result = 1
     finally:
       elapsed = time.time() - start
       hours, remainder = divmod(elapsed, 3600)
diff --git a/manifest_xml.py b/manifest_xml.py
index cfbd9efa..130e17c2 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -253,11 +253,13 @@ class XmlManifest(object):
         else:
           value = p.work_git.rev_parse(HEAD + '^0')
         e.setAttribute('revision', value)
-        if peg_rev_upstream and value != p.revisionExpr:
-          # Only save the origin if the origin is not a sha1, and the default
-          # isn't our value, and the if the default doesn't already have that
-          # covered.
-          e.setAttribute('upstream', p.revisionExpr)
+        if peg_rev_upstream:
+          if p.upstream:
+            e.setAttribute('upstream', p.upstream)
+          elif value != p.revisionExpr:
+            # Only save the origin if the origin is not a sha1, and the default
+            # isn't our value
+            e.setAttribute('upstream', p.revisionExpr)
       else:
         revision = self.remotes[remoteName].revision or d.revisionExpr
         if not revision or revision != p.revisionExpr:
diff --git a/project.py b/project.py
index 003489aa..d0d3b6eb 100644
--- a/project.py
+++ b/project.py
@@ -16,6 +16,7 @@ from __future__ import print_function
 import contextlib
 import errno
 import filecmp
+import glob
 import os
 import random
 import re
@@ -233,28 +234,60 @@ class _CopyFile(object):
         _error('Cannot copy file %s to %s', src, dest)
 
 class _LinkFile(object):
-  def __init__(self, src, dest, abssrc, absdest):
+  def __init__(self, git_worktree, src, dest, relsrc, absdest):
+    self.git_worktree = git_worktree
     self.src = src
     self.dest = dest
-    self.abs_src = abssrc
+    self.src_rel_to_dest = relsrc
     self.abs_dest = absdest
 
-  def _Link(self):
-    src = self.abs_src
-    dest = self.abs_dest
+  def __linkIt(self, relSrc, absDest):
     # link file if it does not exist or is out of date
-    if not os.path.islink(dest) or os.readlink(dest) != src:
+    if not os.path.islink(absDest) or (os.readlink(absDest) != relSrc):
       try:
         # remove existing file first, since it might be read-only
-        if os.path.exists(dest):
-          os.remove(dest)
+        if os.path.exists(absDest):
+          os.remove(absDest)
         else:
-          dest_dir = os.path.dirname(dest)
+          dest_dir = os.path.dirname(absDest)
           if not os.path.isdir(dest_dir):
             os.makedirs(dest_dir)
-        os.symlink(src, dest)
+        os.symlink(relSrc, absDest)
       except IOError:
-        _error('Cannot link file %s to %s', src, dest)
+        _error('Cannot link file %s to %s', relSrc, absDest)
+
+  def _Link(self):
+    """Link the self.rel_src_to_dest and self.abs_dest. Handles wild cards
+    on the src linking all of the files in the source in to the destination
+    directory.
+    """
+    # We use the absSrc to handle the situation where the current directory
+    # is not the root of the repo
+    absSrc = os.path.join(self.git_worktree, self.src)
+    if os.path.exists(absSrc):
+      # Entity exists so just a simple one to one link operation
+      self.__linkIt(self.src_rel_to_dest, self.abs_dest)
+    else:
+      # Entity doesn't exist assume there is a wild card
+      absDestDir = self.abs_dest
+      if os.path.exists(absDestDir) and not os.path.isdir(absDestDir):
+        _error('Link error: src with wildcard, %s must be a directory',
+            absDestDir)
+      else:
+        absSrcFiles = glob.glob(absSrc)
+        for absSrcFile in absSrcFiles:
+          # Create a releative path from source dir to destination dir
+          absSrcDir = os.path.dirname(absSrcFile)
+          relSrcDir = os.path.relpath(absSrcDir, absDestDir)
+
+          # Get the source file name
+          srcFile = os.path.basename(absSrcFile)
+
+          # Now form the final full paths to srcFile. They will be
+          # absolute for the desintaiton and relative for the srouce.
+          absDest = os.path.join(absDestDir, srcFile)
+          relSrc = os.path.join(relSrcDir, srcFile)
+          self.__linkIt(relSrc, absDest)
 
 class RemoteSpec(object):
   def __init__(self,
@@ -535,7 +568,8 @@ class Project(object):
                upstream=None,
                parent=None,
                is_derived=False,
-               dest_branch=None):
+               dest_branch=None,
+               optimized_fetch=False):
     """Init a Project object.
 
     Args:
@@ -557,6 +591,8 @@ class Project(object):
       is_derived: False if the project was explicitly defined in the manifest;
                   True if the project is a discovered submodule.
       dest_branch: The branch to which to push changes for review by default.
+      optimized_fetch: If True, when a project is set to a sha1 revision, only
+                       fetch from the remote if the sha1 is not present locally.
     """
     self.manifest = manifest
     self.name = name
@@ -585,6 +621,7 @@ class Project(object):
     self.upstream = upstream
     self.parent = parent
     self.is_derived = is_derived
+    self.optimized_fetch = optimized_fetch
     self.subprojects = []
 
     self.snapshots = {}
@@ -1066,7 +1103,8 @@ class Project(object):
       current_branch_only=False,
       clone_bundle=True,
       no_tags=False,
-      archive=False):
+      archive=False,
+      optimized_fetch=False):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
@@ -1134,8 +1172,9 @@ class Project(object):
       elif self.manifest.default.sync_c:
         current_branch_only = True
 
-    has_sha1 = ID_RE.match(self.revisionExpr) and self._CheckForSha1()
-    if (not has_sha1  #Need to fetch since we don't already have this revision
+    need_to_fetch = not (optimized_fetch and \
+      (ID_RE.match(self.revisionExpr) and self._CheckForSha1()))
+    if (need_to_fetch
         and not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
                                   current_branch_only=current_branch_only,
                                   no_tags=no_tags)):
@@ -1358,9 +1397,10 @@ class Project(object):
 
   def AddLinkFile(self, src, dest, absdest):
     # dest should already be an absolute path, but src is project relative
-    # make src an absolute path
-    abssrc = os.path.join(self.worktree, src)
-    self.linkfiles.append(_LinkFile(src, dest, abssrc, absdest))
+    # make src relative path to dest
+    absdestdir = os.path.dirname(absdest)
+    relsrc = os.path.relpath(os.path.join(self.worktree, src), absdestdir)
+    self.linkfiles.append(_LinkFile(self.worktree, src, dest, relsrc, absdest))
 
   def AddAnnotation(self, name, value, keep):
     self.annotations.append(_Annotation(name, value, keep))
@@ -1401,7 +1441,7 @@ class Project(object):
     branch = self.GetBranch(name)
     branch.remote = self.GetRemote(self.remote.name)
     branch.merge = self.revisionExpr
-    if not branch.merge.startswith('refs/'):
+    if not branch.merge.startswith('refs/') and not ID_RE.match(self.revisionExpr):
       branch.merge = R_HEADS + self.revisionExpr
     revid = self.GetRevisionId(all_refs)
 
@@ -1841,6 +1881,8 @@ class Project(object):
       cmd.append('--quiet')
     if not self.worktree:
       cmd.append('--update-head-ok')
+    if self.manifest.IsMirror:
+      cmd.append('--prune')
     cmd.append(name)
 
     # If using depth then we should not get all the tags since they may
@@ -1860,7 +1902,7 @@ class Project(object):
 
     if not self.manifest.IsMirror:
       branch = self.revisionExpr
-      if is_sha1 and depth:
+      if is_sha1 and depth and git_require((1, 8, 3)):
         # Shallow checkout of a specific commit, fetch from that commit and not
         # the heads only as the commit might be deeper in the history.
         spec.append(branch)
@@ -1905,6 +1947,9 @@ class Project(object):
         # mode, we just tried sync'ing from the upstream field; it doesn't exist, thus
         # abort the optimization attempt and do a full sync.
         break
+      elif ret < 0:
+        # Git died with a signal, exit immediately
+        break
       time.sleep(random.randint(30, 45))
 
     if initial:
@@ -1920,8 +1965,15 @@ class Project(object):
       # got what we wanted, else trigger a second run of all
       # refs.
       if not self._CheckForSha1():
-        return self._RemoteFetch(name=name, current_branch_only=False,
-                                 initial=False, quiet=quiet, alt_dir=alt_dir)
+        if not depth:
+          # Avoid infinite recursion when depth is True (since depth implies
+          # current_branch_only)
+          return self._RemoteFetch(name=name, current_branch_only=False,
+                                   initial=False, quiet=quiet, alt_dir=alt_dir)
+        if self.clone_depth:
+          self.clone_depth = None
+          return self._RemoteFetch(name=name, current_branch_only=current_branch_only,
+                                   initial=False, quiet=quiet, alt_dir=alt_dir)
 
     return ok
 
diff --git a/subcmds/cherry_pick.py b/subcmds/cherry_pick.py
index 520e4c32..1f7dffdc 100644
--- a/subcmds/cherry_pick.py
+++ b/subcmds/cherry_pick.py
@@ -76,6 +76,7 @@ change id will be added.
                      capture_stdout = True,
                      capture_stderr = True)
       p.stdin.write(new_msg)
+      p.stdin.close()
       if p.Wait() != 0:
         print("error: Failed to update commit message", file=sys.stderr)
         sys.exit(1)
diff --git a/subcmds/forall.py b/subcmds/forall.py
index 88b23fbd..b93cd6d0 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -20,6 +20,7 @@ import multiprocessing
 import re
 import os
 import select
+import signal
 import sys
 import subprocess
 
@@ -150,11 +151,15 @@ without iterating through the remaining projects.
     attributes that we need.
 
     """
+    if not self.manifest.IsMirror:
+      lrev = project.GetRevisionId()
+    else:
+      lrev = None
     return {
       'name': project.name,
       'relpath': project.relpath,
       'remote_name': project.remote.name,
-      'lrev': project.GetRevisionId(),
+      'lrev': lrev,
       'rrev': project.revisionExpr,
       'annotations': dict((a.name, a.value) for a in project.annotations),
       'gitdir': project.gitdir,
@@ -200,6 +205,13 @@ without iterating through the remaining projects.
     mirror = self.manifest.IsMirror
     rc = 0
 
+    smart_sync_manifest_name = "smart_sync_override.xml"
+    smart_sync_manifest_path = os.path.join(
+      self.manifest.manifestProject.worktree, smart_sync_manifest_name)
+
+    if os.path.isfile(smart_sync_manifest_path):
+      self.manifest.Override(smart_sync_manifest_path)
+
     if not opt.regex:
       projects = self.GetProjects(args)
     else:
@@ -207,14 +219,12 @@ without iterating through the remaining projects.
 
     os.environ['REPO_COUNT'] = str(len(projects))
 
-    pool = multiprocessing.Pool(opt.jobs)
+    pool = multiprocessing.Pool(opt.jobs, InitWorker)
     try:
       config = self.manifest.manifestProject.config
       results_it = pool.imap(
          DoWorkWrapper,
-         ([mirror, opt, cmd, shell, cnt, config, self._SerializeProject(p)]
-          for cnt, p in enumerate(projects))
-      )
+         self.ProjectArgs(projects, mirror, opt, cmd, shell, config))
       pool.close()
       for r in results_it:
         rc = rc or r
@@ -236,12 +246,28 @@ without iterating through the remaining projects.
     if rc != 0:
       sys.exit(rc)
 
+  def ProjectArgs(self, projects, mirror, opt, cmd, shell, config):
+    for cnt, p in enumerate(projects):
+      try:
+        project = self._SerializeProject(p)
+      except Exception as e:
+        print('Project list error: %r' % e,
+              file=sys.stderr)
+        return
+      except KeyboardInterrupt:
+        print('Project list interrupted',
+              file=sys.stderr)
+        return
+      yield [mirror, opt, cmd, shell, cnt, config, project]
 
 class WorkerKeyboardInterrupt(Exception):
   """ Keyboard interrupt exception for worker processes. """
   pass
 
 
+def InitWorker():
+  signal.signal(signal.SIGINT, signal.SIG_IGN)
+
 def DoWorkWrapper(args):
   """ A wrapper around the DoWork() method.
 
@@ -263,7 +289,9 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
   def setenv(name, val):
     if val is None:
       val = ''
-    env[name] = val.encode()
+    if hasattr(val, 'encode'):
+      val = val.encode()
+    env[name] = val
 
   setenv('REPO_PROJECT', project['name'])
   setenv('REPO_PATH', project['relpath'])
diff --git a/subcmds/init.py b/subcmds/init.py
index b73de71c..dbb6ddda 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -27,7 +27,7 @@ else:
   import imp
   import urlparse
   urllib = imp.new_module('urllib')
-  urllib.parse = urlparse.urlparse
+  urllib.parse = urlparse
 
 from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
@@ -153,7 +153,7 @@ to update the working directory files.
       # server where this git is located, so let's save that here.
       mirrored_manifest_git = None
       if opt.reference:
-        manifest_git_path = urllib.parse(opt.manifest_url).path[1:]
+        manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:]
         mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
         if not mirrored_manifest_git.endswith(".git"):
           mirrored_manifest_git += ".git"
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 2bdab3a6..ec333ae7 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -131,6 +131,10 @@ of a project from server.
 The -c/--current-branch option can be used to only fetch objects that
 are on the branch specified by a project's revision.
 
+The --optimized-fetch option can be used to only fetch projects that
+are fixed to a sha1 revision if the sha1 revision does not already
+exist locally.
+
 SSH Connections
 ---------------
 
@@ -206,6 +210,9 @@ later is required to fix a server side protocol bug.
     p.add_option('--no-tags',
                  dest='no_tags', action='store_true',
                  help="don't fetch tags")
+    p.add_option('--optimized-fetch',
+                 dest='optimized_fetch', action='store_true',
+                 help='only fetch projects fixed to sha1 if revision does not exist locally')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -275,7 +282,8 @@ later is required to fix a server side protocol bug.
           quiet=opt.quiet,
           current_branch_only=opt.current_branch_only,
           clone_bundle=not opt.no_clone_bundle,
-          no_tags=opt.no_tags, archive=self.manifest.IsArchive)
+          no_tags=opt.no_tags, archive=self.manifest.IsArchive,
+          optimized_fetch=opt.optimized_fetch)
         self._fetch_times.Set(project, time.time() - start)
 
         # Lock around all the rest of the code, since printing, updating a set
@@ -509,6 +517,9 @@ later is required to fix a server side protocol bug.
       self.manifest.Override(opt.manifest_name)
 
     manifest_name = opt.manifest_name
+    smart_sync_manifest_name = "smart_sync_override.xml"
+    smart_sync_manifest_path = os.path.join(
+      self.manifest.manifestProject.worktree, smart_sync_manifest_name)
 
     if opt.smart_sync or opt.smart_tag:
       if not self.manifest.manifest_server:
@@ -575,17 +586,16 @@ later is required to fix a server side protocol bug.
           [success, manifest_str] = server.GetManifest(opt.smart_tag)
 
         if success:
-          manifest_name = "smart_sync_override.xml"
-          manifest_path = os.path.join(self.manifest.manifestProject.worktree,
-                                       manifest_name)
+          manifest_name = smart_sync_manifest_name
           try:
-            f = open(manifest_path, 'w')
+            f = open(smart_sync_manifest_path, 'w')
             try:
               f.write(manifest_str)
             finally:
               f.close()
-          except IOError:
-            print('error: cannot write manifest to %s' % manifest_path,
+          except IOError as e:
+            print('error: cannot write manifest to %s:\n%s'
+                  % (smart_sync_manifest_path, e),
                   file=sys.stderr)
             sys.exit(1)
           self._ReloadManifest(manifest_name)
@@ -602,6 +612,13 @@ later is required to fix a server side protocol bug.
               % (self.manifest.manifest_server, e.errcode, e.errmsg),
               file=sys.stderr)
         sys.exit(1)
+    else:  # Not smart sync or smart tag mode
+      if os.path.isfile(smart_sync_manifest_path):
+        try:
+          os.remove(smart_sync_manifest_path)
+        except OSError as e:
+          print('error: failed to remove existing smart sync override manifest: %s' %
+                e, file=sys.stderr)
 
     rp = self.manifest.repoProject
     rp.PreSync()
@@ -615,7 +632,8 @@ later is required to fix a server side protocol bug.
     if not opt.local_only:
       mp.Sync_NetworkHalf(quiet=opt.quiet,
                           current_branch_only=opt.current_branch_only,
-                          no_tags=opt.no_tags)
+                          no_tags=opt.no_tags,
+                          optimized_fetch=opt.optimized_fetch)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)