11 files changed, 560 insertions, 141 deletions

diff --git a/command.py b/command.py
index 38cacd3b..997acec0 100644
--- a/command.py
+++ b/command.py
@@ -106,13 +106,13 @@ class Command(object):
   def _UpdatePathToProjectMap(self, project):
     self._by_path[project.worktree] = project
 
-  def _GetProjectByPath(self, path):
+  def _GetProjectByPath(self, manifest, path):
     project = None
     if os.path.exists(path):
       oldpath = None
       while path \
         and path != oldpath \
-        and path != self.manifest.topdir:
+        and path != manifest.topdir:
         try:
           project = self._by_path[path]
           break
@@ -126,13 +126,16 @@ class Command(object):
         pass
     return project
 
-  def GetProjects(self, args, groups='', missing_ok=False, submodules_ok=False):
+  def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
+                  submodules_ok=False):
     """A list of projects that match the arguments.
     """
-    all_projects_list = self.manifest.projects
+    if not manifest:
+      manifest = self.manifest
+    all_projects_list = manifest.projects
     result = []
 
-    mp = self.manifest.manifestProject
+    mp = manifest.manifestProject
 
     if not groups:
         groups = mp.config.GetString('manifest.groups')
@@ -155,11 +158,11 @@ class Command(object):
       self._ResetPathToProjectMap(all_projects_list)
 
       for arg in args:
-        projects = self.manifest.GetProjectsWithName(arg)
+        projects = manifest.GetProjectsWithName(arg)
 
         if not projects:
           path = os.path.abspath(arg).replace('\\', '/')
-          project = self._GetProjectByPath(path)
+          project = self._GetProjectByPath(manifest, path)
 
           # If it's not a derived project, update path->project mapping and
           # search again, as arg might actually point to a derived subproject.
@@ -170,7 +173,7 @@ class Command(object):
               self._UpdatePathToProjectMap(subproject)
               search_again = True
             if search_again:
-              project = self._GetProjectByPath(path) or project
+              project = self._GetProjectByPath(manifest, path) or project
 
           if project:
             projects = [project]
@@ -227,3 +230,8 @@ class MirrorSafeCommand(object):
   """Command permits itself to run within a mirror,
      and does not require a working directory.
   """
+
+class RequiresGitcCommand(object):
+  """Command that requires GITC to be available, but does
+     not require the local client to be a GITC client.
+  """
diff --git a/git_config.py b/git_config.py
index 8ded7c25..0379181a 100644
--- a/git_config.py
+++ b/git_config.py
@@ -15,6 +15,8 @@
 
 from __future__ import print_function
 
+import contextlib
+import errno
 import json
 import os
 import re
@@ -502,6 +504,43 @@ def GetSchemeFromUrl(url):
     return m.group(1)
   return None
 
+@contextlib.contextmanager
+def GetUrlCookieFile(url, quiet):
+  if url.startswith('persistent-'):
+    try:
+      p = subprocess.Popen(
+          ['git-remote-persistent-https', '-print_config', url],
+          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE)
+      try:
+        cookieprefix = 'http.cookiefile='
+        proxyprefix = 'http.proxy='
+        cookiefile = None
+        proxy = None
+        for line in p.stdout:
+          line = line.strip()
+          if line.startswith(cookieprefix):
+            cookiefile = line[len(cookieprefix):]
+          if line.startswith(proxyprefix):
+            proxy = line[len(proxyprefix):]
+        # Leave subprocess open, as cookie file may be transient.
+        if cookiefile or proxy:
+          yield cookiefile, proxy
+          return
+      finally:
+        p.stdin.close()
+        if p.wait():
+          err_msg = p.stderr.read()
+          if ' -print_config' in err_msg:
+            pass  # Persistent proxy doesn't support -print_config.
+          elif not quiet:
+            print(err_msg, file=sys.stderr)
+    except OSError as e:
+      if e.errno == errno.ENOENT:
+        pass  # No persistent proxy.
+      raise
+  yield GitConfig.ForUser().GetString('http.cookiefile'), None
+
 def _preconnect(url):
   m = URI_ALL.match(url)
   if m:
diff --git a/gitc_utils.py b/gitc_utils.py
new file mode 100644
index 00000000..dd38f890
--- /dev/null
+++ b/gitc_utils.py
@@ -0,0 +1,158 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import platform
+import re
+import sys
+import time
+
+import git_command
+import git_config
+import wrapper
+
+GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
+NUM_BATCH_RETRIEVE_REVISIONID = 300
+
+def get_gitc_manifest_dir():
+  return wrapper.Wrapper().get_gitc_manifest_dir()
+
+def parse_clientdir(gitc_fs_path):
+  """Parse a path in the GITC FS and return its client name.
+
+  @param gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
+
+  @returns: The GITC client name
+  """
+  if (gitc_fs_path == GITC_FS_ROOT_DIR or
+      not gitc_fs_path.startswith(GITC_FS_ROOT_DIR)):
+    return None
+  return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split('/')[0]
+
+def _set_project_revisions(projects):
+  """Sets the revisionExpr for a list of projects.
+
+  Because of the limit of open file descriptors allowed, length of projects
+  should not be overly large. Recommend calling this function multiple times
+  with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
+
+  @param projects: List of project objects to set the revionExpr for.
+  """
+  # Retrieve the commit id for each project based off of it's current
+  # revisionExpr and it is not already a commit id.
+  project_gitcmds = [(
+      project, git_command.GitCommand(None,
+                                      ['ls-remote',
+                                       project.remote.url,
+                                       project.revisionExpr],
+                                      capture_stdout=True, cwd='/tmp'))
+      for project in projects if not git_config.IsId(project.revisionExpr)]
+  for proj, gitcmd in project_gitcmds:
+    if gitcmd.Wait():
+      print('FATAL: Failed to retrieve revisionExpr for %s' % proj)
+      sys.exit(1)
+    proj.revisionExpr = gitcmd.stdout.split('\t')[0]
+
+def _manifest_groups(manifest):
+  """Returns the manifest group string that should be synced
+
+  This is the same logic used by Command.GetProjects(), which is used during
+  repo sync
+
+  @param manifest: The XmlManifest object
+  """
+  mp = manifest.manifestProject
+  groups = mp.config.GetString('manifest.groups')
+  if not groups:
+    groups = 'default,platform-' + platform.system().lower()
+  return groups
+
+def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
+  """Generate a manifest for shafsd to use for this GITC client.
+
+  @param gitc_manifest: Current gitc manifest, or None if there isn't one yet.
+  @param manifest: A GitcManifest object loaded with the current repo manifest.
+  @param paths: List of project paths we want to update.
+  """
+
+  print('Generating GITC Manifest by fetching revision SHAs for each '
+        'project.')
+  if paths is None:
+    paths = manifest.paths.keys()
+
+  groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x]
+
+  # Convert the paths to projects, and filter them to the matched groups.
+  projects = [manifest.paths[p] for p in paths]
+  projects = [p for p in projects if p.MatchesGroups(groups)]
+
+  if gitc_manifest is not None:
+    for path, proj in manifest.paths.iteritems():
+      if not proj.MatchesGroups(groups):
+        continue
+
+      if not proj.upstream and not git_config.IsId(proj.revisionExpr):
+        proj.upstream = proj.revisionExpr
+
+      if not path in gitc_manifest.paths:
+        # Any new projects need their first revision, even if we weren't asked
+        # for them.
+        projects.append(proj)
+      elif not path in paths:
+        # And copy revisions from the previous manifest if we're not updating
+        # them now.
+        gitc_proj = gitc_manifest.paths[path]
+        if gitc_proj.old_revision:
+          proj.revisionExpr = None
+          proj.old_revision = gitc_proj.old_revision
+        else:
+          proj.revisionExpr = gitc_proj.revisionExpr
+
+  index = 0
+  while index < len(projects):
+    _set_project_revisions(
+        projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
+    index += NUM_BATCH_RETRIEVE_REVISIONID
+
+  if gitc_manifest is not None:
+    for path, proj in gitc_manifest.paths.iteritems():
+      if proj.old_revision and path in paths:
+        # If we updated a project that has been started, keep the old-revision
+        # updated.
+        repo_proj = manifest.paths[path]
+        repo_proj.old_revision = repo_proj.revisionExpr
+        repo_proj.revisionExpr = None
+
+  # Convert URLs from relative to absolute.
+  for name, remote in manifest.remotes.iteritems():
+    remote.fetchUrl = remote.resolvedFetchUrl
+
+  # Save the manifest.
+  save_manifest(manifest)
+
+def save_manifest(manifest, client_dir=None):
+  """Save the manifest file in the client_dir.
+
+  @param client_dir: Client directory to save the manifest in.
+  @param manifest: Manifest object to save.
+  """
+  if not client_dir:
+    client_dir = manifest.gitc_client_dir
+  with open(os.path.join(client_dir, '.manifest'), 'w') as f:
+    manifest.Save(f, groups=_manifest_groups(manifest))
+  # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
+  # Give the GITC filesystem time to register the manifest changes.
+  time.sleep(3)
diff --git a/main.py b/main.py
index 6736abc9..a5979a87 100755
--- a/main.py
+++ b/main.py
@@ -42,6 +42,7 @@ from git_command import git, GitCommand
 from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
+from command import RequiresGitcCommand
 from subcmds.version import Version
 from editor import Editor
 from error import DownloadError
@@ -51,7 +52,8 @@ from error import ManifestParseError
 from error import NoManifestException
 from error import NoSuchProjectError
 from error import RepoChangedException
-from manifest_xml import XmlManifest
+import gitc_utils
+from manifest_xml import GitcManifest, XmlManifest
 from pager import RunPager
 from wrapper import WrapperPath, Wrapper
 
@@ -129,6 +131,12 @@ class _Repo(object):
 
     cmd.repodir = self.repodir
     cmd.manifest = XmlManifest(cmd.repodir)
+    cmd.gitc_manifest = None
+    gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
+    if gitc_client_name:
+      cmd.gitc_manifest = GitcManifest(cmd.repodir, gitc_client_name)
+      cmd.manifest.isGitcClient = True
+
     Editor.globalConfig = cmd.manifest.globalConfig
 
     if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
@@ -136,6 +144,11 @@ class _Repo(object):
             file=sys.stderr)
       return 1
 
+    if isinstance(cmd, RequiresGitcCommand) and not gitc_utils.get_gitc_manifest_dir():
+      print("fatal: '%s' requires GITC to be available" % name,
+            file=sys.stderr)
+      return 1
+
     try:
       copts, cargs = cmd.OptionParser.parse_args(argv)
       copts = cmd.ReadEnvironmentOptions(copts)
diff --git a/manifest_xml.py b/manifest_xml.py
index 7e719600..3ac607ec 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -29,6 +29,7 @@ else:
   urllib = imp.new_module('urllib')
   urllib.parse = urlparse
 
+import gitc_utils
 from git_config import GitConfig
 from git_refs import R_HEADS, HEAD
 from project import RemoteSpec, Project, MetaProject
@@ -112,6 +113,7 @@ class XmlManifest(object):
     self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
     self.globalConfig = GitConfig.ForUser()
     self.localManifestWarning = False
+    self.isGitcClient = False
 
     self.repoProject = MetaProject(self, 'repo',
       gitdir   = os.path.join(repodir, 'repo/.git'),
@@ -165,12 +167,13 @@ class XmlManifest(object):
   def _ParseGroups(self, groups):
     return [x for x in re.split(r'[,\s]+', groups) if x]
 
-  def Save(self, fd, peg_rev=False, peg_rev_upstream=True):
+  def Save(self, fd, peg_rev=False, peg_rev_upstream=True, groups=None):
     """Write the current manifest out to the given file descriptor.
     """
     mp = self.manifestProject
 
-    groups = mp.config.GetString('manifest.groups')
+    if groups is None:
+      groups = mp.config.GetString('manifest.groups')
     if groups:
       groups = self._ParseGroups(groups)
 
@@ -303,6 +306,11 @@ class XmlManifest(object):
       if p.sync_s:
         e.setAttribute('sync-s', 'true')
 
+      if p.clone_depth:
+        e.setAttribute('clone-depth', str(p.clone_depth))
+
+      self._output_manifest_project_extras(p, e)
+
       if p.subprojects:
         subprojects = set(subp.name for subp in p.subprojects)
         output_projects(p, e, list(sorted(subprojects)))
@@ -320,6 +328,10 @@ class XmlManifest(object):
 
     doc.writexml(fd, '', '  ', '\n', 'UTF-8')
 
+  def _output_manifest_project_extras(self, p, e):
+    """Manifests can modify e if they support extra project attributes."""
+    pass
+
   @property
   def paths(self):
     self._Load()
@@ -709,7 +721,7 @@ class XmlManifest(object):
   def _UnjoinName(self, parent_name, name):
     return os.path.relpath(name, parent_name)
 
-  def _ParseProject(self, node, parent = None):
+  def _ParseProject(self, node, parent = None, **extra_proj_attrs):
     """
     reads a <project> element from the manifest file
     """
@@ -804,7 +816,8 @@ class XmlManifest(object):
                       clone_depth = clone_depth,
                       upstream = upstream,
                       parent = parent,
-                      dest_branch = dest_branch)
+                      dest_branch = dest_branch,
+                      **extra_proj_attrs)
 
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
@@ -935,3 +948,26 @@ class XmlManifest(object):
       diff['added'].append(toProjects[proj])
 
     return diff
+
+
+class GitcManifest(XmlManifest):
+
+  def __init__(self, repodir, gitc_client_name):
+    """Initialize the GitcManifest object."""
+    super(GitcManifest, self).__init__(repodir)
+    self.isGitcClient = True
+    self.gitc_client_name = gitc_client_name
+    self.gitc_client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
+                                        gitc_client_name)
+    self.manifestFile = os.path.join(self.gitc_client_dir, '.manifest')
+
+  def _ParseProject(self, node, parent = None):
+    """Override _ParseProject and add support for GITC specific attributes."""
+    return super(GitcManifest, self)._ParseProject(
+        node, parent=parent, old_revision=node.getAttribute('old-revision'))
+
+  def _output_manifest_project_extras(self, p, e):
+    """Output GITC Specific Project attributes"""
+    if p.old_revision:
+        e.setAttribute('old-revision', str(p.old_revision))
+
diff --git a/project.py b/project.py
index a117f4df..5d8f61e1 100644
--- a/project.py
+++ b/project.py
@@ -13,7 +13,6 @@
 # limitations under the License.
 
 from __future__ import print_function
-import contextlib
 import errno
 import filecmp
 import glob
@@ -31,7 +30,7 @@ import traceback
 
 from color import Coloring
 from git_command import GitCommand, git_require
-from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
+from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, ID_RE
 from error import GitError, HookError, UploadError, DownloadError
 from error import ManifestInvalidRevisionError
 from error import NoManifestException
@@ -64,6 +63,10 @@ def _error(fmt, *args):
   msg = fmt % args
   print('error: %s' % msg, file=sys.stderr)
 
+def _warn(fmt, *args):
+  msg = fmt % args
+  print('warn: %s' % msg, file=sys.stderr)
+
 def not_rev(r):
   return '^' + r
 
@@ -569,7 +572,8 @@ class Project(object):
                parent=None,
                is_derived=False,
                dest_branch=None,
-               optimized_fetch=False):
+               optimized_fetch=False,
+               old_revision=None):
     """Init a Project object.
 
     Args:
@@ -593,6 +597,7 @@ class Project(object):
       dest_branch: The branch to which to push changes for review by default.
       optimized_fetch: If True, when a project is set to a sha1 revision, only
                        fetch from the remote if the sha1 is not present locally.
+      old_revision: saved git commit id for open GITC projects.
     """
     self.manifest = manifest
     self.name = name
@@ -640,6 +645,7 @@ class Project(object):
     self.bare_ref = GitRefs(gitdir)
     self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
     self.dest_branch = dest_branch
+    self.old_revision = old_revision
 
     # This will be filled in if a project is later identified to be the
     # project containing repo hooks.
@@ -1093,8 +1099,7 @@ class Project(object):
         tar.extractall(path=path)
         return True
     except (IOError, tarfile.TarError) as e:
-      print("error: Cannot extract archive %s: "
-            "%s" % (tarpath, str(e)), file=sys.stderr)
+      _error("Cannot extract archive %s: %s", tarpath, str(e))
     return False
 
   def Sync_NetworkHalf(self,
@@ -1111,8 +1116,7 @@ class Project(object):
     """
     if archive and not isinstance(self, MetaProject):
       if self.remote.url.startswith(('http://', 'https://')):
-        print("error: %s: Cannot fetch archives from http/https "
-              "remotes." % self.name, file=sys.stderr)
+        _error("%s: Cannot fetch archives from http/https remotes.", self.name)
         return False
 
       name = self.relpath.replace('\\', '/')
@@ -1123,7 +1127,7 @@ class Project(object):
       try:
         self._FetchArchive(tarpath, cwd=topdir)
       except GitError as e:
-        print('error: %s' % str(e), file=sys.stderr)
+        _error('%s', e)
         return False
 
       # From now on, we only need absolute tarpath
@@ -1134,8 +1138,7 @@ class Project(object):
       try:
         os.remove(tarpath)
       except OSError as e:
-        print("warn: Cannot remove archive %s: "
-              "%s" % (tarpath, str(e)), file=sys.stderr)
+        _warn("Cannot remove archive %s: %s", tarpath, str(e))
       self._CopyAndLinkFiles()
       return True
     if is_new is None:
@@ -1195,6 +1198,8 @@ class Project(object):
     self._InitHooks()
 
   def _CopyAndLinkFiles(self):
+    if self.manifest.isGitcClient:
+      return
     for copyfile in self.copyfiles:
       copyfile._Copy()
     for linkfile in self.linkfiles:
@@ -1270,6 +1275,8 @@ class Project(object):
         # Except if the head needs to be detached
         #
         if not syncbuf.detach_head:
+          # The copy/linkfile config may have changed.
+          self._CopyAndLinkFiles()
           return
       else:
         lost = self._revlist(not_rev(revid), HEAD)
@@ -1287,6 +1294,8 @@ class Project(object):
     if head == revid:
       # No changes; don't do anything further.
       #
+      # The copy/linkfile config may have changed.
+      self._CopyAndLinkFiles()
       return
 
     branch = self.GetBranch(branch)
@@ -1425,9 +1434,11 @@ class Project(object):
 
 ## Branch Management ##
 
-  def StartBranch(self, name):
+  def StartBranch(self, name, branch_merge=''):
     """Create a new branch off the manifest's revision.
     """
+    if not branch_merge:
+      branch_merge = self.revisionExpr
     head = self.work_git.GetHead()
     if head == (R_HEADS + name):
       return True
@@ -1441,9 +1452,9 @@ class Project(object):
 
     branch = self.GetBranch(name)
     branch.remote = self.GetRemote(self.remote.name)
-    branch.merge = self.revisionExpr
-    if not branch.merge.startswith('refs/') and not ID_RE.match(self.revisionExpr):
-      branch.merge = R_HEADS + self.revisionExpr
+    branch.merge = branch_merge
+    if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
+      branch.merge = R_HEADS + branch_merge
     revid = self.GetRevisionId(all_refs)
 
     if head.startswith(R_HEADS):
@@ -1451,7 +1462,6 @@ class Project(object):
         head = all_refs[head]
       except KeyError:
         head = None
-
     if revid and head and revid == head:
       ref = os.path.join(self.gitdir, R_HEADS + name)
       try:
@@ -2030,7 +2040,7 @@ class Project(object):
         os.remove(tmpPath)
     if 'http_proxy' in os.environ and 'darwin' == sys.platform:
       cmd += ['--proxy', os.environ['http_proxy']]
-    with self._GetBundleCookieFile(srcUrl, quiet) as cookiefile:
+    with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, proxy):
       if cookiefile:
         cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
       if srcUrl.startswith('persistent-'):
@@ -2078,40 +2088,6 @@ class Project(object):
     except OSError:
       return False
 
-  @contextlib.contextmanager
-  def _GetBundleCookieFile(self, url, quiet):
-    if url.startswith('persistent-'):
-      try:
-        p = subprocess.Popen(
-            ['git-remote-persistent-https', '-print_config', url],
-            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE)
-        try:
-          prefix = 'http.cookiefile='
-          cookiefile = None
-          for line in p.stdout:
-            line = line.strip()
-            if line.startswith(prefix):
-              cookiefile = line[len(prefix):]
-              break
-          # Leave subprocess open, as cookie file may be transient.
-          if cookiefile:
-            yield cookiefile
-            return
-        finally:
-          p.stdin.close()
-          if p.wait():
-            err_msg = p.stderr.read()
-            if ' -print_config' in err_msg:
-              pass  # Persistent proxy doesn't support -print_config.
-            elif not quiet:
-              print(err_msg, file=sys.stderr)
-      except OSError as e:
-        if e.errno == errno.ENOENT:
-          pass  # No persistent proxy.
-        raise
-    yield GitConfig.ForUser().GetString('http.cookiefile')
-
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
     if quiet:
@@ -2182,8 +2158,8 @@ class Project(object):
         try:
           self._CheckDirReference(self.objdir, self.gitdir, share_refs=False)
         except GitError as e:
-          print("Retrying clone after deleting %s" % force_sync, file=sys.stderr)
           if force_sync:
+            print("Retrying clone after deleting %s" % self.gitdir, file=sys.stderr)
             try:
               shutil.rmtree(os.path.realpath(self.gitdir))
               if self.worktree and os.path.exists(
@@ -2261,7 +2237,7 @@ class Project(object):
         if filecmp.cmp(stock_hook, dst, shallow=False):
           os.remove(dst)
         else:
-          _error("%s: Not replacing %s hook", self.relpath, name)
+          _warn("%s: Not replacing locally modified %s hook", self.relpath, name)
           continue
       try:
         os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
@@ -2320,7 +2296,10 @@ class Project(object):
         # Fail if the links are pointing to the wrong place
         if src != dst:
           raise GitError('--force-sync not enabled; cannot overwrite a local '
-                         'work tree')
+                         'work tree. If you\'re comfortable with the '
+                         'possibility of losing the work tree\'s git metadata,'
+                         ' use `repo sync --force-sync {0}` to '
+                         'proceed.'.format(self.relpath))
 
   def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
     """Update |dotgit| to reference |gitdir|, using symlinks where possible.
diff --git a/repo b/repo
index bf8fa3dc..b7d8024f 100755
--- a/repo
+++ b/repo
@@ -108,7 +108,7 @@ S_repo = 'repo'                 # special repo repository
 S_manifests = 'manifests'       # special manifest repository
 REPO_MAIN = S_repo + '/main.py' # main script
 MIN_PYTHON_VERSION = (2, 6)     # minimum supported python version
-GITC_MANIFEST_DIR = '/usr/local/google/gitc'
+GITC_CONFIG_FILE = '/gitc/.config'
 
 
 import errno
@@ -214,6 +214,7 @@ group.add_option('--config-name',
                  help='Always prompt for name/e-mail')
 
 def _GitcInitOptions(init_optparse):
+  init_optparse.set_usage("repo gitc-init -u url -c client [options]")
   g = init_optparse.add_option_group('GITC options')
   g.add_option('-f', '--manifest-file',
                dest='manifest_file',
@@ -222,6 +223,21 @@ def _GitcInitOptions(init_optparse):
                dest='gitc_client',
                help='The name for the new gitc_client instance.')
 
+_gitc_manifest_dir = None
+def get_gitc_manifest_dir():
+  global _gitc_manifest_dir
+  if _gitc_manifest_dir is None:
+    _gitc_manifest_dir = ''
+    try:
+      with open(GITC_CONFIG_FILE, 'r') as gitc_config:
+        for line in gitc_config:
+          match = re.match('gitc_dir=(?P<gitc_manifest_dir>.*)', line)
+          if match:
+            _gitc_manifest_dir = match.group('gitc_manifest_dir')
+    except IOError:
+      pass
+  return _gitc_manifest_dir
+
 class CloneFailure(Exception):
   """Indicate the remote clone of repo itself failed.
   """
@@ -255,7 +271,15 @@ def _Init(args, gitc_init=False):
 
   try:
     if gitc_init:
-      client_dir = os.path.join(GITC_MANIFEST_DIR, opt.gitc_client)
+      gitc_manifest_dir = get_gitc_manifest_dir()
+      if not gitc_manifest_dir:
+        _print('fatal: GITC filesystem is not available. Exiting...',
+               file=sys.stderr)
+        sys.exit(1)
+      if not opt.gitc_client:
+        _print('fatal: GITC client (-c) is required.', file=sys.stderr)
+        sys.exit(1)
+      client_dir = os.path.join(gitc_manifest_dir, opt.gitc_client)
       if not os.path.exists(client_dir):
         os.makedirs(client_dir)
       os.chdir(client_dir)
@@ -661,6 +685,10 @@ def _ParseArguments(args):
 
 
 def _Usage():
+  gitc_usage = ""
+  if get_gitc_manifest_dir():
+    gitc_usage = "  gitc-init Initialize a GITC Client.\n"
+
   _print(
 """usage: repo COMMAND [ARGS]
 
@@ -669,7 +697,8 @@ repo is not yet installed.  Use "repo init" to install it here.
 The most commonly used repo commands are:
 
   init      Install repo in the current working directory
-  help      Display detailed help on a command
+""" + gitc_usage +
+"""  help      Display detailed help on a command
 
 For access to the full online help, install repo ("repo init").
 """, file=sys.stderr)
@@ -681,6 +710,10 @@ def _Help(args):
     if args[0] == 'init':
       init_optparse.print_help()
       sys.exit(0)
+    elif args[0] == 'gitc-init':
+      _GitcInitOptions(init_optparse)
+      init_optparse.print_help()
+      sys.exit(0)
     else:
       _print("error: '%s' is not a bootstrap command.\n"
              '        For access to online help, install repo ("repo init").'
@@ -746,6 +779,12 @@ def main(orig_args):
   wrapper_path = os.path.abspath(__file__)
   my_main, my_git = _RunSelf(wrapper_path)
 
+  cwd = os.getcwd()
+  if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
+    _print('error: repo cannot be used in the GITC local manifest directory.'
+           '\nIf you want to work on this GITC client please rerun this '
+           'command from the corresponding client under /gitc/', file=sys.stderr)
+    sys.exit(1)
   if not repo_main:
     if opt.help:
       _Usage()
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py
index 9b9cefda..c243bb3b 100644
--- a/subcmds/gitc_init.py
+++ b/subcmds/gitc_init.py
@@ -15,19 +15,15 @@
 
 from __future__ import print_function
 import os
-import shutil
 import sys
 
-import git_command
+import gitc_utils
+from command import RequiresGitcCommand
+from manifest_xml import GitcManifest
 from subcmds import init
 
 
-GITC_MANIFEST_DIR = '/usr/local/google/gitc'
-GITC_FS_ROOT_DIR = '/gitc/sha/rw'
-NUM_BATCH_RETRIEVE_REVISIONID = 300
-
-
-class GitcInit(init.Init):
+class GitcInit(init.Init, RequiresGitcCommand):
   common = True
   helpSummary = "Initialize a GITC Client."
   helpUsage = """
@@ -39,7 +35,7 @@ with the GITC file system.
 
 This command will setup the client directory, initialize repo, just
 like repo init does, and then downloads the manifest collection
-and installs in in the .repo/directory of the GITC client.
+and installs it in the .repo/directory of the GITC client.
 
 Once this is done, a GITC manifest is generated by pulling the HEAD
 SHA for each project and generates the properly formatted XML file
@@ -65,59 +61,24 @@ use for this GITC client.
     if not opt.gitc_client:
       print('fatal: gitc client (-c) is required', file=sys.stderr)
       sys.exit(1)
-    self.client_dir = os.path.join(GITC_MANIFEST_DIR, opt.gitc_client)
-    if not os.path.exists(GITC_MANIFEST_DIR):
-      os.makedirs(GITC_MANIFEST_DIR)
+    self.client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
+                                   opt.gitc_client)
+    if not os.path.exists(gitc_utils.get_gitc_manifest_dir()):
+      os.makedirs(gitc_utils.get_gitc_manifest_dir())
     if not os.path.exists(self.client_dir):
       os.mkdir(self.client_dir)
     super(GitcInit, self).Execute(opt, args)
+
+    manifest_file = self.manifest.manifestFile
     if opt.manifest_file:
       if not os.path.exists(opt.manifest_file):
         print('fatal: Specified manifest file %s does not exist.' %
               opt.manifest_file)
         sys.exit(1)
-      shutil.copyfile(opt.manifest_file,
-                      os.path.join(self.client_dir, '.manifest'))
-    else:
-      self._GenerateGITCManifest()
-    print('Please run `cd %s` to view your GITC client.' %
-          os.path.join(GITC_FS_ROOT_DIR, opt.gitc_client))
-
-  def _SetProjectRevisions(self, projects, branch):
-    """Sets the revisionExpr for a list of projects.
+      manifest_file = opt.manifest_file
 
-    Because of the limit of open file descriptors allowed, length of projects
-    should not be overly large. Recommend calling this function multiple times
-    with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
-
-    @param projects: List of project objects to set the revionExpr for.
-    @param branch: The remote branch to retrieve the SHA from. If branch is
-                   None, 'HEAD' is used.
-    """
-    project_gitcmds = [(
-        project, git_command.GitCommand(None,
-                                        ['ls-remote',
-                                         project.remote.url,
-                                         branch], capture_stdout=True))
-        for project in projects]
-    for proj, gitcmd in project_gitcmds:
-      if gitcmd.Wait():
-        print('FATAL: Failed to retrieve revisionID for %s' % project)
-        sys.exit(1)
-      proj.revisionExpr = gitcmd.stdout.split('\t')[0]
-
-  def _GenerateGITCManifest(self):
-    """Generate a manifest for shafsd to use for this GITC client."""
-    print('Generating GITC Manifest by fetching revision SHAs for each '
-          'project.')
-    manifest = self.manifest
-    project_gitcmd_dict = {}
-    index = 0
-    while index < len(manifest.projects):
-      self._SetProjectRevisions(
-          manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)],
-          manifest.default.revisionExpr)
-      index += NUM_BATCH_RETRIEVE_REVISIONID
-    # Save the manifest.
-    with open(os.path.join(self.client_dir, '.manifest'), 'w') as f:
-      manifest.Save(f)
+    manifest = GitcManifest(self.repodir, opt.gitc_client)
+    manifest.Override(manifest_file)
+    gitc_utils.generate_gitc_manifest(None, manifest)
+    print('Please run `cd %s` to view your GITC client.' %
+          os.path.join(gitc_utils.GITC_FS_ROOT_DIR, opt.gitc_client))
diff --git a/subcmds/help.py b/subcmds/help.py
index 4aa3f863..ae5b8f08 100644
--- a/subcmds/help.py
+++ b/subcmds/help.py
@@ -19,7 +19,8 @@ import sys
 from formatter import AbstractFormatter, DumbWriter
 
 from color import Coloring
-from command import PagedCommand, MirrorSafeCommand
+from command import PagedCommand, MirrorSafeCommand, RequiresGitcCommand
+import gitc_utils
 
 class Help(PagedCommand, MirrorSafeCommand):
   common = False
@@ -54,9 +55,17 @@ Displays detailed usage information about a command.
   def _PrintCommonCommands(self):
     print('usage: repo COMMAND [ARGS]')
     print('The most commonly used repo commands are:')
+
+    def gitc_supported(cmd):
+      if not isinstance(cmd, RequiresGitcCommand):
+        return True
+      if gitc_utils.get_gitc_manifest_dir():
+        return True
+      return False
+
     commandNames = list(sorted([name
                     for name, command in self.commands.items()
-                    if command.common]))
+                    if command.common and gitc_supported(command)]))
 
     maxlen = 0
     for name in commandNames:
diff --git a/subcmds/start.py b/subcmds/start.py
index 60ad41e0..940c3413 100644
--- a/subcmds/start.py
+++ b/subcmds/start.py
@@ -14,11 +14,15 @@
 # limitations under the License.
 
 from __future__ import print_function
+import os
 import sys
+
 from command import Command
 from git_config import IsId
 from git_command import git
+import gitc_utils
 from progress import Progress
+from project import SyncBuffer
 
 class Start(Command):
   common = True
@@ -53,20 +57,50 @@ revision specified in the manifest.
         print("error: at least one project must be specified", file=sys.stderr)
         sys.exit(1)
 
-    all_projects = self.GetProjects(projects)
+    if self.gitc_manifest:
+      all_projects = self.GetProjects(projects, manifest=self.gitc_manifest,
+                                      missing_ok=True)
+      for project in all_projects:
+        if project.old_revision:
+          project.already_synced = True
+        else:
+          project.already_synced = False
+          project.old_revision = project.revisionExpr
+        project.revisionExpr = None
+      # Save the GITC manifest.
+      gitc_utils.save_manifest(self.gitc_manifest)
 
+    all_projects = self.GetProjects(projects,
+                                    missing_ok=bool(self.gitc_manifest))
     pm = Progress('Starting %s' % nb, len(all_projects))
     for project in all_projects:
       pm.update()
+
+      if self.gitc_manifest:
+        gitc_project = self.gitc_manifest.paths[project.relpath]
+        # Sync projects that have not been opened.
+        if not gitc_project.already_synced:
+          proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir,
+                                       project.relpath)
+          project.worktree = proj_localdir
+          if not os.path.exists(proj_localdir):
+            os.makedirs(proj_localdir)
+          project.Sync_NetworkHalf()
+          sync_buf = SyncBuffer(self.manifest.manifestProject.config)
+          project.Sync_LocalHalf(sync_buf)
+          project.revisionId = gitc_project.old_revision
+
       # If the current revision is a specific SHA1 then we can't push back
       # to it; so substitute with dest_branch if defined, or with manifest
       # default revision instead.
+      branch_merge = ''
       if IsId(project.revisionExpr):
         if project.dest_branch:
-          project.revisionExpr = project.dest_branch
+          branch_merge = project.dest_branch
         else:
-          project.revisionExpr = self.manifest.default.revisionExpr
-      if not project.StartBranch(nb):
+          branch_merge = self.manifest.default.revisionExpr
+
+      if not project.StartBranch(nb, branch_merge=branch_merge):
         err.append(project)
     pm.end()
 
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 43d450be..a99d7e74 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -23,18 +23,26 @@ import shutil
 import socket
 import subprocess
 import sys
+import tempfile
 import time
 
 from pyversion import is_python3
 if is_python3():
+  import http.cookiejar as cookielib
+  import urllib.error
   import urllib.parse
+  import urllib.request
   import xmlrpc.client
 else:
+  import cookielib
   import imp
+  import urllib2
   import urlparse
   import xmlrpclib
   urllib = imp.new_module('urllib')
+  urllib.error = urllib2
   urllib.parse = urlparse
+  urllib.request = urllib2
   xmlrpc = imp.new_module('xmlrpc')
   xmlrpc.client = xmlrpclib
 
@@ -57,7 +65,9 @@ except ImportError:
   multiprocessing = None
 
 from git_command import GIT, git_require
+from git_config import GetUrlCookieFile
 from git_refs import R_HEADS, HEAD
+import gitc_utils
 from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
@@ -65,6 +75,7 @@ from error import RepoChangedException, GitError, ManifestParseError
 from project import SyncBuffer
 from progress import Progress
 from wrapper import Wrapper
+from manifest_xml import GitcManifest
 
 _ONE_DAY_S = 24 * 60 * 60
 
@@ -554,19 +565,18 @@ later is required to fix a server side protocol bug.
           try:
             info = netrc.netrc()
           except IOError:
-            print('.netrc file does not exist or could not be opened',
-                  file=sys.stderr)
+            # .netrc file does not exist or could not be opened
+            pass
           else:
             try:
               parse_result = urllib.parse.urlparse(manifest_server)
               if parse_result.hostname:
-                username, _account, password = \
-                  info.authenticators(parse_result.hostname)
-            except TypeError:
-              # TypeError is raised when the given hostname is not present
-              # in the .netrc file.
-              print('No credentials found for %s in .netrc'
-                    % parse_result.hostname, file=sys.stderr)
+                auth = info.authenticators(parse_result.hostname)
+                if auth:
+                  username, _account, password = auth
+                else:
+                  print('No credentials found for %s in .netrc'
+                        % parse_result.hostname, file=sys.stderr)
             except netrc.NetrcParseError as e:
               print('Error parsing .netrc file: %s' % e, file=sys.stderr)
 
@@ -575,8 +585,12 @@ later is required to fix a server side protocol bug.
                                                     (username, password),
                                                     1)
 
+      transport = PersistentTransport(manifest_server)
+      if manifest_server.startswith('persistent-'):
+        manifest_server = manifest_server[len('persistent-'):]
+
       try:
-        server = xmlrpc.client.Server(manifest_server)
+        server = xmlrpc.client.Server(manifest_server, transport=transport)
         if opt.smart_sync:
           p = self.manifest.manifestProject
           b = p.GetBranch(p.CurrentBranch)
@@ -656,6 +670,42 @@ later is required to fix a server side protocol bug.
       self._ReloadManifest(manifest_name)
       if opt.jobs is None:
         self.jobs = self.manifest.default.sync_j
+
+    if self.gitc_manifest:
+      gitc_manifest_projects = self.GetProjects(args,
+                                                missing_ok=True)
+      gitc_projects = []
+      opened_projects = []
+      for project in gitc_manifest_projects:
+        if project.relpath in self.gitc_manifest.paths and \
+           self.gitc_manifest.paths[project.relpath].old_revision:
+          opened_projects.append(project.relpath)
+        else:
+          gitc_projects.append(project.relpath)
+
+      if not args:
+        gitc_projects = None
+
+      if gitc_projects != [] and not opt.local_only:
+        print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
+        manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
+        if manifest_name:
+          manifest.Override(manifest_name)
+        else:
+          manifest.Override(self.manifest.manifestFile)
+        gitc_utils.generate_gitc_manifest(self.gitc_manifest,
+                                          manifest,
+                                          gitc_projects)
+        print('GITC client successfully synced.')
+
+      # The opened projects need to be synced as normal, therefore we
+      # generate a new args list to represent the opened projects.
+      # TODO: make this more reliable -- if there's a project name/path overlap,
+      # this may choose the wrong project.
+      args = [os.path.relpath(self.manifest.paths[p].worktree, os.getcwd())
+              for p in opened_projects]
+      if not args:
+        return
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
                                     submodules_ok=opt.fetch_submodules)
@@ -850,3 +900,96 @@ class _FetchTimes(object):
         os.remove(self._path)
       except OSError:
         pass
+
+# This is a replacement for xmlrpc.client.Transport using urllib2
+# and supporting persistent-http[s]. It cannot change hosts from
+# request to request like the normal transport, the real url
+# is passed during initialization.
+class PersistentTransport(xmlrpc.client.Transport):
+  def __init__(self, orig_host):
+    self.orig_host = orig_host
+
+  def request(self, host, handler, request_body, verbose=False):
+    with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
+      # Python doesn't understand cookies with the #HttpOnly_ prefix
+      # Since we're only using them for HTTP, copy the file temporarily,
+      # stripping those prefixes away.
+      if cookiefile:
+        tmpcookiefile = tempfile.NamedTemporaryFile()
+        try:
+          with open(cookiefile) as f:
+            for line in f:
+              if line.startswith("#HttpOnly_"):
+                line = line[len("#HttpOnly_"):]
+              tmpcookiefile.write(line)
+          tmpcookiefile.flush()
+
+          cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
+          cookiejar.load()
+        finally:
+          tmpcookiefile.close()
+      else:
+        cookiejar = cookielib.CookieJar()
+
+      proxyhandler = urllib.request.ProxyHandler
+      if proxy:
+        proxyhandler = urllib.request.ProxyHandler({
+            "http": proxy,
+            "https": proxy })
+
+      opener = urllib.request.build_opener(
+          urllib.request.HTTPCookieProcessor(cookiejar),
+          proxyhandler)
+
+      url = urllib.parse.urljoin(self.orig_host, handler)
+      parse_results = urllib.parse.urlparse(url)
+
+      scheme = parse_results.scheme
+      if scheme == 'persistent-http':
+        scheme = 'http'
+      if scheme == 'persistent-https':
+        # If we're proxying through persistent-https, use http. The
+        # proxy itself will do the https.
+        if proxy:
+          scheme = 'http'
+        else:
+          scheme = 'https'
+
+      # Parse out any authentication information using the base class
+      host, extra_headers, _ = self.get_host_info(parse_results.netloc)
+
+      url = urllib.parse.urlunparse((
+          scheme,
+          host,
+          parse_results.path,
+          parse_results.params,
+          parse_results.query,
+          parse_results.fragment))
+
+      request = urllib.request.Request(url, request_body)
+      if extra_headers is not None:
+        for (name, header) in extra_headers:
+          request.add_header(name, header)
+      request.add_header('Content-Type', 'text/xml')
+      try:
+        response = opener.open(request)
+      except urllib.error.HTTPError as e:
+        if e.code == 501:
+          # We may have been redirected through a login process
+          # but our POST turned into a GET. Retry.
+          response = opener.open(request)
+        else:
+          raise
+
+      p, u = xmlrpc.client.getparser()
+      while 1:
+        data = response.read(1024)
+        if not data:
+          break
+        p.feed(data)
+      p.close()
+      return u.close()
+
+  def close(self):
+    pass
+