4 files changed, 219 insertions, 72 deletions

diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py
index 9b9cefda..c243bb3b 100644
--- a/subcmds/gitc_init.py
+++ b/subcmds/gitc_init.py
@@ -15,19 +15,15 @@
 
 from __future__ import print_function
 import os
-import shutil
 import sys
 
-import git_command
+import gitc_utils
+from command import RequiresGitcCommand
+from manifest_xml import GitcManifest
 from subcmds import init
 
 
-GITC_MANIFEST_DIR = '/usr/local/google/gitc'
-GITC_FS_ROOT_DIR = '/gitc/sha/rw'
-NUM_BATCH_RETRIEVE_REVISIONID = 300
-
-
-class GitcInit(init.Init):
+class GitcInit(init.Init, RequiresGitcCommand):
   common = True
   helpSummary = "Initialize a GITC Client."
   helpUsage = """
@@ -39,7 +35,7 @@ with the GITC file system.
 
 This command will setup the client directory, initialize repo, just
 like repo init does, and then downloads the manifest collection
-and installs in in the .repo/directory of the GITC client.
+and installs it in the .repo/directory of the GITC client.
 
 Once this is done, a GITC manifest is generated by pulling the HEAD
 SHA for each project and generates the properly formatted XML file
@@ -65,59 +61,24 @@ use for this GITC client.
     if not opt.gitc_client:
       print('fatal: gitc client (-c) is required', file=sys.stderr)
       sys.exit(1)
-    self.client_dir = os.path.join(GITC_MANIFEST_DIR, opt.gitc_client)
-    if not os.path.exists(GITC_MANIFEST_DIR):
-      os.makedirs(GITC_MANIFEST_DIR)
+    self.client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
+                                   opt.gitc_client)
+    if not os.path.exists(gitc_utils.get_gitc_manifest_dir()):
+      os.makedirs(gitc_utils.get_gitc_manifest_dir())
     if not os.path.exists(self.client_dir):
       os.mkdir(self.client_dir)
     super(GitcInit, self).Execute(opt, args)
+
+    manifest_file = self.manifest.manifestFile
     if opt.manifest_file:
       if not os.path.exists(opt.manifest_file):
         print('fatal: Specified manifest file %s does not exist.' %
               opt.manifest_file)
         sys.exit(1)
-      shutil.copyfile(opt.manifest_file,
-                      os.path.join(self.client_dir, '.manifest'))
-    else:
-      self._GenerateGITCManifest()
-    print('Please run `cd %s` to view your GITC client.' %
-          os.path.join(GITC_FS_ROOT_DIR, opt.gitc_client))
-
-  def _SetProjectRevisions(self, projects, branch):
-    """Sets the revisionExpr for a list of projects.
+      manifest_file = opt.manifest_file
 
-    Because of the limit of open file descriptors allowed, length of projects
-    should not be overly large. Recommend calling this function multiple times
-    with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
-
-    @param projects: List of project objects to set the revionExpr for.
-    @param branch: The remote branch to retrieve the SHA from. If branch is
-                   None, 'HEAD' is used.
-    """
-    project_gitcmds = [(
-        project, git_command.GitCommand(None,
-                                        ['ls-remote',
-                                         project.remote.url,
-                                         branch], capture_stdout=True))
-        for project in projects]
-    for proj, gitcmd in project_gitcmds:
-      if gitcmd.Wait():
-        print('FATAL: Failed to retrieve revisionID for %s' % project)
-        sys.exit(1)
-      proj.revisionExpr = gitcmd.stdout.split('\t')[0]
-
-  def _GenerateGITCManifest(self):
-    """Generate a manifest for shafsd to use for this GITC client."""
-    print('Generating GITC Manifest by fetching revision SHAs for each '
-          'project.')
-    manifest = self.manifest
-    project_gitcmd_dict = {}
-    index = 0
-    while index < len(manifest.projects):
-      self._SetProjectRevisions(
-          manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)],
-          manifest.default.revisionExpr)
-      index += NUM_BATCH_RETRIEVE_REVISIONID
-    # Save the manifest.
-    with open(os.path.join(self.client_dir, '.manifest'), 'w') as f:
-      manifest.Save(f)
+    manifest = GitcManifest(self.repodir, opt.gitc_client)
+    manifest.Override(manifest_file)
+    gitc_utils.generate_gitc_manifest(None, manifest)
+    print('Please run `cd %s` to view your GITC client.' %
+          os.path.join(gitc_utils.GITC_FS_ROOT_DIR, opt.gitc_client))
diff --git a/subcmds/help.py b/subcmds/help.py
index 4aa3f863..ae5b8f08 100644
--- a/subcmds/help.py
+++ b/subcmds/help.py
@@ -19,7 +19,8 @@ import sys
 from formatter import AbstractFormatter, DumbWriter
 
 from color import Coloring
-from command import PagedCommand, MirrorSafeCommand
+from command import PagedCommand, MirrorSafeCommand, RequiresGitcCommand
+import gitc_utils
 
 class Help(PagedCommand, MirrorSafeCommand):
   common = False
@@ -54,9 +55,17 @@ Displays detailed usage information about a command.
   def _PrintCommonCommands(self):
     print('usage: repo COMMAND [ARGS]')
     print('The most commonly used repo commands are:')
+
+    def gitc_supported(cmd):
+      if not isinstance(cmd, RequiresGitcCommand):
+        return True
+      if gitc_utils.get_gitc_manifest_dir():
+        return True
+      return False
+
     commandNames = list(sorted([name
                     for name, command in self.commands.items()
-                    if command.common]))
+                    if command.common and gitc_supported(command)]))
 
     maxlen = 0
     for name in commandNames:
diff --git a/subcmds/start.py b/subcmds/start.py
index 60ad41e0..940c3413 100644
--- a/subcmds/start.py
+++ b/subcmds/start.py
@@ -14,11 +14,15 @@
 # limitations under the License.
 
 from __future__ import print_function
+import os
 import sys
+
 from command import Command
 from git_config import IsId
 from git_command import git
+import gitc_utils
 from progress import Progress
+from project import SyncBuffer
 
 class Start(Command):
   common = True
@@ -53,20 +57,50 @@ revision specified in the manifest.
         print("error: at least one project must be specified", file=sys.stderr)
         sys.exit(1)
 
-    all_projects = self.GetProjects(projects)
+    if self.gitc_manifest:
+      all_projects = self.GetProjects(projects, manifest=self.gitc_manifest,
+                                      missing_ok=True)
+      for project in all_projects:
+        if project.old_revision:
+          project.already_synced = True
+        else:
+          project.already_synced = False
+          project.old_revision = project.revisionExpr
+        project.revisionExpr = None
+      # Save the GITC manifest.
+      gitc_utils.save_manifest(self.gitc_manifest)
 
+    all_projects = self.GetProjects(projects,
+                                    missing_ok=bool(self.gitc_manifest))
     pm = Progress('Starting %s' % nb, len(all_projects))
     for project in all_projects:
       pm.update()
+
+      if self.gitc_manifest:
+        gitc_project = self.gitc_manifest.paths[project.relpath]
+        # Sync projects that have not been opened.
+        if not gitc_project.already_synced:
+          proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir,
+                                       project.relpath)
+          project.worktree = proj_localdir
+          if not os.path.exists(proj_localdir):
+            os.makedirs(proj_localdir)
+          project.Sync_NetworkHalf()
+          sync_buf = SyncBuffer(self.manifest.manifestProject.config)
+          project.Sync_LocalHalf(sync_buf)
+          project.revisionId = gitc_project.old_revision
+
       # If the current revision is a specific SHA1 then we can't push back
       # to it; so substitute with dest_branch if defined, or with manifest
       # default revision instead.
+      branch_merge = ''
       if IsId(project.revisionExpr):
         if project.dest_branch:
-          project.revisionExpr = project.dest_branch
+          branch_merge = project.dest_branch
         else:
-          project.revisionExpr = self.manifest.default.revisionExpr
-      if not project.StartBranch(nb):
+          branch_merge = self.manifest.default.revisionExpr
+
+      if not project.StartBranch(nb, branch_merge=branch_merge):
         err.append(project)
     pm.end()
 
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 43d450be..a99d7e74 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -23,18 +23,26 @@ import shutil
 import socket
 import subprocess
 import sys
+import tempfile
 import time
 
 from pyversion import is_python3
 if is_python3():
+  import http.cookiejar as cookielib
+  import urllib.error
   import urllib.parse
+  import urllib.request
   import xmlrpc.client
 else:
+  import cookielib
   import imp
+  import urllib2
   import urlparse
   import xmlrpclib
   urllib = imp.new_module('urllib')
+  urllib.error = urllib2
   urllib.parse = urlparse
+  urllib.request = urllib2
   xmlrpc = imp.new_module('xmlrpc')
   xmlrpc.client = xmlrpclib
 
@@ -57,7 +65,9 @@ except ImportError:
   multiprocessing = None
 
 from git_command import GIT, git_require
+from git_config import GetUrlCookieFile
 from git_refs import R_HEADS, HEAD
+import gitc_utils
 from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
@@ -65,6 +75,7 @@ from error import RepoChangedException, GitError, ManifestParseError
 from project import SyncBuffer
 from progress import Progress
 from wrapper import Wrapper
+from manifest_xml import GitcManifest
 
 _ONE_DAY_S = 24 * 60 * 60
 
@@ -554,19 +565,18 @@ later is required to fix a server side protocol bug.
           try:
             info = netrc.netrc()
           except IOError:
-            print('.netrc file does not exist or could not be opened',
-                  file=sys.stderr)
+            # .netrc file does not exist or could not be opened
+            pass
           else:
             try:
               parse_result = urllib.parse.urlparse(manifest_server)
               if parse_result.hostname:
-                username, _account, password = \
-                  info.authenticators(parse_result.hostname)
-            except TypeError:
-              # TypeError is raised when the given hostname is not present
-              # in the .netrc file.
-              print('No credentials found for %s in .netrc'
-                    % parse_result.hostname, file=sys.stderr)
+                auth = info.authenticators(parse_result.hostname)
+                if auth:
+                  username, _account, password = auth
+                else:
+                  print('No credentials found for %s in .netrc'
+                        % parse_result.hostname, file=sys.stderr)
             except netrc.NetrcParseError as e:
               print('Error parsing .netrc file: %s' % e, file=sys.stderr)
 
@@ -575,8 +585,12 @@ later is required to fix a server side protocol bug.
                                                     (username, password),
                                                     1)
 
+      transport = PersistentTransport(manifest_server)
+      if manifest_server.startswith('persistent-'):
+        manifest_server = manifest_server[len('persistent-'):]
+
       try:
-        server = xmlrpc.client.Server(manifest_server)
+        server = xmlrpc.client.Server(manifest_server, transport=transport)
         if opt.smart_sync:
           p = self.manifest.manifestProject
           b = p.GetBranch(p.CurrentBranch)
@@ -656,6 +670,42 @@ later is required to fix a server side protocol bug.
       self._ReloadManifest(manifest_name)
       if opt.jobs is None:
         self.jobs = self.manifest.default.sync_j
+
+    if self.gitc_manifest:
+      gitc_manifest_projects = self.GetProjects(args,
+                                                missing_ok=True)
+      gitc_projects = []
+      opened_projects = []
+      for project in gitc_manifest_projects:
+        if project.relpath in self.gitc_manifest.paths and \
+           self.gitc_manifest.paths[project.relpath].old_revision:
+          opened_projects.append(project.relpath)
+        else:
+          gitc_projects.append(project.relpath)
+
+      if not args:
+        gitc_projects = None
+
+      if gitc_projects != [] and not opt.local_only:
+        print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
+        manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
+        if manifest_name:
+          manifest.Override(manifest_name)
+        else:
+          manifest.Override(self.manifest.manifestFile)
+        gitc_utils.generate_gitc_manifest(self.gitc_manifest,
+                                          manifest,
+                                          gitc_projects)
+        print('GITC client successfully synced.')
+
+      # The opened projects need to be synced as normal, therefore we
+      # generate a new args list to represent the opened projects.
+      # TODO: make this more reliable -- if there's a project name/path overlap,
+      # this may choose the wrong project.
+      args = [os.path.relpath(self.manifest.paths[p].worktree, os.getcwd())
+              for p in opened_projects]
+      if not args:
+        return
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
                                     submodules_ok=opt.fetch_submodules)
@@ -850,3 +900,96 @@ class _FetchTimes(object):
         os.remove(self._path)
       except OSError:
         pass
+
+# This is a replacement for xmlrpc.client.Transport using urllib2
+# and supporting persistent-http[s]. It cannot change hosts from
+# request to request like the normal transport, the real url
+# is passed during initialization.
+class PersistentTransport(xmlrpc.client.Transport):
+  def __init__(self, orig_host):
+    self.orig_host = orig_host
+
+  def request(self, host, handler, request_body, verbose=False):
+    with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
+      # Python doesn't understand cookies with the #HttpOnly_ prefix
+      # Since we're only using them for HTTP, copy the file temporarily,
+      # stripping those prefixes away.
+      if cookiefile:
+        tmpcookiefile = tempfile.NamedTemporaryFile()
+        try:
+          with open(cookiefile) as f:
+            for line in f:
+              if line.startswith("#HttpOnly_"):
+                line = line[len("#HttpOnly_"):]
+              tmpcookiefile.write(line)
+          tmpcookiefile.flush()
+
+          cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
+          cookiejar.load()
+        finally:
+          tmpcookiefile.close()
+      else:
+        cookiejar = cookielib.CookieJar()
+
+      proxyhandler = urllib.request.ProxyHandler
+      if proxy:
+        proxyhandler = urllib.request.ProxyHandler({
+            "http": proxy,
+            "https": proxy })
+
+      opener = urllib.request.build_opener(
+          urllib.request.HTTPCookieProcessor(cookiejar),
+          proxyhandler)
+
+      url = urllib.parse.urljoin(self.orig_host, handler)
+      parse_results = urllib.parse.urlparse(url)
+
+      scheme = parse_results.scheme
+      if scheme == 'persistent-http':
+        scheme = 'http'
+      if scheme == 'persistent-https':
+        # If we're proxying through persistent-https, use http. The
+        # proxy itself will do the https.
+        if proxy:
+          scheme = 'http'
+        else:
+          scheme = 'https'
+
+      # Parse out any authentication information using the base class
+      host, extra_headers, _ = self.get_host_info(parse_results.netloc)
+
+      url = urllib.parse.urlunparse((
+          scheme,
+          host,
+          parse_results.path,
+          parse_results.params,
+          parse_results.query,
+          parse_results.fragment))
+
+      request = urllib.request.Request(url, request_body)
+      if extra_headers is not None:
+        for (name, header) in extra_headers:
+          request.add_header(name, header)
+      request.add_header('Content-Type', 'text/xml')
+      try:
+        response = opener.open(request)
+      except urllib.error.HTTPError as e:
+        if e.code == 501:
+          # We may have been redirected through a login process
+          # but our POST turned into a GET. Retry.
+          response = opener.open(request)
+        else:
+          raise
+
+      p, u = xmlrpc.client.getparser()
+      while 1:
+        data = response.read(1024)
+        if not data:
+          break
+        p.feed(data)
+      p.close()
+      return u.close()
+
+  def close(self):
+    pass
+