From 8d20116038ff78b22069dd4e993b5819775f03d1 Mon Sep 17 00:00:00 2001 From: David James Date: Fri, 11 Oct 2013 17:03:19 -0700 Subject: repo: Support multiple branches for the same project. It is often useful to be able to include the same project more than once, but with different branches and placed in different paths in the workspace. Add this feature. This CL adds the concept of an object directory. The object directory stores objects that can be shared amongst several working trees. For newly synced repositories, we set up the git repo now to share its objects with an object repo. Each worktree for a given repo shares objects, but has an independent set of references and branches. This ensures that repo only has to update the objects once; however the references for each worktree are updated separately. Storing the references separately is needed to ensure that commits to a branch on one worktree will not change the HEAD commits of the others. One nice side effect of sharing objects between different worktrees is that you can easily cherry-pick changes between the two worktrees without needing to fetch them. Bug: Issue 141 Change-Id: I5e2f4e1a7abb56f9d3f310fa6fd0c17019330ecd --- subcmds/sync.py | 54 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 12 deletions(-) (limited to 'subcmds/sync.py') diff --git a/subcmds/sync.py b/subcmds/sync.py index e9d52b7b..d1a06412 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -219,9 +219,25 @@ later is required to fix a server side protocol bug. dest='repo_upgraded', action='store_true', help=SUPPRESS_HELP) - def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event): + def _FetchProjectList(self, opt, projects, *args): """Main function of the fetch threads when jobs are > 1. + Delegates most of the work to _FetchHelper. + + Args: + opt: Program options returned from optparse. See _Options(). + projects: Projects to fetch. + *args: Remaining arguments to pass to _FetchHelper. See the + _FetchHelper docstring for details. + """ + for project in projects: + success = self._FetchHelper(opt, project, *args) + if not success and not opt.force_broken: + break + + def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event): + """Fetch git objects for a single project. + Args: opt: Program options returned from optparse. See _Options(). project: Project object for the project to fetch. @@ -235,6 +251,9 @@ later is required to fix a server side protocol bug. can be started up. err_event: We'll set this event in the case of an error (after printing out info about the error). + + Returns: + Whether the fetch was successful. """ # We'll set to true once we've locked the lock. did_lock = False @@ -281,6 +300,8 @@ later is required to fix a server side protocol bug. lock.release() sem.release() + return success + def _Fetch(self, projects, opt): fetched = set() pm = Progress('Fetching projects', len(projects)) @@ -303,20 +324,24 @@ later is required to fix a server side protocol bug. else: sys.exit(1) else: + objdir_project_map = dict() + for project in projects: + objdir_project_map.setdefault(project.objdir, []).append(project) + threads = set() lock = _threading.Lock() sem = _threading.Semaphore(self.jobs) err_event = _threading.Event() - for project in projects: + for project_list in objdir_project_map.values(): # Check for any errors before starting any new threads. # ...we'll let existing threads finish, though. if err_event.isSet(): break sem.acquire() - t = _threading.Thread(target = self._FetchHelper, + t = _threading.Thread(target = self._FetchProjectList, args = (opt, - project, + project_list, lock, fetched, pm, @@ -342,6 +367,10 @@ later is required to fix a server side protocol bug. return fetched def _GCProjects(self, projects): + gitdirs = {} + for project in projects: + gitdirs[project.gitdir] = project.bare_git + has_dash_c = git_require((1, 7, 2)) if multiprocessing and has_dash_c: cpu_count = multiprocessing.cpu_count() @@ -350,8 +379,8 @@ later is required to fix a server side protocol bug. jobs = min(self.jobs, cpu_count) if jobs < 2: - for project in projects: - project.bare_git.gc('--auto') + for bare_git in gitdirs.values(): + bare_git.gc('--auto') return config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1} @@ -360,10 +389,10 @@ later is required to fix a server side protocol bug. sem = _threading.Semaphore(jobs) err_event = _threading.Event() - def GC(project): + def GC(bare_git): try: try: - project.bare_git.gc('--auto', config=config) + bare_git.gc('--auto', config=config) except GitError: err_event.set() except: @@ -372,11 +401,11 @@ later is required to fix a server side protocol bug. finally: sem.release() - for project in projects: + for bare_git in gitdirs.values(): if err_event.isSet(): break sem.acquire() - t = _threading.Thread(target=GC, args=(project,)) + t = _threading.Thread(target=GC, args=(bare_git,)) t.daemon = True threads.add(t) t.start() @@ -416,12 +445,13 @@ later is required to fix a server side protocol bug. if path not in new_project_paths: # If the path has already been deleted, we don't need to do it if os.path.exists(self.manifest.topdir + '/' + path): + gitdir = os.path.join(self.manifest.topdir, path, '.git') project = Project( manifest = self.manifest, name = path, remote = RemoteSpec('origin'), - gitdir = os.path.join(self.manifest.topdir, - path, '.git'), + gitdir = gitdir, + objdir = gitdir, worktree = os.path.join(self.manifest.topdir, path), relpath = path, revisionExpr = 'HEAD', -- cgit v1.2.3-54-g00ecf