diff options
-rw-r--r-- | gitc_utils.py | 69 | ||||
-rw-r--r-- | subcmds/gitc_init.py | 63 | ||||
-rw-r--r-- | subcmds/sync.py | 29 |
3 files changed, 108 insertions, 53 deletions
diff --git a/gitc_utils.py b/gitc_utils.py new file mode 100644 index 00000000..bf79bd28 --- /dev/null +++ b/gitc_utils.py | |||
@@ -0,0 +1,69 @@ | |||
1 | # | ||
2 | # Copyright (C) 2015 The Android Open Source Project | ||
3 | # | ||
4 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | # you may not use this file except in compliance with the License. | ||
6 | # You may obtain a copy of the License at | ||
7 | # | ||
8 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | # | ||
10 | # Unless required by applicable law or agreed to in writing, software | ||
11 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | # See the License for the specific language governing permissions and | ||
14 | # limitations under the License. | ||
15 | |||
16 | from __future__ import print_function | ||
17 | import os | ||
18 | import shutil | ||
19 | |||
20 | import git_command | ||
21 | import git_config | ||
22 | |||
23 | |||
24 | # TODO (sbasi) - Remove this constant and fetch manifest dir from /gitc/.config | ||
25 | GITC_MANIFEST_DIR = '/usr/local/google/gitc/' | ||
26 | GITC_FS_ROOT_DIR = '/gitc/manifest-rw/' | ||
27 | NUM_BATCH_RETRIEVE_REVISIONID = 300 | ||
28 | |||
29 | def _set_project_revisions(projects): | ||
30 | """Sets the revisionExpr for a list of projects. | ||
31 | |||
32 | Because of the limit of open file descriptors allowed, length of projects | ||
33 | should not be overly large. Recommend calling this function multiple times | ||
34 | with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects. | ||
35 | |||
36 | @param projects: List of project objects to set the revionExpr for. | ||
37 | """ | ||
38 | # Retrieve the commit id for each project based off of it's current | ||
39 | # revisionExpr and it is not already a commit id. | ||
40 | project_gitcmds = [( | ||
41 | project, git_command.GitCommand(None, | ||
42 | ['ls-remote', | ||
43 | project.remote.url, | ||
44 | project.revisionExpr], | ||
45 | capture_stdout=True, cwd='/tmp')) | ||
46 | for project in projects if not git_config.IsId(project.revisionExpr)] | ||
47 | for proj, gitcmd in project_gitcmds: | ||
48 | if gitcmd.Wait(): | ||
49 | print('FATAL: Failed to retrieve revisionExpr for %s' % project) | ||
50 | sys.exit(1) | ||
51 | proj.revisionExpr = gitcmd.stdout.split('\t')[0] | ||
52 | |||
53 | def generate_gitc_manifest(client_dir, manifest): | ||
54 | """Generate a manifest for shafsd to use for this GITC client. | ||
55 | |||
56 | @param client_dir: GITC client directory to install the .manifest file in. | ||
57 | @param manifest: XmlManifest object representing the repo manifest. | ||
58 | """ | ||
59 | print('Generating GITC Manifest by fetching revision SHAs for each ' | ||
60 | 'project.') | ||
61 | project_gitcmd_dict = {} | ||
62 | index = 0 | ||
63 | while index < len(manifest.projects): | ||
64 | _set_project_revisions( | ||
65 | manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)]) | ||
66 | index += NUM_BATCH_RETRIEVE_REVISIONID | ||
67 | # Save the manifest. | ||
68 | with open(os.path.join(client_dir, '.manifest'), 'w') as f: | ||
69 | manifest.Save(f) | ||
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py index 9b9cefda..03d8cc30 100644 --- a/subcmds/gitc_init.py +++ b/subcmds/gitc_init.py | |||
@@ -18,15 +18,10 @@ import os | |||
18 | import shutil | 18 | import shutil |
19 | import sys | 19 | import sys |
20 | 20 | ||
21 | import git_command | 21 | import gitc_utils |
22 | from subcmds import init | 22 | from subcmds import init |
23 | 23 | ||
24 | 24 | ||
25 | GITC_MANIFEST_DIR = '/usr/local/google/gitc' | ||
26 | GITC_FS_ROOT_DIR = '/gitc/sha/rw' | ||
27 | NUM_BATCH_RETRIEVE_REVISIONID = 300 | ||
28 | |||
29 | |||
30 | class GitcInit(init.Init): | 25 | class GitcInit(init.Init): |
31 | common = True | 26 | common = True |
32 | helpSummary = "Initialize a GITC Client." | 27 | helpSummary = "Initialize a GITC Client." |
@@ -65,59 +60,21 @@ use for this GITC client. | |||
65 | if not opt.gitc_client: | 60 | if not opt.gitc_client: |
66 | print('fatal: gitc client (-c) is required', file=sys.stderr) | 61 | print('fatal: gitc client (-c) is required', file=sys.stderr) |
67 | sys.exit(1) | 62 | sys.exit(1) |
68 | self.client_dir = os.path.join(GITC_MANIFEST_DIR, opt.gitc_client) | 63 | self.client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR, |
69 | if not os.path.exists(GITC_MANIFEST_DIR): | 64 | opt.gitc_client) |
70 | os.makedirs(GITC_MANIFEST_DIR) | 65 | if not os.path.exists(gitc_utils.GITC_MANIFEST_DIR): |
66 | os.makedirs(gitc_utils.GITC_MANIFEST_DIR) | ||
71 | if not os.path.exists(self.client_dir): | 67 | if not os.path.exists(self.client_dir): |
72 | os.mkdir(self.client_dir) | 68 | os.mkdir(self.client_dir) |
73 | super(GitcInit, self).Execute(opt, args) | 69 | super(GitcInit, self).Execute(opt, args) |
70 | # Make the destination manifest file a symlink to repo's so both repo and | ||
71 | # GITC refer to the same manifest. | ||
74 | if opt.manifest_file: | 72 | if opt.manifest_file: |
75 | if not os.path.exists(opt.manifest_file): | 73 | if not os.path.exists(opt.manifest_file): |
76 | print('fatal: Specified manifest file %s does not exist.' % | 74 | print('fatal: Specified manifest file %s does not exist.' % |
77 | opt.manifest_file) | 75 | opt.manifest_file) |
78 | sys.exit(1) | 76 | sys.exit(1) |
79 | shutil.copyfile(opt.manifest_file, | 77 | self.manifest.Override(opt.manifest_file) |
80 | os.path.join(self.client_dir, '.manifest')) | 78 | gitc_utils.generate_gitc_manifest(self.client_dir, self.manifest) |
81 | else: | ||
82 | self._GenerateGITCManifest() | ||
83 | print('Please run `cd %s` to view your GITC client.' % | 79 | print('Please run `cd %s` to view your GITC client.' % |
84 | os.path.join(GITC_FS_ROOT_DIR, opt.gitc_client)) | 80 | os.path.join(gitc_utils.GITC_FS_ROOT_DIR, opt.gitc_client)) \ No newline at end of file |
85 | |||
86 | def _SetProjectRevisions(self, projects, branch): | ||
87 | """Sets the revisionExpr for a list of projects. | ||
88 | |||
89 | Because of the limit of open file descriptors allowed, length of projects | ||
90 | should not be overly large. Recommend calling this function multiple times | ||
91 | with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects. | ||
92 | |||
93 | @param projects: List of project objects to set the revionExpr for. | ||
94 | @param branch: The remote branch to retrieve the SHA from. If branch is | ||
95 | None, 'HEAD' is used. | ||
96 | """ | ||
97 | project_gitcmds = [( | ||
98 | project, git_command.GitCommand(None, | ||
99 | ['ls-remote', | ||
100 | project.remote.url, | ||
101 | branch], capture_stdout=True)) | ||
102 | for project in projects] | ||
103 | for proj, gitcmd in project_gitcmds: | ||
104 | if gitcmd.Wait(): | ||
105 | print('FATAL: Failed to retrieve revisionID for %s' % project) | ||
106 | sys.exit(1) | ||
107 | proj.revisionExpr = gitcmd.stdout.split('\t')[0] | ||
108 | |||
109 | def _GenerateGITCManifest(self): | ||
110 | """Generate a manifest for shafsd to use for this GITC client.""" | ||
111 | print('Generating GITC Manifest by fetching revision SHAs for each ' | ||
112 | 'project.') | ||
113 | manifest = self.manifest | ||
114 | project_gitcmd_dict = {} | ||
115 | index = 0 | ||
116 | while index < len(manifest.projects): | ||
117 | self._SetProjectRevisions( | ||
118 | manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)], | ||
119 | manifest.default.revisionExpr) | ||
120 | index += NUM_BATCH_RETRIEVE_REVISIONID | ||
121 | # Save the manifest. | ||
122 | with open(os.path.join(self.client_dir, '.manifest'), 'w') as f: | ||
123 | manifest.Save(f) | ||
diff --git a/subcmds/sync.py b/subcmds/sync.py index 43d450be..652a0c0d 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
@@ -58,6 +58,7 @@ except ImportError: | |||
58 | 58 | ||
59 | from git_command import GIT, git_require | 59 | from git_command import GIT, git_require |
60 | from git_refs import R_HEADS, HEAD | 60 | from git_refs import R_HEADS, HEAD |
61 | import gitc_utils | ||
61 | from project import Project | 62 | from project import Project |
62 | from project import RemoteSpec | 63 | from project import RemoteSpec |
63 | from command import Command, MirrorSafeCommand | 64 | from command import Command, MirrorSafeCommand |
@@ -184,6 +185,9 @@ later is required to fix a server side protocol bug. | |||
184 | help="overwrite an existing git directory if it needs to " | 185 | help="overwrite an existing git directory if it needs to " |
185 | "point to a different object directory. WARNING: this " | 186 | "point to a different object directory. WARNING: this " |
186 | "may cause loss of data") | 187 | "may cause loss of data") |
188 | p.add_option('--force-gitc', | ||
189 | dest='force_gitc', action='store_true', | ||
190 | help="actually sync sources in the gitc client directory.") | ||
187 | p.add_option('-l', '--local-only', | 191 | p.add_option('-l', '--local-only', |
188 | dest='local_only', action='store_true', | 192 | dest='local_only', action='store_true', |
189 | help="only update working tree, don't fetch") | 193 | help="only update working tree, don't fetch") |
@@ -526,6 +530,25 @@ later is required to fix a server side protocol bug. | |||
526 | print('error: both -u and -p must be given', file=sys.stderr) | 530 | print('error: both -u and -p must be given', file=sys.stderr) |
527 | sys.exit(1) | 531 | sys.exit(1) |
528 | 532 | ||
533 | cwd = os.getcwd() | ||
534 | if cwd.startswith(gitc_utils.GITC_MANIFEST_DIR) and not opt.force_gitc: | ||
535 | print('WARNING this will pull all the sources like a normal repo sync.\n' | ||
536 | '\nIf you want to update your GITC Client View please rerun this ' | ||
537 | 'command in \n%s%s.\nOr if you actually want to pull the sources, ' | ||
538 | 'rerun with --force-gitc.' % | ||
539 | (gitc_utils.GITC_FS_ROOT_DIR, | ||
540 | cwd.split(gitc_utils.GITC_MANIFEST_DIR)[1])) | ||
541 | sys.exit(1) | ||
542 | |||
543 | self._gitc_sync = False | ||
544 | if cwd.startswith(gitc_utils.GITC_FS_ROOT_DIR): | ||
545 | self._gitc_sync = True | ||
546 | self._client_name = cwd.split(gitc_utils.GITC_FS_ROOT_DIR)[1].split( | ||
547 | '/')[0] | ||
548 | self._client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR, | ||
549 | self._client_name) | ||
550 | print('Updating GITC client: %s' % self._client_name) | ||
551 | |||
529 | if opt.manifest_name: | 552 | if opt.manifest_name: |
530 | self.manifest.Override(opt.manifest_name) | 553 | self.manifest.Override(opt.manifest_name) |
531 | 554 | ||
@@ -642,6 +665,12 @@ later is required to fix a server side protocol bug. | |||
642 | if opt.repo_upgraded: | 665 | if opt.repo_upgraded: |
643 | _PostRepoUpgrade(self.manifest, quiet=opt.quiet) | 666 | _PostRepoUpgrade(self.manifest, quiet=opt.quiet) |
644 | 667 | ||
668 | if self._gitc_sync: | ||
669 | gitc_utils.generate_gitc_manifest(self._client_dir, self.manifest) | ||
670 | print('GITC client successfully synced.') | ||
671 | return | ||
672 | |||
673 | |||
645 | if not opt.local_only: | 674 | if not opt.local_only: |
646 | mp.Sync_NetworkHalf(quiet=opt.quiet, | 675 | mp.Sync_NetworkHalf(quiet=opt.quiet, |
647 | current_branch_only=opt.current_branch_only, | 676 | current_branch_only=opt.current_branch_only, |