summaryrefslogtreecommitdiffstats
path: root/subcmds/sync.py
diff options
context:
space:
mode:
Diffstat (limited to 'subcmds/sync.py')
-rw-r--r--subcmds/sync.py71
1 files changed, 42 insertions, 29 deletions
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 0c0f0cf3..e4e7a971 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -1190,6 +1190,45 @@ later is required to fix a server side protocol bug.
1190 'release. Use `--auto-gc` instead.', file=sys.stderr) 1190 'release. Use `--auto-gc` instead.', file=sys.stderr)
1191 opt.auto_gc = True 1191 opt.auto_gc = True
1192 1192
1193 def _ValidateOptionsWithManifest(self, opt, mp):
1194 """Like ValidateOptions, but after we've updated the manifest.
1195
1196 Needed to handle sync-xxx option defaults in the manifest.
1197
1198 Args:
1199 opt: The options to process.
1200 mp: The manifest project to pull defaults from.
1201 """
1202 if not opt.jobs:
1203 # If the user hasn't made a choice, use the manifest value.
1204 opt.jobs = mp.manifest.default.sync_j
1205 if opt.jobs:
1206 # If --jobs has a non-default value, propagate it as the default for
1207 # --jobs-xxx flags too.
1208 if not opt.jobs_network:
1209 opt.jobs_network = opt.jobs
1210 if not opt.jobs_checkout:
1211 opt.jobs_checkout = opt.jobs
1212 else:
1213 # Neither user nor manifest have made a choice, so setup defaults.
1214 if not opt.jobs_network:
1215 opt.jobs_network = 1
1216 if not opt.jobs_checkout:
1217 opt.jobs_checkout = DEFAULT_LOCAL_JOBS
1218 opt.jobs = os.cpu_count()
1219
1220 # Try to stay under user rlimit settings.
1221 #
1222 # Since each worker requires at 3 file descriptors to run `git fetch`, use
1223 # that to scale down the number of jobs. Unfortunately there isn't an easy
1224 # way to determine this reliably as systems change, but it was last measured
1225 # by hand in 2011.
1226 soft_limit, _ = _rlimit_nofile()
1227 jobs_soft_limit = max(1, (soft_limit - 5) // 3)
1228 opt.jobs = min(opt.jobs, jobs_soft_limit)
1229 opt.jobs_network = min(opt.jobs_network, jobs_soft_limit)
1230 opt.jobs_checkout = min(opt.jobs_checkout, jobs_soft_limit)
1231
1193 def Execute(self, opt, args): 1232 def Execute(self, opt, args):
1194 manifest = self.outer_manifest 1233 manifest = self.outer_manifest
1195 if not opt.outer_manifest: 1234 if not opt.outer_manifest:
@@ -1240,35 +1279,9 @@ later is required to fix a server side protocol bug.
1240 else: 1279 else:
1241 print('Skipping update of local manifest project.') 1280 print('Skipping update of local manifest project.')
1242 1281
1243 # Now that the manifests are up-to-date, setup the jobs value. 1282 # Now that the manifests are up-to-date, setup options whose defaults might
1244 if opt.jobs is None: 1283 # be in the manifest.
1245 # User has not made a choice, so use the manifest settings. 1284 self._ValidateOptionsWithManifest(opt, mp)
1246 opt.jobs = mp.default.sync_j
1247 if opt.jobs is not None:
1248 # Neither user nor manifest have made a choice.
1249 if opt.jobs_network is None:
1250 opt.jobs_network = opt.jobs
1251 if opt.jobs_checkout is None:
1252 opt.jobs_checkout = opt.jobs
1253 # Setup defaults if jobs==0.
1254 if not opt.jobs:
1255 if not opt.jobs_network:
1256 opt.jobs_network = 1
1257 if not opt.jobs_checkout:
1258 opt.jobs_checkout = DEFAULT_LOCAL_JOBS
1259 opt.jobs = os.cpu_count()
1260
1261 # Try to stay under user rlimit settings.
1262 #
1263 # Since each worker requires at 3 file descriptors to run `git fetch`, use
1264 # that to scale down the number of jobs. Unfortunately there isn't an easy
1265 # way to determine this reliably as systems change, but it was last measured
1266 # by hand in 2011.
1267 soft_limit, _ = _rlimit_nofile()
1268 jobs_soft_limit = max(1, (soft_limit - 5) // 3)
1269 opt.jobs = min(opt.jobs, jobs_soft_limit)
1270 opt.jobs_network = min(opt.jobs_network, jobs_soft_limit)
1271 opt.jobs_checkout = min(opt.jobs_checkout, jobs_soft_limit)
1272 1285
1273 superproject_logging_data = {} 1286 superproject_logging_data = {}
1274 self._UpdateProjectsRevisionId(opt, args, superproject_logging_data, 1287 self._UpdateProjectsRevisionId(opt, args, superproject_logging_data,