diff options
-rw-r--r-- | subcmds/sync.py | 68 | ||||
-rw-r--r-- | tests/test_subcmds_sync.py | 102 |
2 files changed, 160 insertions, 10 deletions
diff --git a/subcmds/sync.py b/subcmds/sync.py index e75a8154..b848d137 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
@@ -989,26 +989,16 @@ later is required to fix a server side protocol bug. | |||
989 | Returns: | 989 | Returns: |
990 | List of all projects that should be checked out. | 990 | List of all projects that should be checked out. |
991 | """ | 991 | """ |
992 | rp = manifest.repoProject | ||
993 | |||
994 | to_fetch = [] | 992 | to_fetch = [] |
995 | now = time.time() | ||
996 | if _ONE_DAY_S <= (now - rp.LastFetch): | ||
997 | to_fetch.append(rp) | ||
998 | to_fetch.extend(all_projects) | 993 | to_fetch.extend(all_projects) |
999 | to_fetch.sort(key=self._fetch_times.Get, reverse=True) | 994 | to_fetch.sort(key=self._fetch_times.Get, reverse=True) |
1000 | 995 | ||
1001 | result = self._Fetch(to_fetch, opt, err_event, ssh_proxy, errors) | 996 | result = self._Fetch(to_fetch, opt, err_event, ssh_proxy, errors) |
1002 | success = result.success | 997 | success = result.success |
1003 | fetched = result.projects | 998 | fetched = result.projects |
1004 | |||
1005 | if not success: | 999 | if not success: |
1006 | err_event.set() | 1000 | err_event.set() |
1007 | 1001 | ||
1008 | # Call self update, unless requested not to | ||
1009 | # TODO(b/42193561): Extract repo update logic to ExecuteHelper. | ||
1010 | if os.environ.get("REPO_SKIP_SELF_UPDATE", "0") == "0": | ||
1011 | _PostRepoFetch(rp, opt.repo_verify) | ||
1012 | if opt.network_only: | 1002 | if opt.network_only: |
1013 | # Bail out now; the rest touches the working tree. | 1003 | # Bail out now; the rest touches the working tree. |
1014 | if err_event.is_set(): | 1004 | if err_event.is_set(): |
@@ -1369,6 +1359,61 @@ later is required to fix a server side protocol bug. | |||
1369 | t.join() | 1359 | t.join() |
1370 | pm.end() | 1360 | pm.end() |
1371 | 1361 | ||
1362 | def _UpdateRepoProject(self, opt, manifest, errors): | ||
1363 | """Fetch the repo project and check for updates.""" | ||
1364 | if opt.local_only: | ||
1365 | return | ||
1366 | |||
1367 | rp = manifest.repoProject | ||
1368 | now = time.time() | ||
1369 | # If we've fetched in the last day, don't bother fetching again. | ||
1370 | if (now - rp.LastFetch) < _ONE_DAY_S: | ||
1371 | return | ||
1372 | |||
1373 | with multiprocessing.Manager() as manager: | ||
1374 | with ssh.ProxyManager(manager) as ssh_proxy: | ||
1375 | ssh_proxy.sock() | ||
1376 | start = time.time() | ||
1377 | buf = TeeStringIO(sys.stdout if opt.verbose else None) | ||
1378 | sync_result = rp.Sync_NetworkHalf( | ||
1379 | quiet=opt.quiet, | ||
1380 | verbose=opt.verbose, | ||
1381 | output_redir=buf, | ||
1382 | current_branch_only=self._GetCurrentBranchOnly( | ||
1383 | opt, manifest | ||
1384 | ), | ||
1385 | force_sync=opt.force_sync, | ||
1386 | clone_bundle=opt.clone_bundle, | ||
1387 | tags=opt.tags, | ||
1388 | archive=manifest.IsArchive, | ||
1389 | optimized_fetch=opt.optimized_fetch, | ||
1390 | retry_fetches=opt.retry_fetches, | ||
1391 | prune=opt.prune, | ||
1392 | ssh_proxy=ssh_proxy, | ||
1393 | clone_filter=manifest.CloneFilter, | ||
1394 | partial_clone_exclude=manifest.PartialCloneExclude, | ||
1395 | clone_filter_for_depth=manifest.CloneFilterForDepth, | ||
1396 | ) | ||
1397 | if sync_result.error: | ||
1398 | errors.append(sync_result.error) | ||
1399 | |||
1400 | finish = time.time() | ||
1401 | self.event_log.AddSync( | ||
1402 | rp, | ||
1403 | event_log.TASK_SYNC_NETWORK, | ||
1404 | start, | ||
1405 | finish, | ||
1406 | sync_result.success, | ||
1407 | ) | ||
1408 | if not sync_result.success: | ||
1409 | logger.error("error: Cannot fetch repo tool %s", rp.name) | ||
1410 | return | ||
1411 | |||
1412 | # After fetching, check if a new version of repo is available and | ||
1413 | # restart. This is only done if the user hasn't explicitly disabled it. | ||
1414 | if os.environ.get("REPO_SKIP_SELF_UPDATE", "0") == "0": | ||
1415 | _PostRepoFetch(rp, opt.repo_verify) | ||
1416 | |||
1372 | def _ReloadManifest(self, manifest_name, manifest): | 1417 | def _ReloadManifest(self, manifest_name, manifest): |
1373 | """Reload the manfiest from the file specified by the |manifest_name|. | 1418 | """Reload the manfiest from the file specified by the |manifest_name|. |
1374 | 1419 | ||
@@ -1871,6 +1916,9 @@ later is required to fix a server side protocol bug. | |||
1871 | # might be in the manifest. | 1916 | # might be in the manifest. |
1872 | self._ValidateOptionsWithManifest(opt, mp) | 1917 | self._ValidateOptionsWithManifest(opt, mp) |
1873 | 1918 | ||
1919 | # Update the repo project and check for new versions of repo. | ||
1920 | self._UpdateRepoProject(opt, manifest, errors) | ||
1921 | |||
1874 | superproject_logging_data = {} | 1922 | superproject_logging_data = {} |
1875 | self._UpdateProjectsRevisionId( | 1923 | self._UpdateProjectsRevisionId( |
1876 | opt, args, superproject_logging_data, manifest | 1924 | opt, args, superproject_logging_data, manifest |
diff --git a/tests/test_subcmds_sync.py b/tests/test_subcmds_sync.py index e7213ed9..9cd19f10 100644 --- a/tests/test_subcmds_sync.py +++ b/tests/test_subcmds_sync.py | |||
@@ -527,6 +527,108 @@ class SyncCommand(unittest.TestCase): | |||
527 | self.assertIn(self.sync_network_half_error, e.aggregate_errors) | 527 | self.assertIn(self.sync_network_half_error, e.aggregate_errors) |
528 | 528 | ||
529 | 529 | ||
530 | class SyncUpdateRepoProject(unittest.TestCase): | ||
531 | """Tests for Sync._UpdateRepoProject.""" | ||
532 | |||
533 | def setUp(self): | ||
534 | """Common setup.""" | ||
535 | self.repodir = tempfile.mkdtemp(".repo") | ||
536 | self.manifest = manifest = mock.MagicMock(repodir=self.repodir) | ||
537 | # Create a repoProject with a mock Sync_NetworkHalf. | ||
538 | repoProject = mock.MagicMock(name="repo") | ||
539 | repoProject.Sync_NetworkHalf = mock.Mock( | ||
540 | return_value=SyncNetworkHalfResult(True, None) | ||
541 | ) | ||
542 | manifest.repoProject = repoProject | ||
543 | manifest.IsArchive = False | ||
544 | manifest.CloneFilter = None | ||
545 | manifest.PartialCloneExclude = None | ||
546 | manifest.CloneFilterForDepth = None | ||
547 | |||
548 | git_event_log = mock.MagicMock(ErrorEvent=mock.Mock(return_value=None)) | ||
549 | self.cmd = sync.Sync(manifest=manifest, git_event_log=git_event_log) | ||
550 | |||
551 | opt, _ = self.cmd.OptionParser.parse_args([]) | ||
552 | opt.local_only = False | ||
553 | opt.repo_verify = False | ||
554 | opt.verbose = False | ||
555 | opt.quiet = True | ||
556 | opt.force_sync = False | ||
557 | opt.clone_bundle = False | ||
558 | opt.tags = False | ||
559 | opt.optimized_fetch = False | ||
560 | opt.retry_fetches = 0 | ||
561 | opt.prune = False | ||
562 | self.opt = opt | ||
563 | self.errors = [] | ||
564 | |||
565 | mock.patch.object(sync.Sync, "_GetCurrentBranchOnly").start() | ||
566 | |||
567 | def tearDown(self): | ||
568 | shutil.rmtree(self.repodir) | ||
569 | mock.patch.stopall() | ||
570 | |||
571 | def test_fetches_when_stale(self): | ||
572 | """Test it fetches when the repo project is stale.""" | ||
573 | self.manifest.repoProject.LastFetch = time.time() - ( | ||
574 | sync._ONE_DAY_S + 1 | ||
575 | ) | ||
576 | |||
577 | with mock.patch.object(sync, "_PostRepoFetch") as mock_post_fetch: | ||
578 | self.cmd._UpdateRepoProject(self.opt, self.manifest, self.errors) | ||
579 | self.manifest.repoProject.Sync_NetworkHalf.assert_called_once() | ||
580 | mock_post_fetch.assert_called_once() | ||
581 | self.assertEqual(self.errors, []) | ||
582 | |||
583 | def test_skips_when_fresh(self): | ||
584 | """Test it skips fetch when repo project is fresh.""" | ||
585 | self.manifest.repoProject.LastFetch = time.time() | ||
586 | |||
587 | with mock.patch.object(sync, "_PostRepoFetch") as mock_post_fetch: | ||
588 | self.cmd._UpdateRepoProject(self.opt, self.manifest, self.errors) | ||
589 | self.manifest.repoProject.Sync_NetworkHalf.assert_not_called() | ||
590 | mock_post_fetch.assert_not_called() | ||
591 | |||
592 | def test_skips_local_only(self): | ||
593 | """Test it does nothing with --local-only.""" | ||
594 | self.opt.local_only = True | ||
595 | self.manifest.repoProject.LastFetch = time.time() - ( | ||
596 | sync._ONE_DAY_S + 1 | ||
597 | ) | ||
598 | |||
599 | with mock.patch.object(sync, "_PostRepoFetch") as mock_post_fetch: | ||
600 | self.cmd._UpdateRepoProject(self.opt, self.manifest, self.errors) | ||
601 | self.manifest.repoProject.Sync_NetworkHalf.assert_not_called() | ||
602 | mock_post_fetch.assert_not_called() | ||
603 | |||
604 | def test_post_repo_fetch_skipped_on_env_var(self): | ||
605 | """Test _PostRepoFetch is skipped when REPO_SKIP_SELF_UPDATE is set.""" | ||
606 | self.manifest.repoProject.LastFetch = time.time() | ||
607 | |||
608 | with mock.patch.dict(os.environ, {"REPO_SKIP_SELF_UPDATE": "1"}): | ||
609 | with mock.patch.object(sync, "_PostRepoFetch") as mock_post_fetch: | ||
610 | self.cmd._UpdateRepoProject( | ||
611 | self.opt, self.manifest, self.errors | ||
612 | ) | ||
613 | mock_post_fetch.assert_not_called() | ||
614 | |||
615 | def test_fetch_failure_is_handled(self): | ||
616 | """Test that a fetch failure is recorded and doesn't crash.""" | ||
617 | self.manifest.repoProject.LastFetch = time.time() - ( | ||
618 | sync._ONE_DAY_S + 1 | ||
619 | ) | ||
620 | fetch_error = GitError("Fetch failed") | ||
621 | self.manifest.repoProject.Sync_NetworkHalf.return_value = ( | ||
622 | SyncNetworkHalfResult(False, fetch_error) | ||
623 | ) | ||
624 | |||
625 | with mock.patch.object(sync, "_PostRepoFetch") as mock_post_fetch: | ||
626 | self.cmd._UpdateRepoProject(self.opt, self.manifest, self.errors) | ||
627 | self.manifest.repoProject.Sync_NetworkHalf.assert_called_once() | ||
628 | mock_post_fetch.assert_not_called() | ||
629 | self.assertEqual(self.errors, [fetch_error]) | ||
630 | |||
631 | |||
530 | class InterleavedSyncTest(unittest.TestCase): | 632 | class InterleavedSyncTest(unittest.TestCase): |
531 | """Tests for interleaved sync.""" | 633 | """Tests for interleaved sync.""" |
532 | 634 | ||