diff options
Diffstat (limited to 'project.py')
-rw-r--r-- | project.py | 404 |
1 files changed, 1 insertions, 403 deletions
@@ -18,7 +18,6 @@ from __future__ import print_function | |||
18 | import errno | 18 | import errno |
19 | import filecmp | 19 | import filecmp |
20 | import glob | 20 | import glob |
21 | import json | ||
22 | import os | 21 | import os |
23 | import random | 22 | import random |
24 | import re | 23 | import re |
@@ -29,13 +28,12 @@ import sys | |||
29 | import tarfile | 28 | import tarfile |
30 | import tempfile | 29 | import tempfile |
31 | import time | 30 | import time |
32 | import traceback | ||
33 | 31 | ||
34 | from color import Coloring | 32 | from color import Coloring |
35 | from git_command import GitCommand, git_require | 33 | from git_command import GitCommand, git_require |
36 | from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \ | 34 | from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \ |
37 | ID_RE | 35 | ID_RE |
38 | from error import GitError, HookError, UploadError, DownloadError | 36 | from error import GitError, UploadError, DownloadError |
39 | from error import ManifestInvalidRevisionError, ManifestInvalidPathError | 37 | from error import ManifestInvalidRevisionError, ManifestInvalidPathError |
40 | from error import NoManifestException | 38 | from error import NoManifestException |
41 | import platform_utils | 39 | import platform_utils |
@@ -451,406 +449,6 @@ class RemoteSpec(object): | |||
451 | self.orig_name = orig_name | 449 | self.orig_name = orig_name |
452 | self.fetchUrl = fetchUrl | 450 | self.fetchUrl = fetchUrl |
453 | 451 | ||
454 | |||
455 | class RepoHook(object): | ||
456 | |||
457 | """A RepoHook contains information about a script to run as a hook. | ||
458 | |||
459 | Hooks are used to run a python script before running an upload (for instance, | ||
460 | to run presubmit checks). Eventually, we may have hooks for other actions. | ||
461 | |||
462 | This shouldn't be confused with files in the 'repo/hooks' directory. Those | ||
463 | files are copied into each '.git/hooks' folder for each project. Repo-level | ||
464 | hooks are associated instead with repo actions. | ||
465 | |||
466 | Hooks are always python. When a hook is run, we will load the hook into the | ||
467 | interpreter and execute its main() function. | ||
468 | """ | ||
469 | |||
470 | def __init__(self, | ||
471 | hook_type, | ||
472 | hooks_project, | ||
473 | topdir, | ||
474 | manifest_url, | ||
475 | abort_if_user_denies=False): | ||
476 | """RepoHook constructor. | ||
477 | |||
478 | Params: | ||
479 | hook_type: A string representing the type of hook. This is also used | ||
480 | to figure out the name of the file containing the hook. For | ||
481 | example: 'pre-upload'. | ||
482 | hooks_project: The project containing the repo hooks. If you have a | ||
483 | manifest, this is manifest.repo_hooks_project. OK if this is None, | ||
484 | which will make the hook a no-op. | ||
485 | topdir: Repo's top directory (the one containing the .repo directory). | ||
486 | Scripts will run with CWD as this directory. If you have a manifest, | ||
487 | this is manifest.topdir | ||
488 | manifest_url: The URL to the manifest git repo. | ||
489 | abort_if_user_denies: If True, we'll throw a HookError() if the user | ||
490 | doesn't allow us to run the hook. | ||
491 | """ | ||
492 | self._hook_type = hook_type | ||
493 | self._hooks_project = hooks_project | ||
494 | self._manifest_url = manifest_url | ||
495 | self._topdir = topdir | ||
496 | self._abort_if_user_denies = abort_if_user_denies | ||
497 | |||
498 | # Store the full path to the script for convenience. | ||
499 | if self._hooks_project: | ||
500 | self._script_fullpath = os.path.join(self._hooks_project.worktree, | ||
501 | self._hook_type + '.py') | ||
502 | else: | ||
503 | self._script_fullpath = None | ||
504 | |||
505 | def _GetHash(self): | ||
506 | """Return a hash of the contents of the hooks directory. | ||
507 | |||
508 | We'll just use git to do this. This hash has the property that if anything | ||
509 | changes in the directory we will return a different has. | ||
510 | |||
511 | SECURITY CONSIDERATION: | ||
512 | This hash only represents the contents of files in the hook directory, not | ||
513 | any other files imported or called by hooks. Changes to imported files | ||
514 | can change the script behavior without affecting the hash. | ||
515 | |||
516 | Returns: | ||
517 | A string representing the hash. This will always be ASCII so that it can | ||
518 | be printed to the user easily. | ||
519 | """ | ||
520 | assert self._hooks_project, "Must have hooks to calculate their hash." | ||
521 | |||
522 | # We will use the work_git object rather than just calling GetRevisionId(). | ||
523 | # That gives us a hash of the latest checked in version of the files that | ||
524 | # the user will actually be executing. Specifically, GetRevisionId() | ||
525 | # doesn't appear to change even if a user checks out a different version | ||
526 | # of the hooks repo (via git checkout) nor if a user commits their own revs. | ||
527 | # | ||
528 | # NOTE: Local (non-committed) changes will not be factored into this hash. | ||
529 | # I think this is OK, since we're really only worried about warning the user | ||
530 | # about upstream changes. | ||
531 | return self._hooks_project.work_git.rev_parse('HEAD') | ||
532 | |||
533 | def _GetMustVerb(self): | ||
534 | """Return 'must' if the hook is required; 'should' if not.""" | ||
535 | if self._abort_if_user_denies: | ||
536 | return 'must' | ||
537 | else: | ||
538 | return 'should' | ||
539 | |||
540 | def _CheckForHookApproval(self): | ||
541 | """Check to see whether this hook has been approved. | ||
542 | |||
543 | We'll accept approval of manifest URLs if they're using secure transports. | ||
544 | This way the user can say they trust the manifest hoster. For insecure | ||
545 | hosts, we fall back to checking the hash of the hooks repo. | ||
546 | |||
547 | Note that we ask permission for each individual hook even though we use | ||
548 | the hash of all hooks when detecting changes. We'd like the user to be | ||
549 | able to approve / deny each hook individually. We only use the hash of all | ||
550 | hooks because there is no other easy way to detect changes to local imports. | ||
551 | |||
552 | Returns: | ||
553 | True if this hook is approved to run; False otherwise. | ||
554 | |||
555 | Raises: | ||
556 | HookError: Raised if the user doesn't approve and abort_if_user_denies | ||
557 | was passed to the consturctor. | ||
558 | """ | ||
559 | if self._ManifestUrlHasSecureScheme(): | ||
560 | return self._CheckForHookApprovalManifest() | ||
561 | else: | ||
562 | return self._CheckForHookApprovalHash() | ||
563 | |||
564 | def _CheckForHookApprovalHelper(self, subkey, new_val, main_prompt, | ||
565 | changed_prompt): | ||
566 | """Check for approval for a particular attribute and hook. | ||
567 | |||
568 | Args: | ||
569 | subkey: The git config key under [repo.hooks.<hook_type>] to store the | ||
570 | last approved string. | ||
571 | new_val: The new value to compare against the last approved one. | ||
572 | main_prompt: Message to display to the user to ask for approval. | ||
573 | changed_prompt: Message explaining why we're re-asking for approval. | ||
574 | |||
575 | Returns: | ||
576 | True if this hook is approved to run; False otherwise. | ||
577 | |||
578 | Raises: | ||
579 | HookError: Raised if the user doesn't approve and abort_if_user_denies | ||
580 | was passed to the consturctor. | ||
581 | """ | ||
582 | hooks_config = self._hooks_project.config | ||
583 | git_approval_key = 'repo.hooks.%s.%s' % (self._hook_type, subkey) | ||
584 | |||
585 | # Get the last value that the user approved for this hook; may be None. | ||
586 | old_val = hooks_config.GetString(git_approval_key) | ||
587 | |||
588 | if old_val is not None: | ||
589 | # User previously approved hook and asked not to be prompted again. | ||
590 | if new_val == old_val: | ||
591 | # Approval matched. We're done. | ||
592 | return True | ||
593 | else: | ||
594 | # Give the user a reason why we're prompting, since they last told | ||
595 | # us to "never ask again". | ||
596 | prompt = 'WARNING: %s\n\n' % (changed_prompt,) | ||
597 | else: | ||
598 | prompt = '' | ||
599 | |||
600 | # Prompt the user if we're not on a tty; on a tty we'll assume "no". | ||
601 | if sys.stdout.isatty(): | ||
602 | prompt += main_prompt + ' (yes/always/NO)? ' | ||
603 | response = input(prompt).lower() | ||
604 | print() | ||
605 | |||
606 | # User is doing a one-time approval. | ||
607 | if response in ('y', 'yes'): | ||
608 | return True | ||
609 | elif response == 'always': | ||
610 | hooks_config.SetString(git_approval_key, new_val) | ||
611 | return True | ||
612 | |||
613 | # For anything else, we'll assume no approval. | ||
614 | if self._abort_if_user_denies: | ||
615 | raise HookError('You must allow the %s hook or use --no-verify.' % | ||
616 | self._hook_type) | ||
617 | |||
618 | return False | ||
619 | |||
620 | def _ManifestUrlHasSecureScheme(self): | ||
621 | """Check if the URI for the manifest is a secure transport.""" | ||
622 | secure_schemes = ('file', 'https', 'ssh', 'persistent-https', 'sso', 'rpc') | ||
623 | parse_results = urllib.parse.urlparse(self._manifest_url) | ||
624 | return parse_results.scheme in secure_schemes | ||
625 | |||
626 | def _CheckForHookApprovalManifest(self): | ||
627 | """Check whether the user has approved this manifest host. | ||
628 | |||
629 | Returns: | ||
630 | True if this hook is approved to run; False otherwise. | ||
631 | """ | ||
632 | return self._CheckForHookApprovalHelper( | ||
633 | 'approvedmanifest', | ||
634 | self._manifest_url, | ||
635 | 'Run hook scripts from %s' % (self._manifest_url,), | ||
636 | 'Manifest URL has changed since %s was allowed.' % (self._hook_type,)) | ||
637 | |||
638 | def _CheckForHookApprovalHash(self): | ||
639 | """Check whether the user has approved the hooks repo. | ||
640 | |||
641 | Returns: | ||
642 | True if this hook is approved to run; False otherwise. | ||
643 | """ | ||
644 | prompt = ('Repo %s run the script:\n' | ||
645 | ' %s\n' | ||
646 | '\n' | ||
647 | 'Do you want to allow this script to run') | ||
648 | return self._CheckForHookApprovalHelper( | ||
649 | 'approvedhash', | ||
650 | self._GetHash(), | ||
651 | prompt % (self._GetMustVerb(), self._script_fullpath), | ||
652 | 'Scripts have changed since %s was allowed.' % (self._hook_type,)) | ||
653 | |||
654 | @staticmethod | ||
655 | def _ExtractInterpFromShebang(data): | ||
656 | """Extract the interpreter used in the shebang. | ||
657 | |||
658 | Try to locate the interpreter the script is using (ignoring `env`). | ||
659 | |||
660 | Args: | ||
661 | data: The file content of the script. | ||
662 | |||
663 | Returns: | ||
664 | The basename of the main script interpreter, or None if a shebang is not | ||
665 | used or could not be parsed out. | ||
666 | """ | ||
667 | firstline = data.splitlines()[:1] | ||
668 | if not firstline: | ||
669 | return None | ||
670 | |||
671 | # The format here can be tricky. | ||
672 | shebang = firstline[0].strip() | ||
673 | m = re.match(r'^#!\s*([^\s]+)(?:\s+([^\s]+))?', shebang) | ||
674 | if not m: | ||
675 | return None | ||
676 | |||
677 | # If the using `env`, find the target program. | ||
678 | interp = m.group(1) | ||
679 | if os.path.basename(interp) == 'env': | ||
680 | interp = m.group(2) | ||
681 | |||
682 | return interp | ||
683 | |||
684 | def _ExecuteHookViaReexec(self, interp, context, **kwargs): | ||
685 | """Execute the hook script through |interp|. | ||
686 | |||
687 | Note: Support for this feature should be dropped ~Jun 2021. | ||
688 | |||
689 | Args: | ||
690 | interp: The Python program to run. | ||
691 | context: Basic Python context to execute the hook inside. | ||
692 | kwargs: Arbitrary arguments to pass to the hook script. | ||
693 | |||
694 | Raises: | ||
695 | HookError: When the hooks failed for any reason. | ||
696 | """ | ||
697 | # This logic needs to be kept in sync with _ExecuteHookViaImport below. | ||
698 | script = """ | ||
699 | import json, os, sys | ||
700 | path = '''%(path)s''' | ||
701 | kwargs = json.loads('''%(kwargs)s''') | ||
702 | context = json.loads('''%(context)s''') | ||
703 | sys.path.insert(0, os.path.dirname(path)) | ||
704 | data = open(path).read() | ||
705 | exec(compile(data, path, 'exec'), context) | ||
706 | context['main'](**kwargs) | ||
707 | """ % { | ||
708 | 'path': self._script_fullpath, | ||
709 | 'kwargs': json.dumps(kwargs), | ||
710 | 'context': json.dumps(context), | ||
711 | } | ||
712 | |||
713 | # We pass the script via stdin to avoid OS argv limits. It also makes | ||
714 | # unhandled exception tracebacks less verbose/confusing for users. | ||
715 | cmd = [interp, '-c', 'import sys; exec(sys.stdin.read())'] | ||
716 | proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) | ||
717 | proc.communicate(input=script.encode('utf-8')) | ||
718 | if proc.returncode: | ||
719 | raise HookError('Failed to run %s hook.' % (self._hook_type,)) | ||
720 | |||
721 | def _ExecuteHookViaImport(self, data, context, **kwargs): | ||
722 | """Execute the hook code in |data| directly. | ||
723 | |||
724 | Args: | ||
725 | data: The code of the hook to execute. | ||
726 | context: Basic Python context to execute the hook inside. | ||
727 | kwargs: Arbitrary arguments to pass to the hook script. | ||
728 | |||
729 | Raises: | ||
730 | HookError: When the hooks failed for any reason. | ||
731 | """ | ||
732 | # Exec, storing global context in the context dict. We catch exceptions | ||
733 | # and convert to a HookError w/ just the failing traceback. | ||
734 | try: | ||
735 | exec(compile(data, self._script_fullpath, 'exec'), context) | ||
736 | except Exception: | ||
737 | raise HookError('%s\nFailed to import %s hook; see traceback above.' % | ||
738 | (traceback.format_exc(), self._hook_type)) | ||
739 | |||
740 | # Running the script should have defined a main() function. | ||
741 | if 'main' not in context: | ||
742 | raise HookError('Missing main() in: "%s"' % self._script_fullpath) | ||
743 | |||
744 | # Call the main function in the hook. If the hook should cause the | ||
745 | # build to fail, it will raise an Exception. We'll catch that convert | ||
746 | # to a HookError w/ just the failing traceback. | ||
747 | try: | ||
748 | context['main'](**kwargs) | ||
749 | except Exception: | ||
750 | raise HookError('%s\nFailed to run main() for %s hook; see traceback ' | ||
751 | 'above.' % (traceback.format_exc(), self._hook_type)) | ||
752 | |||
753 | def _ExecuteHook(self, **kwargs): | ||
754 | """Actually execute the given hook. | ||
755 | |||
756 | This will run the hook's 'main' function in our python interpreter. | ||
757 | |||
758 | Args: | ||
759 | kwargs: Keyword arguments to pass to the hook. These are often specific | ||
760 | to the hook type. For instance, pre-upload hooks will contain | ||
761 | a project_list. | ||
762 | """ | ||
763 | # Keep sys.path and CWD stashed away so that we can always restore them | ||
764 | # upon function exit. | ||
765 | orig_path = os.getcwd() | ||
766 | orig_syspath = sys.path | ||
767 | |||
768 | try: | ||
769 | # Always run hooks with CWD as topdir. | ||
770 | os.chdir(self._topdir) | ||
771 | |||
772 | # Put the hook dir as the first item of sys.path so hooks can do | ||
773 | # relative imports. We want to replace the repo dir as [0] so | ||
774 | # hooks can't import repo files. | ||
775 | sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:] | ||
776 | |||
777 | # Initial global context for the hook to run within. | ||
778 | context = {'__file__': self._script_fullpath} | ||
779 | |||
780 | # Add 'hook_should_take_kwargs' to the arguments to be passed to main. | ||
781 | # We don't actually want hooks to define their main with this argument-- | ||
782 | # it's there to remind them that their hook should always take **kwargs. | ||
783 | # For instance, a pre-upload hook should be defined like: | ||
784 | # def main(project_list, **kwargs): | ||
785 | # | ||
786 | # This allows us to later expand the API without breaking old hooks. | ||
787 | kwargs = kwargs.copy() | ||
788 | kwargs['hook_should_take_kwargs'] = True | ||
789 | |||
790 | # See what version of python the hook has been written against. | ||
791 | data = open(self._script_fullpath).read() | ||
792 | interp = self._ExtractInterpFromShebang(data) | ||
793 | reexec = False | ||
794 | if interp: | ||
795 | prog = os.path.basename(interp) | ||
796 | if prog.startswith('python2') and sys.version_info.major != 2: | ||
797 | reexec = True | ||
798 | elif prog.startswith('python3') and sys.version_info.major == 2: | ||
799 | reexec = True | ||
800 | |||
801 | # Attempt to execute the hooks through the requested version of Python. | ||
802 | if reexec: | ||
803 | try: | ||
804 | self._ExecuteHookViaReexec(interp, context, **kwargs) | ||
805 | except OSError as e: | ||
806 | if e.errno == errno.ENOENT: | ||
807 | # We couldn't find the interpreter, so fallback to importing. | ||
808 | reexec = False | ||
809 | else: | ||
810 | raise | ||
811 | |||
812 | # Run the hook by importing directly. | ||
813 | if not reexec: | ||
814 | self._ExecuteHookViaImport(data, context, **kwargs) | ||
815 | finally: | ||
816 | # Restore sys.path and CWD. | ||
817 | sys.path = orig_syspath | ||
818 | os.chdir(orig_path) | ||
819 | |||
820 | def Run(self, user_allows_all_hooks, **kwargs): | ||
821 | """Run the hook. | ||
822 | |||
823 | If the hook doesn't exist (because there is no hooks project or because | ||
824 | this particular hook is not enabled), this is a no-op. | ||
825 | |||
826 | Args: | ||
827 | user_allows_all_hooks: If True, we will never prompt about running the | ||
828 | hook--we'll just assume it's OK to run it. | ||
829 | kwargs: Keyword arguments to pass to the hook. These are often specific | ||
830 | to the hook type. For instance, pre-upload hooks will contain | ||
831 | a project_list. | ||
832 | |||
833 | Raises: | ||
834 | HookError: If there was a problem finding the hook or the user declined | ||
835 | to run a required hook (from _CheckForHookApproval). | ||
836 | """ | ||
837 | # No-op if there is no hooks project or if hook is disabled. | ||
838 | if ((not self._hooks_project) or (self._hook_type not in | ||
839 | self._hooks_project.enabled_repo_hooks)): | ||
840 | return | ||
841 | |||
842 | # Bail with a nice error if we can't find the hook. | ||
843 | if not os.path.isfile(self._script_fullpath): | ||
844 | raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath) | ||
845 | |||
846 | # Make sure the user is OK with running the hook. | ||
847 | if (not user_allows_all_hooks) and (not self._CheckForHookApproval()): | ||
848 | return | ||
849 | |||
850 | # Run the hook with the same version of python we're using. | ||
851 | self._ExecuteHook(**kwargs) | ||
852 | |||
853 | |||
854 | class Project(object): | 452 | class Project(object): |
855 | # These objects can be shared between several working trees. | 453 | # These objects can be shared between several working trees. |
856 | shareable_files = ['description', 'info'] | 454 | shareable_files = ['description', 'info'] |