summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.github/workflows/test-ci.yml2
-rw-r--r--command.py34
-rw-r--r--completion.bash67
-rw-r--r--docs/internal-fs-layout.md8
-rw-r--r--docs/manifest-format.md58
-rw-r--r--docs/release-process.md172
-rw-r--r--error.py4
-rw-r--r--fetch.py41
-rw-r--r--git_command.py116
-rw-r--r--git_config.py291
-rw-r--r--git_superproject.py293
-rw-r--r--git_trace2_event_log.py45
-rwxr-xr-xmain.py126
-rw-r--r--man/repo-abandon.136
-rw-r--r--man/repo-branch.11
-rw-r--r--man/repo-branches.159
-rw-r--r--man/repo-checkout.136
-rw-r--r--man/repo-cherry-pick.128
-rw-r--r--man/repo-diff.135
-rw-r--r--man/repo-diffmanifests.161
-rw-r--r--man/repo-download.144
-rw-r--r--man/repo-forall.1128
-rw-r--r--man/repo-gitc-delete.131
-rw-r--r--man/repo-gitc-init.1150
-rw-r--r--man/repo-grep.1119
-rw-r--r--man/repo-help.133
-rw-r--r--man/repo-info.140
-rw-r--r--man/repo-init.1170
-rw-r--r--man/repo-list.161
-rw-r--r--man/repo-manifest.1548
-rw-r--r--man/repo-overview.139
-rw-r--r--man/repo-prune.128
-rw-r--r--man/repo-rebase.155
-rw-r--r--man/repo-selfupdate.135
-rw-r--r--man/repo-smartsync.1118
-rw-r--r--man/repo-stage.130
-rw-r--r--man/repo-start.141
-rw-r--r--man/repo-status.198
-rw-r--r--man/repo-sync.1209
-rw-r--r--man/repo-upload.1175
-rw-r--r--man/repo-version.124
-rw-r--r--man/repo.1133
-rw-r--r--manifest_xml.py189
-rw-r--r--platform_utils.py33
-rw-r--r--project.py152
-rwxr-xr-xrelease/update-manpages102
-rwxr-xr-xrepo29
-rw-r--r--requirements.json4
-rwxr-xr-xrun_tests4
-rwxr-xr-xsetup.py2
-rw-r--r--ssh.py277
-rw-r--r--subcmds/abandon.py2
-rw-r--r--subcmds/branches.py2
-rw-r--r--subcmds/checkout.py2
-rw-r--r--subcmds/cherry_pick.py2
-rw-r--r--subcmds/diff.py4
-rw-r--r--subcmds/diffmanifests.py6
-rw-r--r--subcmds/download.py2
-rw-r--r--subcmds/forall.py18
-rw-r--r--subcmds/gitc_delete.py4
-rw-r--r--subcmds/gitc_init.py2
-rw-r--r--subcmds/grep.py2
-rw-r--r--subcmds/help.py19
-rw-r--r--subcmds/info.py17
-rw-r--r--subcmds/init.py99
-rw-r--r--subcmds/list.py28
-rw-r--r--subcmds/manifest.py27
-rw-r--r--subcmds/overview.py17
-rw-r--r--subcmds/prune.py2
-rw-r--r--subcmds/rebase.py20
-rw-r--r--subcmds/selfupdate.py2
-rw-r--r--subcmds/smartsync.py2
-rw-r--r--subcmds/stage.py2
-rw-r--r--subcmds/start.py2
-rw-r--r--subcmds/status.py2
-rw-r--r--subcmds/sync.py306
-rw-r--r--subcmds/upload.py112
-rw-r--r--subcmds/version.py4
-rw-r--r--tests/fixtures/test.gitconfig10
-rw-r--r--tests/test_git_command.py27
-rw-r--r--tests/test_git_config.py19
-rw-r--r--tests/test_git_superproject.py260
-rw-r--r--tests/test_git_trace2_event_log.py72
-rw-r--r--tests/test_manifest_xml.py346
-rw-r--r--tests/test_platform_utils.py50
-rw-r--r--tests/test_ssh.py74
-rw-r--r--tests/test_subcmds.py30
-rw-r--r--tox.ini3
88 files changed, 5271 insertions, 941 deletions
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
index ec6f3791..19881858 100644
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@@ -14,7 +14,7 @@ jobs:
14 fail-fast: false 14 fail-fast: false
15 matrix: 15 matrix:
16 os: [ubuntu-latest, macos-latest, windows-latest] 16 os: [ubuntu-latest, macos-latest, windows-latest]
17 python-version: [3.5, 3.6, 3.7, 3.8, 3.9] 17 python-version: [3.6, 3.7, 3.8, 3.9]
18 runs-on: ${{ matrix.os }} 18 runs-on: ${{ matrix.os }}
19 19
20 steps: 20 steps:
diff --git a/command.py b/command.py
index 9b1220dc..b972a0be 100644
--- a/command.py
+++ b/command.py
@@ -15,7 +15,6 @@
15import multiprocessing 15import multiprocessing
16import os 16import os
17import optparse 17import optparse
18import platform
19import re 18import re
20import sys 19import sys
21 20
@@ -25,6 +24,10 @@ from error import InvalidProjectGroupsError
25import progress 24import progress
26 25
27 26
27# Are we generating man-pages?
28GENERATE_MANPAGES = os.environ.get('_REPO_GENERATE_MANPAGES_') == ' indeed! '
29
30
28# Number of projects to submit to a single worker process at a time. 31# Number of projects to submit to a single worker process at a time.
29# This number represents a tradeoff between the overhead of IPC and finer 32# This number represents a tradeoff between the overhead of IPC and finer
30# grained opportunity for parallelism. This particular value was chosen by 33# grained opportunity for parallelism. This particular value was chosen by
@@ -43,15 +46,32 @@ class Command(object):
43 """Base class for any command line action in repo. 46 """Base class for any command line action in repo.
44 """ 47 """
45 48
46 common = False 49 # Singleton for all commands to track overall repo command execution and
50 # provide event summary to callers. Only used by sync subcommand currently.
51 #
52 # NB: This is being replaced by git trace2 events. See git_trace2_event_log.
47 event_log = EventLog() 53 event_log = EventLog()
48 manifest = None 54
49 _optparse = None 55 # Whether this command is a "common" one, i.e. whether the user would commonly
56 # use it or it's a more uncommon command. This is used by the help command to
57 # show short-vs-full summaries.
58 COMMON = False
50 59
51 # Whether this command supports running in parallel. If greater than 0, 60 # Whether this command supports running in parallel. If greater than 0,
52 # it is the number of parallel jobs to default to. 61 # it is the number of parallel jobs to default to.
53 PARALLEL_JOBS = None 62 PARALLEL_JOBS = None
54 63
64 def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None,
65 git_event_log=None):
66 self.repodir = repodir
67 self.client = client
68 self.manifest = manifest
69 self.gitc_manifest = gitc_manifest
70 self.git_event_log = git_event_log
71
72 # Cache for the OptionParser property.
73 self._optparse = None
74
55 def WantPager(self, _opt): 75 def WantPager(self, _opt):
56 return False 76 return False
57 77
@@ -106,10 +126,14 @@ class Command(object):
106 help='only show errors') 126 help='only show errors')
107 127
108 if self.PARALLEL_JOBS is not None: 128 if self.PARALLEL_JOBS is not None:
129 default = 'based on number of CPU cores'
130 if not GENERATE_MANPAGES:
131 # Only include active cpu count if we aren't generating man pages.
132 default = f'%default; {default}'
109 p.add_option( 133 p.add_option(
110 '-j', '--jobs', 134 '-j', '--jobs',
111 type=int, default=self.PARALLEL_JOBS, 135 type=int, default=self.PARALLEL_JOBS,
112 help='number of jobs to run in parallel (default: %s)' % self.PARALLEL_JOBS) 136 help=f'number of jobs to run in parallel (default: {default})')
113 137
114 def _Options(self, p): 138 def _Options(self, p):
115 """Initialize the option parser with subcommand-specific options.""" 139 """Initialize the option parser with subcommand-specific options."""
diff --git a/completion.bash b/completion.bash
index 0b52d29c..09291d5c 100644
--- a/completion.bash
+++ b/completion.bash
@@ -14,6 +14,9 @@
14 14
15# Programmable bash completion. https://github.com/scop/bash-completion 15# Programmable bash completion. https://github.com/scop/bash-completion
16 16
17# TODO: Handle interspersed options. We handle `repo h<tab>`, but not
18# `repo --time h<tab>`.
19
17# Complete the list of repo subcommands. 20# Complete the list of repo subcommands.
18__complete_repo_list_commands() { 21__complete_repo_list_commands() {
19 local repo=${COMP_WORDS[0]} 22 local repo=${COMP_WORDS[0]}
@@ -37,6 +40,7 @@ __complete_repo_list_branches() {
37__complete_repo_list_projects() { 40__complete_repo_list_projects() {
38 local repo=${COMP_WORDS[0]} 41 local repo=${COMP_WORDS[0]}
39 "${repo}" list -n 2>/dev/null 42 "${repo}" list -n 2>/dev/null
43 "${repo}" list -p --relative-to=. 2>/dev/null
40} 44}
41 45
42# Complete the repo <command> argument. 46# Complete the repo <command> argument.
@@ -66,6 +70,48 @@ __complete_repo_command_projects() {
66 COMPREPLY=($(compgen -W "$(__complete_repo_list_projects)" -- "${current}")) 70 COMPREPLY=($(compgen -W "$(__complete_repo_list_projects)" -- "${current}"))
67} 71}
68 72
73# Complete `repo help`.
74__complete_repo_command_help() {
75 local current=$1
76 # CWORD=1 is "start".
77 # CWORD=2 is the <subcommand> which we complete here.
78 if [[ ${COMP_CWORD} -eq 2 ]]; then
79 COMPREPLY=(
80 $(compgen -W "$(__complete_repo_list_commands)" -- "${current}")
81 )
82 fi
83}
84
85# Complete `repo forall`.
86__complete_repo_command_forall() {
87 local current=$1
88 # CWORD=1 is "forall".
89 # CWORD=2+ are <projects> *until* we hit the -c option.
90 local i
91 for (( i = 0; i < COMP_CWORD; ++i )); do
92 if [[ "${COMP_WORDS[i]}" == "-c" ]]; then
93 return 0
94 fi
95 done
96
97 COMPREPLY=(
98 $(compgen -W "$(__complete_repo_list_projects)" -- "${current}")
99 )
100}
101
102# Complete `repo start`.
103__complete_repo_command_start() {
104 local current=$1
105 # CWORD=1 is "start".
106 # CWORD=2 is the <branch> which we don't complete.
107 # CWORD=3+ are <projects> which we complete here.
108 if [[ ${COMP_CWORD} -gt 2 ]]; then
109 COMPREPLY=(
110 $(compgen -W "$(__complete_repo_list_projects)" -- "${current}")
111 )
112 fi
113}
114
69# Complete the repo subcommand arguments. 115# Complete the repo subcommand arguments.
70__complete_repo_arg() { 116__complete_repo_arg() {
71 if [[ ${COMP_CWORD} -le 1 ]]; then 117 if [[ ${COMP_CWORD} -le 1 ]]; then
@@ -86,21 +132,8 @@ __complete_repo_arg() {
86 return 0 132 return 0
87 ;; 133 ;;
88 134
89 help) 135 help|start|forall)
90 if [[ ${COMP_CWORD} -eq 2 ]]; then 136 __complete_repo_command_${command} "${current}"
91 COMPREPLY=(
92 $(compgen -W "$(__complete_repo_list_commands)" -- "${current}")
93 )
94 fi
95 return 0
96 ;;
97
98 start)
99 if [[ ${COMP_CWORD} -gt 2 ]]; then
100 COMPREPLY=(
101 $(compgen -W "$(__complete_repo_list_projects)" -- "${current}")
102 )
103 fi
104 return 0 137 return 0
105 ;; 138 ;;
106 139
@@ -118,4 +151,6 @@ __complete_repo() {
118 return 0 151 return 0
119} 152}
120 153
121complete -F __complete_repo repo 154# Fallback to the default complete methods if we aren't able to provide anything
155# useful. This will allow e.g. local paths to be used when it makes sense.
156complete -F __complete_repo -o bashdefault -o default repo
diff --git a/docs/internal-fs-layout.md b/docs/internal-fs-layout.md
index 0c59f988..af6a4523 100644
--- a/docs/internal-fs-layout.md
+++ b/docs/internal-fs-layout.md
@@ -110,6 +110,8 @@ Instead, you should use standard Git workflows like [git worktree] or
110[gitsubmodules] with [superprojects]. 110[gitsubmodules] with [superprojects].
111*** 111***
112 112
113* `copy-link-files.json`: Tracking file used by `repo sync` to determine when
114 copyfile or linkfile are added or removed and need corresponding updates.
113* `project.list`: Tracking file used by `repo sync` to determine when projects 115* `project.list`: Tracking file used by `repo sync` to determine when projects
114 are added or removed and need corresponding updates in the checkout. 116 are added or removed and need corresponding updates in the checkout.
115* `projects/`: Bare checkouts of every project synced by the manifest. The 117* `projects/`: Bare checkouts of every project synced by the manifest. The
@@ -144,12 +146,18 @@ Instead, you should use standard Git workflows like [git worktree] or
144 146
145The `.repo/manifests.git/config` file is used to track settings for the entire 147The `.repo/manifests.git/config` file is used to track settings for the entire
146repo client checkout. 148repo client checkout.
149
147Most settings use the `[repo]` section to avoid conflicts with git. 150Most settings use the `[repo]` section to avoid conflicts with git.
151
152Everything under `[repo.syncstate.*]` is used to keep track of sync details for logging
153purposes.
154
148User controlled settings are initialized when running `repo init`. 155User controlled settings are initialized when running `repo init`.
149 156
150| Setting | `repo init` Option | Use/Meaning | 157| Setting | `repo init` Option | Use/Meaning |
151|------------------- |---------------------------|-------------| 158|------------------- |---------------------------|-------------|
152| manifest.groups | `--groups` & `--platform` | The manifest groups to sync | 159| manifest.groups | `--groups` & `--platform` | The manifest groups to sync |
160| manifest.standalone | `--standalone-manifest` | Download manifest as static file instead of creating checkout |
153| repo.archive | `--archive` | Use `git archive` for checkouts | 161| repo.archive | `--archive` | Use `git archive` for checkouts |
154| repo.clonebundle | `--clone-bundle` | Whether the initial sync used clone.bundle explicitly | 162| repo.clonebundle | `--clone-bundle` | Whether the initial sync used clone.bundle explicitly |
155| repo.clonefilter | `--clone-filter` | Filter setting when using [partial git clones] | 163| repo.clonefilter | `--clone-filter` | Filter setting when using [partial git clones] |
diff --git a/docs/manifest-format.md b/docs/manifest-format.md
index da83d0dd..8e0049b3 100644
--- a/docs/manifest-format.md
+++ b/docs/manifest-format.md
@@ -31,11 +31,12 @@ following DTD:
31 extend-project*, 31 extend-project*,
32 repo-hooks?, 32 repo-hooks?,
33 superproject?, 33 superproject?,
34 contactinfo?,
34 include*)> 35 include*)>
35 36
36 <!ELEMENT notice (#PCDATA)> 37 <!ELEMENT notice (#PCDATA)>
37 38
38 <!ELEMENT remote EMPTY> 39 <!ELEMENT remote (annotation*)>
39 <!ATTLIST remote name ID #REQUIRED> 40 <!ATTLIST remote name ID #REQUIRED>
40 <!ATTLIST remote alias CDATA #IMPLIED> 41 <!ATTLIST remote alias CDATA #IMPLIED>
41 <!ATTLIST remote fetch CDATA #REQUIRED> 42 <!ATTLIST remote fetch CDATA #REQUIRED>
@@ -89,20 +90,26 @@ following DTD:
89 <!ELEMENT extend-project EMPTY> 90 <!ELEMENT extend-project EMPTY>
90 <!ATTLIST extend-project name CDATA #REQUIRED> 91 <!ATTLIST extend-project name CDATA #REQUIRED>
91 <!ATTLIST extend-project path CDATA #IMPLIED> 92 <!ATTLIST extend-project path CDATA #IMPLIED>
93 <!ATTLIST extend-project dest-path CDATA #IMPLIED>
92 <!ATTLIST extend-project groups CDATA #IMPLIED> 94 <!ATTLIST extend-project groups CDATA #IMPLIED>
93 <!ATTLIST extend-project revision CDATA #IMPLIED> 95 <!ATTLIST extend-project revision CDATA #IMPLIED>
94 <!ATTLIST extend-project remote CDATA #IMPLIED> 96 <!ATTLIST extend-project remote CDATA #IMPLIED>
95 97
96 <!ELEMENT remove-project EMPTY> 98 <!ELEMENT remove-project EMPTY>
97 <!ATTLIST remove-project name CDATA #REQUIRED> 99 <!ATTLIST remove-project name CDATA #REQUIRED>
100 <!ATTLIST remove-project optional CDATA #IMPLIED>
98 101
99 <!ELEMENT repo-hooks EMPTY> 102 <!ELEMENT repo-hooks EMPTY>
100 <!ATTLIST repo-hooks in-project CDATA #REQUIRED> 103 <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
101 <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED> 104 <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED>
102 105
103 <!ELEMENT superproject (EMPTY)> 106 <!ELEMENT superproject EMPTY>
104 <!ATTLIST superproject name CDATA #REQUIRED> 107 <!ATTLIST superproject name CDATA #REQUIRED>
105 <!ATTLIST superproject remote IDREF #IMPLIED> 108 <!ATTLIST superproject remote IDREF #IMPLIED>
109 <!ATTLIST superproject revision CDATA #IMPLIED>
110
111 <!ELEMENT contactinfo EMPTY>
112 <!ATTLIST contactinfo bugurl CDATA #REQUIRED>
106 113
107 <!ELEMENT include EMPTY> 114 <!ELEMENT include EMPTY>
108 <!ATTLIST include name CDATA #REQUIRED> 115 <!ATTLIST include name CDATA #REQUIRED>
@@ -331,6 +338,11 @@ against changes to the original manifest.
331Attribute `path`: If specified, limit the change to projects checked out 338Attribute `path`: If specified, limit the change to projects checked out
332at the specified path, rather than all projects with the given name. 339at the specified path, rather than all projects with the given name.
333 340
341Attribute `dest-path`: If specified, a path relative to the top directory
342of the repo client where the Git working directory for this project
343should be placed. This is used to move a project in the checkout by
344overriding the existing `path` setting.
345
334Attribute `groups`: List of additional groups to which this project 346Attribute `groups`: List of additional groups to which this project
335belongs. Same syntax as the corresponding element of `project`. 347belongs. Same syntax as the corresponding element of `project`.
336 348
@@ -343,12 +355,12 @@ project. Same syntax as the corresponding element of `project`.
343### Element annotation 355### Element annotation
344 356
345Zero or more annotation elements may be specified as children of a 357Zero or more annotation elements may be specified as children of a
346project element. Each element describes a name-value pair that will be 358project or remote element. Each element describes a name-value pair.
347exported into each project's environment during a 'forall' command, 359For projects, this name-value pair will be exported into each project's
348prefixed with REPO__. In addition, there is an optional attribute 360environment during a 'forall' command, prefixed with `REPO__`. In addition,
349"keep" which accepts the case insensitive values "true" (default) or 361there is an optional attribute "keep" which accepts the case insensitive values
350"false". This attribute determines whether or not the annotation will 362"true" (default) or "false". This attribute determines whether or not the
351be kept when exported with the manifest subcommand. 363annotation will be kept when exported with the manifest subcommand.
352 364
353### Element copyfile 365### Element copyfile
354 366
@@ -389,6 +401,9 @@ This element is mostly useful in a local manifest file, where
389the user can remove a project, and possibly replace it with their 401the user can remove a project, and possibly replace it with their
390own definition. 402own definition.
391 403
404Attribute `optional`: Set to true to ignore remove-project elements with no
405matching `project` element.
406
392### Element repo-hooks 407### Element repo-hooks
393 408
394NB: See the [practical documentation](./repo-hooks.md) for using repo hooks. 409NB: See the [practical documentation](./repo-hooks.md) for using repo hooks.
@@ -405,7 +420,7 @@ Attribute `enabled-list`: List of hooks to use, whitespace or comma separated.
405### Element superproject 420### Element superproject
406 421
407*** 422***
408 *Note*: This is currently a WIP. 423*Note*: This is currently a WIP.
409*** 424***
410 425
411NB: See the [git superprojects documentation]( 426NB: See the [git superprojects documentation](
@@ -424,6 +439,24 @@ same meaning as project's name attribute. See the
424Attribute `remote`: Name of a previously defined remote element. 439Attribute `remote`: Name of a previously defined remote element.
425If not supplied the remote given by the default element is used. 440If not supplied the remote given by the default element is used.
426 441
442Attribute `revision`: Name of the Git branch the manifest wants
443to track for this superproject. If not supplied the revision given
444by the remote element is used if applicable, else the default
445element is used.
446
447### Element contactinfo
448
449***
450*Note*: This is currently a WIP.
451***
452
453This element is used to let manifest authors self-register contact info.
454It has "bugurl" as a required atrribute. This element can be repeated,
455and any later entries will clobber earlier ones. This would allow manifest
456authors who extend manifests to specify their own contact info.
457
458Attribute `bugurl`: The URL to file a bug against the manifest owner.
459
427### Element include 460### Element include
428 461
429This element provides the capability of including another manifest 462This element provides the capability of including another manifest
@@ -468,6 +501,9 @@ these extra projects.
468Manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml` will 501Manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml` will
469be loaded in alphabetical order. 502be loaded in alphabetical order.
470 503
504Projects from local manifest files are added into
505local::<local manifest filename> group.
506
471The legacy `$TOP_DIR/.repo/local_manifest.xml` path is no longer supported. 507The legacy `$TOP_DIR/.repo/local_manifest.xml` path is no longer supported.
472 508
473 509
diff --git a/docs/release-process.md b/docs/release-process.md
index 43209eb0..f71a4110 100644
--- a/docs/release-process.md
+++ b/docs/release-process.md
@@ -83,7 +83,8 @@ control how repo finds updates:
83* `--repo-rev`: This tells repo which branch to use for the full project. 83* `--repo-rev`: This tells repo which branch to use for the full project.
84 It defaults to the `stable` branch (`REPO_REV` in the launcher script). 84 It defaults to the `stable` branch (`REPO_REV` in the launcher script).
85 85
86Whenever `repo sync` is run, repo will check to see if an update is available. 86Whenever `repo sync` is run, repo will, once every 24 hours, see if an update
87is available.
87It fetches the latest repo-rev from the repo-url. 88It fetches the latest repo-rev from the repo-url.
88Then it verifies that the latest commit in the branch has a valid signed tag 89Then it verifies that the latest commit in the branch has a valid signed tag
89using `git tag -v` (which uses gpg). 90using `git tag -v` (which uses gpg).
@@ -95,6 +96,11 @@ If that tag is valid, then repo will warn and use that commit instead.
95 96
96If that tag cannot be verified, it gives up and forces the user to resolve. 97If that tag cannot be verified, it gives up and forces the user to resolve.
97 98
99### Force an update
100
101The `repo selfupdate` command can be used to force an immediate update.
102It is not subject to the 24 hour limitation.
103
98## Branch management 104## Branch management
99 105
100All development happens on the `main` branch and should generally be stable. 106All development happens on the `main` branch and should generally be stable.
@@ -202,80 +208,132 @@ Things in bold indicate stuff to take note of, but does not guarantee that we
202still support them. 208still support them.
203Things in italics are things we used to care about but probably don't anymore. 209Things in italics are things we used to care about but probably don't anymore.
204 210
205| Date | EOL | [Git][rel-g] | [Python][rel-p] | [Ubuntu][rel-u] / [Debian][rel-d] | Git | Python | 211| Date | EOL | [Git][rel-g] | [Python][rel-p] | [SSH][rel-o] | [Ubuntu][rel-u] / [Debian][rel-d] | Git | Python | SSH |
206|:--------:|:------------:|--------------|-----------------|-----------------------------------|-----|--------| 212|:--------:|:------------:|:------------:|:---------------:|:------------:|-----------------------------------|-----|--------|-----|
207| Oct 2008 | *Oct 2013* | | 2.6.0 | *10.04 Lucid* - 10.10 Maverick / *Squeeze* | 213| Apr 2008 | | | | 5.0 |
214| Jun 2008 | | | | 5.1 |
215| Oct 2008 | *Oct 2013* | | 2.6.0 | | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
208| Dec 2008 | *Feb 2009* | | 3.0.0 | 216| Dec 2008 | *Feb 2009* | | 3.0.0 |
209| Feb 2009 | *Mar 2012* | | | Debian 5 Lenny | 1.5.6.5 | 2.5.2 | 217| Feb 2009 | | | | 5.2 |
210| Jun 2009 | *Jun 2016* | | 3.1.0 | *10.04 Lucid* - 10.10 Maverick / *Squeeze* | 218| Feb 2009 | *Mar 2012* | | | | Debian 5 Lenny | 1.5.6.5 | 2.5.2 |
211| Feb 2010 | *Oct 2012* | 1.7.0 | | *10.04 Lucid* - *12.04 Precise* - 12.10 Quantal | 219| Jun 2009 | *Jun 2016* | | 3.1.0 | | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
212| Apr 2010 | *Apr 2015* | | | *10.04 Lucid* | 1.7.0.4 | 2.6.5 3.1.2 | 220| Sep 2009 | | | | 5.3 | *10.04 Lucid* |
213| Jul 2010 | *Dec 2019* | | **2.7.0** | 11.04 Natty - **<current>** | 221| Feb 2010 | *Oct 2012* | 1.7.0 | | | *10.04 Lucid* - *12.04 Precise* - 12.10 Quantal |
214| Oct 2010 | | | | 10.10 Maverick | 1.7.1 | 2.6.6 3.1.3 | 222| Mar 2010 | | | | 5.4 |
215| Feb 2011 | *Feb 2016* | | | Debian 6 Squeeze | 1.7.2.5 | 2.6.6 3.1.3 | 223| Apr 2010 | | | | 5.5 | 10.10 Maverick |
216| Apr 2011 | | | | 11.04 Natty | 1.7.4 | 2.7.1 3.2.0 | 224| Apr 2010 | *Apr 2015* | | | | *10.04 Lucid* | 1.7.0.4 | 2.6.5 3.1.2 | 5.3 |
217| Oct 2011 | *Feb 2016* | | 3.2.0 | 11.04 Natty - 12.10 Quantal | 225| Jul 2010 | *Dec 2019* | | *2.7.0* | | 11.04 Natty - *<current>* |
218| Oct 2011 | | | | 11.10 Ocelot | 1.7.5.4 | 2.7.2 3.2.2 | 226| Aug 2010 | | | | 5.6 |
219| Apr 2012 | *Apr 2019* | | | *12.04 Precise* | 1.7.9.5 | 2.7.3 3.2.3 | 227| Oct 2010 | | | | | 10.10 Maverick | 1.7.1 | 2.6.6 3.1.3 | 5.5 |
220| Sep 2012 | *Sep 2017* | | 3.3.0 | 13.04 Raring - 13.10 Saucy | 228| Jan 2011 | | | | 5.7 |
221| Oct 2012 | *Dec 2014* | 1.8.0 | | 13.04 Raring - 13.10 Saucy | 229| Feb 2011 | | | | 5.8 | 11.04 Natty |
222| Oct 2012 | | | | 12.10 Quantal | 1.7.10.4 | 2.7.3 3.2.3 | 230| Feb 2011 | *Feb 2016* | | | | Debian 6 Squeeze | 1.7.2.5 | 2.6.6 3.1.3 |
223| Apr 2013 | | | | 13.04 Raring | 1.8.1.2 | 2.7.4 3.3.1 | 231| Apr 2011 | | | | | 11.04 Natty | 1.7.4 | 2.7.1 3.2.0 | 5.8 |
224| May 2013 | *May 2018* | | | Debian 7 Wheezy | 1.7.10.4 | 2.7.3 3.2.3 | 232| Sep 2011 | | | | 5.9 | *12.04 Precise* |
225| Oct 2013 | | | | 13.10 Saucy | 1.8.3.2 | 2.7.5 3.3.2 | 233| Oct 2011 | *Feb 2016* | | 3.2.0 | | 11.04 Natty - 12.10 Quantal |
226| Feb 2014 | *Dec 2014* | **1.9.0** | | **14.04 Trusty** | 234| Oct 2011 | | | | | 11.10 Ocelot | 1.7.5.4 | 2.7.2 3.2.2 | 5.8 |
227| Mar 2014 | *Mar 2019* | | **3.4.0** | **14.04 Trusty** - 15.10 Wily / **Jessie** | 235| Apr 2012 | | | | 6.0 | 12.10 Quantal |
228| Apr 2014 | **Apr 2022** | | | **14.04 Trusty** | 1.9.1 | 2.7.5 3.4.0 | 236| Apr 2012 | *Apr 2019* | | | | *12.04 Precise* | 1.7.9.5 | 2.7.3 3.2.3 | 5.9 |
237| Aug 2012 | | | | 6.1 | 13.04 Raring |
238| Sep 2012 | *Sep 2017* | | 3.3.0 | | 13.04 Raring - 13.10 Saucy |
239| Oct 2012 | *Dec 2014* | 1.8.0 | | | 13.04 Raring - 13.10 Saucy |
240| Oct 2012 | | | | | 12.10 Quantal | 1.7.10.4 | 2.7.3 3.2.3 | 6.0 |
241| Mar 2013 | | | | 6.2 | 13.10 Saucy |
242| Apr 2013 | | | | | 13.04 Raring | 1.8.1.2 | 2.7.4 3.3.1 | 6.1 |
243| May 2013 | *May 2018* | | | | Debian 7 Wheezy | 1.7.10.4 | 2.7.3 3.2.3 |
244| Sep 2013 | | | | 6.3 |
245| Oct 2013 | | | | | 13.10 Saucy | 1.8.3.2 | 2.7.5 3.3.2 | 6.2 |
246| Nov 2013 | | | | 6.4 |
247| Jan 2014 | | | | 6.5 |
248| Feb 2014 | *Dec 2014* | **1.9.0** | | | *14.04 Trusty* |
249| Mar 2014 | *Mar 2019* | | *3.4.0* | | *14.04 Trusty* - 15.10 Wily / *Jessie* |
250| Mar 2014 | | | | 6.6 | *14.04 Trusty* - 14.10 Utopic |
251| Apr 2014 | *Apr 2022* | | | | *14.04 Trusty* | 1.9.1 | 2.7.5 3.4.0 | 6.6 |
229| May 2014 | *Dec 2014* | 2.0.0 | 252| May 2014 | *Dec 2014* | 2.0.0 |
230| Aug 2014 | *Dec 2014* | **2.1.0** | | 14.10 Utopic - 15.04 Vivid / **Jessie** | 253| Aug 2014 | *Dec 2014* | *2.1.0* | | | 14.10 Utopic - 15.04 Vivid / *Jessie* |
231| Oct 2014 | | | | 14.10 Utopic | 2.1.0 | 2.7.8 3.4.2 | 254| Oct 2014 | | | | 6.7 | 15.04 Vivid |
255| Oct 2014 | | | | | 14.10 Utopic | 2.1.0 | 2.7.8 3.4.2 | 6.6 |
232| Nov 2014 | *Sep 2015* | 2.2.0 | 256| Nov 2014 | *Sep 2015* | 2.2.0 |
233| Feb 2015 | *Sep 2015* | 2.3.0 | 257| Feb 2015 | *Sep 2015* | 2.3.0 |
258| Mar 2015 | | | | 6.8 |
234| Apr 2015 | *May 2017* | 2.4.0 | 259| Apr 2015 | *May 2017* | 2.4.0 |
235| Apr 2015 | **Jun 2020** | | | **Debian 8 Jessie** | 2.1.4 | 2.7.9 3.4.2 | 260| Apr 2015 | *Jun 2020* | | | | *Debian 8 Jessie* | 2.1.4 | 2.7.9 3.4.2 |
236| Apr 2015 | | | | 15.04 Vivid | 2.1.4 | 2.7.9 3.4.3 | 261| Apr 2015 | | | | | 15.04 Vivid | 2.1.4 | 2.7.9 3.4.3 | 6.7 |
237| Jul 2015 | *May 2017* | 2.5.0 | | 15.10 Wily | 262| Jul 2015 | *May 2017* | 2.5.0 | | | 15.10 Wily |
263| Jul 2015 | | | | 6.9 | 15.10 Wily |
264| Aug 2015 | | | | 7.0 |
265| Aug 2015 | | | | 7.1 |
238| Sep 2015 | *May 2017* | 2.6.0 | 266| Sep 2015 | *May 2017* | 2.6.0 |
239| Sep 2015 | **Sep 2020** | | **3.5.0** | **16.04 Xenial** - 17.04 Zesty / **Stretch** | 267| Sep 2015 | *Sep 2020* | | *3.5.0* | | *16.04 Xenial* - 17.04 Zesty / *Stretch* |
240| Oct 2015 | | | | 15.10 Wily | 2.5.0 | 2.7.9 3.4.3 | 268| Oct 2015 | | | | | 15.10 Wily | 2.5.0 | 2.7.9 3.4.3 | 6.9 |
241| Jan 2016 | *Jul 2017* | **2.7.0** | | **16.04 Xenial** | 269| Jan 2016 | *Jul 2017* | *2.7.0* | | | *16.04 Xenial* |
270| Feb 2016 | | | | 7.2 | *16.04 Xenial* |
242| Mar 2016 | *Jul 2017* | 2.8.0 | 271| Mar 2016 | *Jul 2017* | 2.8.0 |
243| Apr 2016 | **Apr 2024** | | | **16.04 Xenial** | 2.7.4 | 2.7.11 3.5.1 | 272| Apr 2016 | *Apr 2024* | | | | *16.04 Xenial* | 2.7.4 | 2.7.11 3.5.1 | 7.2 |
244| Jun 2016 | *Jul 2017* | 2.9.0 | | 16.10 Yakkety | 273| Jun 2016 | *Jul 2017* | 2.9.0 | | | 16.10 Yakkety |
274| Jul 2016 | | | | 7.3 | 16.10 Yakkety |
245| Sep 2016 | *Sep 2017* | 2.10.0 | 275| Sep 2016 | *Sep 2017* | 2.10.0 |
246| Oct 2016 | | | | 16.10 Yakkety | 2.9.3 | 2.7.11 3.5.1 | 276| Oct 2016 | | | | | 16.10 Yakkety | 2.9.3 | 2.7.11 3.5.1 | 7.3 |
247| Nov 2016 | *Sep 2017* | **2.11.0** | | 17.04 Zesty / **Stretch** | 277| Nov 2016 | *Sep 2017* | *2.11.0* | | | 17.04 Zesty / *Stretch* |
248| Dec 2016 | **Dec 2021** | | **3.6.0** | 17.10 Artful - **18.04 Bionic** - 18.10 Cosmic | 278| Dec 2016 | **Dec 2021** | | **3.6.0** | | 17.10 Artful - **18.04 Bionic** - 18.10 Cosmic |
279| Dec 2016 | | | | 7.4 | 17.04 Zesty / *Debian 9 Stretch* |
249| Feb 2017 | *Sep 2017* | 2.12.0 | 280| Feb 2017 | *Sep 2017* | 2.12.0 |
250| Apr 2017 | | | | 17.04 Zesty | 2.11.0 | 2.7.13 3.5.3 | 281| Mar 2017 | | | | 7.5 | 17.10 Artful |
282| Apr 2017 | | | | | 17.04 Zesty | 2.11.0 | 2.7.13 3.5.3 | 7.4 |
251| May 2017 | *May 2018* | 2.13.0 | 283| May 2017 | *May 2018* | 2.13.0 |
252| Jun 2017 | **Jun 2022** | | | **Debian 9 Stretch** | 2.11.0 | 2.7.13 3.5.3 | 284| Jun 2017 | *Jun 2022* | | | | *Debian 9 Stretch* | 2.11.0 | 2.7.13 3.5.3 | 7.4 |
253| Aug 2017 | *Dec 2019* | 2.14.0 | | 17.10 Artful | 285| Aug 2017 | *Dec 2019* | 2.14.0 | | | 17.10 Artful |
254| Oct 2017 | *Dec 2019* | 2.15.0 | 286| Oct 2017 | *Dec 2019* | 2.15.0 |
255| Oct 2017 | | | | 17.10 Artful | 2.14.1 | 2.7.14 3.6.3 | 287| Oct 2017 | | | | 7.6 | **18.04 Bionic** |
288| Oct 2017 | | | | | 17.10 Artful | 2.14.1 | 2.7.14 3.6.3 | 7.5 |
256| Jan 2018 | *Dec 2019* | 2.16.0 | 289| Jan 2018 | *Dec 2019* | 2.16.0 |
257| Apr 2018 | *Dec 2019* | 2.17.0 | | **18.04 Bionic** | 290| Apr 2018 | *Mar 2021* | **2.17.0** | | | **18.04 Bionic** |
258| Apr 2018 | **Apr 2028** | | | **18.04 Bionic** | 2.17.0 | 2.7.15 3.6.5 | 291| Apr 2018 | | | | 7.7 | 18.10 Cosmic |
259| Jun 2018 | *Dec 2019* | 2.18.0 | 292| Apr 2018 | **Apr 2028** | | | | **18.04 Bionic** | 2.17.0 | 2.7.15 3.6.5 | 7.6 |
260| Jun 2018 | **Jun 2023** | | 3.7.0 | 19.04 Disco - **20.04 Focal** / **Buster** | 293| Jun 2018 | *Mar 2021* | 2.18.0 |
261| Sep 2018 | *Dec 2019* | 2.19.0 | | 18.10 Cosmic | 294| Jun 2018 | **Jun 2023** | | 3.7.0 | | 19.04 Disco - **20.04 Focal** / **Buster** |
262| Oct 2018 | | | | 18.10 Cosmic | 2.19.1 | 2.7.15 3.6.6 | 295| Aug 2018 | | | | 7.8 |
263| Dec 2018 | *Dec 2019* | **2.20.0** | | 19.04 Disco / **Buster** | 296| Sep 2018 | *Mar 2021* | 2.19.0 | | | 18.10 Cosmic |
264| Feb 2019 | *Dec 2019* | 2.21.0 | 297| Oct 2018 | | | | 7.9 | 19.04 Disco / **Buster** |
265| Apr 2019 | | | | 19.04 Disco | 2.20.1 | 2.7.16 3.7.3 | 298| Oct 2018 | | | | | 18.10 Cosmic | 2.19.1 | 2.7.15 3.6.6 | 7.7 |
299| Dec 2018 | *Mar 2021* | **2.20.0** | | | 19.04 Disco - 19.10 Eoan / **Buster** |
300| Feb 2019 | *Mar 2021* | 2.21.0 |
301| Apr 2019 | | | | 8.0 | 19.10 Eoan |
302| Apr 2019 | | | | | 19.04 Disco | 2.20.1 | 2.7.16 3.7.3 | 7.9 |
266| Jun 2019 | | 2.22.0 | 303| Jun 2019 | | 2.22.0 |
267| Jul 2019 | **Jul 2024** | | | **Debian 10 Buster** | 2.20.1 | 2.7.16 3.7.3 | 304| Jul 2019 | **Jul 2024** | | | | **Debian 10 Buster** | 2.20.1 | 2.7.16 3.7.3 | 7.9 |
268| Aug 2019 | | 2.23.0 | 305| Aug 2019 | *Mar 2021* | 2.23.0 |
269| Oct 2019 | **Oct 2024** | | 3.8.0 | 306| Oct 2019 | **Oct 2024** | | 3.8.0 | | **20.04 Focal** - 20.10 Groovy |
270| Oct 2019 | | | | 19.10 Eoan | 2.20.1 | 2.7.17 3.7.5 | 307| Oct 2019 | | | | 8.1 |
271| Nov 2019 | | 2.24.0 | 308| Oct 2019 | | | | | 19.10 Eoan | 2.20.1 | 2.7.17 3.7.5 | 8.0 |
272| Jan 2020 | | 2.25.0 | | **20.04 Focal** | 309| Nov 2019 | *Mar 2021* | 2.24.0 |
273| Apr 2020 | **Apr 2030** | | | **20.04 Focal** | 2.25.0 | 2.7.17 3.7.5 | 310| Jan 2020 | *Mar 2021* | 2.25.0 | | | **20.04 Focal** |
311| Feb 2020 | | | | 8.2 | **20.04 Focal** |
312| Mar 2020 | *Mar 2021* | 2.26.0 |
313| Apr 2020 | **Apr 2030** | | | | **20.04 Focal** | 2.25.1 | 2.7.17 3.8.2 | 8.2 |
314| May 2020 | *Mar 2021* | 2.27.0 | | | 20.10 Groovy |
315| May 2020 | | | | 8.3 |
316| Jul 2020 | *Mar 2021* | 2.28.0 |
317| Sep 2020 | | | | 8.4 | 21.04 Hirsute / **Bullseye** |
318| Oct 2020 | *Mar 2021* | 2.29.0 |
319| Oct 2020 | | | | | 20.10 Groovy | 2.27.0 | 2.7.18 3.8.6 | 8.3 |
320| Oct 2020 | **Oct 2025** | | 3.9.0 | | 21.04 Hirsute / **Bullseye** |
321| Dec 2020 | *Mar 2021* | 2.30.0 | | | 21.04 Hirsute / **Bullseye** |
322| Mar 2021 | | 2.31.0 |
323| Mar 2021 | | | | 8.5 |
324| Apr 2021 | | | | 8.6 |
325| Apr 2021 | *Jan 2022* | | | | 21.04 Hirsute | 2.30.2 | 2.7.18 3.9.4 | 8.4 |
326| Jun 2021 | | 2.32.0 |
327| Aug 2021 | | 2.33.0 |
328| Aug 2021 | | | | 8.7 |
329| Aug 2021 | **Aug 2026** | | | | **Debian 11 Bullseye** | 2.30.2 | 2.7.18 3.9.2 | 8.4 |
330| **Date** | **EOL** | **[Git][rel-g]** | **[Python][rel-p]** | **[SSH][rel-o]** | **[Ubuntu][rel-u] / [Debian][rel-d]** | **Git** | **Python** | **SSH** |
274 331
275 332
276[contact]: ../README.md#contact 333[contact]: ../README.md#contact
277[rel-d]: https://en.wikipedia.org/wiki/Debian_version_history 334[rel-d]: https://en.wikipedia.org/wiki/Debian_version_history
278[rel-g]: https://en.wikipedia.org/wiki/Git#Releases 335[rel-g]: https://en.wikipedia.org/wiki/Git#Releases
336[rel-o]: https://www.openssh.com/releasenotes.html
279[rel-p]: https://en.wikipedia.org/wiki/History_of_Python#Table_of_versions 337[rel-p]: https://en.wikipedia.org/wiki/History_of_Python#Table_of_versions
280[rel-u]: https://en.wikipedia.org/wiki/Ubuntu_version_history#Table_of_versions 338[rel-u]: https://en.wikipedia.org/wiki/Ubuntu_version_history#Table_of_versions
281[example announcement]: https://groups.google.com/d/topic/repo-discuss/UGBNismWo1M/discussion 339[example announcement]: https://groups.google.com/d/topic/repo-discuss/UGBNismWo1M/discussion
diff --git a/error.py b/error.py
index 25ff80d1..cbefcb7e 100644
--- a/error.py
+++ b/error.py
@@ -13,10 +13,6 @@
13# limitations under the License. 13# limitations under the License.
14 14
15 15
16# URL to file bug reports for repo tool issues.
17BUG_REPORT_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue'
18
19
20class ManifestParseError(Exception): 16class ManifestParseError(Exception):
21 """Failed to parse the manifest file. 17 """Failed to parse the manifest file.
22 """ 18 """
diff --git a/fetch.py b/fetch.py
new file mode 100644
index 00000000..91d40cda
--- /dev/null
+++ b/fetch.py
@@ -0,0 +1,41 @@
1# Copyright (C) 2021 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""This module contains functions used to fetch files from various sources."""
16
17import subprocess
18import sys
19from urllib.parse import urlparse
20
21def fetch_file(url):
22 """Fetch a file from the specified source using the appropriate protocol.
23
24 Returns:
25 The contents of the file as bytes.
26 """
27 scheme = urlparse(url).scheme
28 if scheme == 'gs':
29 cmd = ['gsutil', 'cat', url]
30 try:
31 result = subprocess.run(
32 cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
33 return result.stdout
34 except subprocess.CalledProcessError as e:
35 print('fatal: error running "gsutil": %s' % e.output,
36 file=sys.stderr)
37 sys.exit(1)
38 if scheme == 'file':
39 with open(url[len('file://'):], 'rb') as f:
40 return f.read()
41 raise ValueError('unsupported url %s' % url)
diff --git a/git_command.py b/git_command.py
index d06fc77c..95db91f2 100644
--- a/git_command.py
+++ b/git_command.py
@@ -12,12 +12,10 @@
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14 14
15import functools
15import os 16import os
16import re
17import sys 17import sys
18import subprocess 18import subprocess
19import tempfile
20from signal import SIGTERM
21 19
22from error import GitError 20from error import GitError
23from git_refs import HEAD 21from git_refs import HEAD
@@ -42,101 +40,15 @@ GIT_DIR = 'GIT_DIR'
42LAST_GITDIR = None 40LAST_GITDIR = None
43LAST_CWD = None 41LAST_CWD = None
44 42
45_ssh_proxy_path = None
46_ssh_sock_path = None
47_ssh_clients = []
48_ssh_version = None
49
50
51def _run_ssh_version():
52 """run ssh -V to display the version number"""
53 return subprocess.check_output(['ssh', '-V'], stderr=subprocess.STDOUT).decode()
54
55
56def _parse_ssh_version(ver_str=None):
57 """parse a ssh version string into a tuple"""
58 if ver_str is None:
59 ver_str = _run_ssh_version()
60 m = re.match(r'^OpenSSH_([0-9.]+)(p[0-9]+)?\s', ver_str)
61 if m:
62 return tuple(int(x) for x in m.group(1).split('.'))
63 else:
64 return ()
65
66
67def ssh_version():
68 """return ssh version as a tuple"""
69 global _ssh_version
70 if _ssh_version is None:
71 try:
72 _ssh_version = _parse_ssh_version()
73 except subprocess.CalledProcessError:
74 print('fatal: unable to detect ssh version', file=sys.stderr)
75 sys.exit(1)
76 return _ssh_version
77
78
79def ssh_sock(create=True):
80 global _ssh_sock_path
81 if _ssh_sock_path is None:
82 if not create:
83 return None
84 tmp_dir = '/tmp'
85 if not os.path.exists(tmp_dir):
86 tmp_dir = tempfile.gettempdir()
87 if ssh_version() < (6, 7):
88 tokens = '%r@%h:%p'
89 else:
90 tokens = '%C' # hash of %l%h%p%r
91 _ssh_sock_path = os.path.join(
92 tempfile.mkdtemp('', 'ssh-', tmp_dir),
93 'master-' + tokens)
94 return _ssh_sock_path
95
96
97def _ssh_proxy():
98 global _ssh_proxy_path
99 if _ssh_proxy_path is None:
100 _ssh_proxy_path = os.path.join(
101 os.path.dirname(__file__),
102 'git_ssh')
103 return _ssh_proxy_path
104
105
106def _add_ssh_client(p):
107 _ssh_clients.append(p)
108
109
110def _remove_ssh_client(p):
111 try:
112 _ssh_clients.remove(p)
113 except ValueError:
114 pass
115
116
117def terminate_ssh_clients():
118 global _ssh_clients
119 for p in _ssh_clients:
120 try:
121 os.kill(p.pid, SIGTERM)
122 p.wait()
123 except OSError:
124 pass
125 _ssh_clients = []
126
127
128_git_version = None
129
130 43
131class _GitCall(object): 44class _GitCall(object):
45 @functools.lru_cache(maxsize=None)
132 def version_tuple(self): 46 def version_tuple(self):
133 global _git_version 47 ret = Wrapper().ParseGitVersion()
134 if _git_version is None: 48 if ret is None:
135 _git_version = Wrapper().ParseGitVersion() 49 print('fatal: unable to detect git version', file=sys.stderr)
136 if _git_version is None: 50 sys.exit(1)
137 print('fatal: unable to detect git version', file=sys.stderr) 51 return ret
138 sys.exit(1)
139 return _git_version
140 52
141 def __getattr__(self, name): 53 def __getattr__(self, name):
142 name = name.replace('_', '-') 54 name = name.replace('_', '-')
@@ -163,7 +75,8 @@ def RepoSourceVersion():
163 proj = os.path.dirname(os.path.abspath(__file__)) 75 proj = os.path.dirname(os.path.abspath(__file__))
164 env[GIT_DIR] = os.path.join(proj, '.git') 76 env[GIT_DIR] = os.path.join(proj, '.git')
165 result = subprocess.run([GIT, 'describe', HEAD], stdout=subprocess.PIPE, 77 result = subprocess.run([GIT, 'describe', HEAD], stdout=subprocess.PIPE,
166 encoding='utf-8', env=env, check=False) 78 stderr=subprocess.DEVNULL, encoding='utf-8',
79 env=env, check=False)
167 if result.returncode == 0: 80 if result.returncode == 0:
168 ver = result.stdout.strip() 81 ver = result.stdout.strip()
169 if ver.startswith('v'): 82 if ver.startswith('v'):
@@ -254,7 +167,7 @@ class GitCommand(object):
254 capture_stderr=False, 167 capture_stderr=False,
255 merge_output=False, 168 merge_output=False,
256 disable_editor=False, 169 disable_editor=False,
257 ssh_proxy=False, 170 ssh_proxy=None,
258 cwd=None, 171 cwd=None,
259 gitdir=None): 172 gitdir=None):
260 env = self._GetBasicEnv() 173 env = self._GetBasicEnv()
@@ -262,8 +175,8 @@ class GitCommand(object):
262 if disable_editor: 175 if disable_editor:
263 env['GIT_EDITOR'] = ':' 176 env['GIT_EDITOR'] = ':'
264 if ssh_proxy: 177 if ssh_proxy:
265 env['REPO_SSH_SOCK'] = ssh_sock() 178 env['REPO_SSH_SOCK'] = ssh_proxy.sock()
266 env['GIT_SSH'] = _ssh_proxy() 179 env['GIT_SSH'] = ssh_proxy.proxy
267 env['GIT_SSH_VARIANT'] = 'ssh' 180 env['GIT_SSH_VARIANT'] = 'ssh'
268 if 'http_proxy' in env and 'darwin' == sys.platform: 181 if 'http_proxy' in env and 'darwin' == sys.platform:
269 s = "'http.proxy=%s'" % (env['http_proxy'],) 182 s = "'http.proxy=%s'" % (env['http_proxy'],)
@@ -346,7 +259,7 @@ class GitCommand(object):
346 raise GitError('%s: %s' % (command[1], e)) 259 raise GitError('%s: %s' % (command[1], e))
347 260
348 if ssh_proxy: 261 if ssh_proxy:
349 _add_ssh_client(p) 262 ssh_proxy.add_client(p)
350 263
351 self.process = p 264 self.process = p
352 if input: 265 if input:
@@ -358,7 +271,8 @@ class GitCommand(object):
358 try: 271 try:
359 self.stdout, self.stderr = p.communicate() 272 self.stdout, self.stderr = p.communicate()
360 finally: 273 finally:
361 _remove_ssh_client(p) 274 if ssh_proxy:
275 ssh_proxy.remove_client(p)
362 self.rc = p.wait() 276 self.rc = p.wait()
363 277
364 @staticmethod 278 @staticmethod
diff --git a/git_config.py b/git_config.py
index fcd0446c..3cd09391 100644
--- a/git_config.py
+++ b/git_config.py
@@ -13,32 +13,28 @@
13# limitations under the License. 13# limitations under the License.
14 14
15import contextlib 15import contextlib
16import datetime
16import errno 17import errno
17from http.client import HTTPException 18from http.client import HTTPException
18import json 19import json
19import os 20import os
20import re 21import re
21import signal
22import ssl 22import ssl
23import subprocess 23import subprocess
24import sys 24import sys
25try:
26 import threading as _threading
27except ImportError:
28 import dummy_threading as _threading
29import time
30import urllib.error 25import urllib.error
31import urllib.request 26import urllib.request
32 27
33from error import GitError, UploadError 28from error import GitError, UploadError
34import platform_utils 29import platform_utils
35from repo_trace import Trace 30from repo_trace import Trace
36
37from git_command import GitCommand 31from git_command import GitCommand
38from git_command import ssh_sock
39from git_command import terminate_ssh_clients
40from git_refs import R_CHANGES, R_HEADS, R_TAGS 32from git_refs import R_CHANGES, R_HEADS, R_TAGS
41 33
34# Prefix that is prepended to all the keys of SyncAnalysisState's data
35# that is saved in the config.
36SYNC_STATE_PREFIX = 'repo.syncstate.'
37
42ID_RE = re.compile(r'^[0-9a-f]{40}$') 38ID_RE = re.compile(r'^[0-9a-f]{40}$')
43 39
44REVIEW_CACHE = dict() 40REVIEW_CACHE = dict()
@@ -74,6 +70,15 @@ class GitConfig(object):
74 70
75 _USER_CONFIG = '~/.gitconfig' 71 _USER_CONFIG = '~/.gitconfig'
76 72
73 _ForSystem = None
74 _SYSTEM_CONFIG = '/etc/gitconfig'
75
76 @classmethod
77 def ForSystem(cls):
78 if cls._ForSystem is None:
79 cls._ForSystem = cls(configfile=cls._SYSTEM_CONFIG)
80 return cls._ForSystem
81
77 @classmethod 82 @classmethod
78 def ForUser(cls): 83 def ForUser(cls):
79 if cls._ForUser is None: 84 if cls._ForUser is None:
@@ -99,6 +104,10 @@ class GitConfig(object):
99 os.path.dirname(self.file), 104 os.path.dirname(self.file),
100 '.repo_' + os.path.basename(self.file) + '.json') 105 '.repo_' + os.path.basename(self.file) + '.json')
101 106
107 def ClearCache(self):
108 """Clear the in-memory cache of config."""
109 self._cache_dict = None
110
102 def Has(self, name, include_defaults=True): 111 def Has(self, name, include_defaults=True):
103 """Return true if this configuration file has the key. 112 """Return true if this configuration file has the key.
104 """ 113 """
@@ -262,6 +271,22 @@ class GitConfig(object):
262 self._branches[b.name] = b 271 self._branches[b.name] = b
263 return b 272 return b
264 273
274 def GetSyncAnalysisStateData(self):
275 """Returns data to be logged for the analysis of sync performance."""
276 return {k: v for k, v in self.DumpConfigDict().items() if k.startswith(SYNC_STATE_PREFIX)}
277
278 def UpdateSyncAnalysisState(self, options, superproject_logging_data):
279 """Update Config's SYNC_STATE_PREFIX* data with the latest sync data.
280
281 Args:
282 options: Options passed to sync returned from optparse. See _Options().
283 superproject_logging_data: A dictionary of superproject data that is to be logged.
284
285 Returns:
286 SyncAnalysisState object.
287 """
288 return SyncAnalysisState(self, options, superproject_logging_data)
289
265 def GetSubSections(self, section): 290 def GetSubSections(self, section):
266 """List all subsection names matching $section.*.* 291 """List all subsection names matching $section.*.*
267 """ 292 """
@@ -327,8 +352,8 @@ class GitConfig(object):
327 Trace(': parsing %s', self.file) 352 Trace(': parsing %s', self.file)
328 with open(self._json) as fd: 353 with open(self._json) as fd:
329 return json.load(fd) 354 return json.load(fd)
330 except (IOError, ValueError): 355 except (IOError, ValueErrorl):
331 platform_utils.remove(self._json) 356 platform_utils.remove(self._json, missing_ok=True)
332 return None 357 return None
333 358
334 def _SaveJson(self, cache): 359 def _SaveJson(self, cache):
@@ -336,8 +361,7 @@ class GitConfig(object):
336 with open(self._json, 'w') as fd: 361 with open(self._json, 'w') as fd:
337 json.dump(cache, fd, indent=2) 362 json.dump(cache, fd, indent=2)
338 except (IOError, TypeError): 363 except (IOError, TypeError):
339 if os.path.exists(self._json): 364 platform_utils.remove(self._json, missing_ok=True)
340 platform_utils.remove(self._json)
341 365
342 def _ReadGit(self): 366 def _ReadGit(self):
343 """ 367 """
@@ -347,9 +371,10 @@ class GitConfig(object):
347 371
348 """ 372 """
349 c = {} 373 c = {}
350 d = self._do('--null', '--list') 374 if not os.path.exists(self.file):
351 if d is None:
352 return c 375 return c
376
377 d = self._do('--null', '--list')
353 for line in d.rstrip('\0').split('\0'): 378 for line in d.rstrip('\0').split('\0'):
354 if '\n' in line: 379 if '\n' in line:
355 key, val = line.split('\n', 1) 380 key, val = line.split('\n', 1)
@@ -365,7 +390,10 @@ class GitConfig(object):
365 return c 390 return c
366 391
367 def _do(self, *args): 392 def _do(self, *args):
368 command = ['config', '--file', self.file, '--includes'] 393 if self.file == self._SYSTEM_CONFIG:
394 command = ['config', '--system', '--includes']
395 else:
396 command = ['config', '--file', self.file, '--includes']
369 command.extend(args) 397 command.extend(args)
370 398
371 p = GitCommand(None, 399 p = GitCommand(None,
@@ -375,7 +403,7 @@ class GitConfig(object):
375 if p.Wait() == 0: 403 if p.Wait() == 0:
376 return p.stdout 404 return p.stdout
377 else: 405 else:
378 GitError('git config %s: %s' % (str(args), p.stderr)) 406 raise GitError('git config %s: %s' % (str(args), p.stderr))
379 407
380 408
381class RepoConfig(GitConfig): 409class RepoConfig(GitConfig):
@@ -440,129 +468,6 @@ class RefSpec(object):
440 return s 468 return s
441 469
442 470
443_master_processes = []
444_master_keys = set()
445_ssh_master = True
446_master_keys_lock = None
447
448
449def init_ssh():
450 """Should be called once at the start of repo to init ssh master handling.
451
452 At the moment, all we do is to create our lock.
453 """
454 global _master_keys_lock
455 assert _master_keys_lock is None, "Should only call init_ssh once"
456 _master_keys_lock = _threading.Lock()
457
458
459def _open_ssh(host, port=None):
460 global _ssh_master
461
462 # Bail before grabbing the lock if we already know that we aren't going to
463 # try creating new masters below.
464 if sys.platform in ('win32', 'cygwin'):
465 return False
466
467 # Acquire the lock. This is needed to prevent opening multiple masters for
468 # the same host when we're running "repo sync -jN" (for N > 1) _and_ the
469 # manifest <remote fetch="ssh://xyz"> specifies a different host from the
470 # one that was passed to repo init.
471 _master_keys_lock.acquire()
472 try:
473
474 # Check to see whether we already think that the master is running; if we
475 # think it's already running, return right away.
476 if port is not None:
477 key = '%s:%s' % (host, port)
478 else:
479 key = host
480
481 if key in _master_keys:
482 return True
483
484 if not _ssh_master or 'GIT_SSH' in os.environ:
485 # Failed earlier, so don't retry.
486 return False
487
488 # We will make two calls to ssh; this is the common part of both calls.
489 command_base = ['ssh',
490 '-o', 'ControlPath %s' % ssh_sock(),
491 host]
492 if port is not None:
493 command_base[1:1] = ['-p', str(port)]
494
495 # Since the key wasn't in _master_keys, we think that master isn't running.
496 # ...but before actually starting a master, we'll double-check. This can
497 # be important because we can't tell that that 'git@myhost.com' is the same
498 # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
499 check_command = command_base + ['-O', 'check']
500 try:
501 Trace(': %s', ' '.join(check_command))
502 check_process = subprocess.Popen(check_command,
503 stdout=subprocess.PIPE,
504 stderr=subprocess.PIPE)
505 check_process.communicate() # read output, but ignore it...
506 isnt_running = check_process.wait()
507
508 if not isnt_running:
509 # Our double-check found that the master _was_ infact running. Add to
510 # the list of keys.
511 _master_keys.add(key)
512 return True
513 except Exception:
514 # Ignore excpetions. We we will fall back to the normal command and print
515 # to the log there.
516 pass
517
518 command = command_base[:1] + ['-M', '-N'] + command_base[1:]
519 try:
520 Trace(': %s', ' '.join(command))
521 p = subprocess.Popen(command)
522 except Exception as e:
523 _ssh_master = False
524 print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
525 % (host, port, str(e)), file=sys.stderr)
526 return False
527
528 time.sleep(1)
529 ssh_died = (p.poll() is not None)
530 if ssh_died:
531 return False
532
533 _master_processes.append(p)
534 _master_keys.add(key)
535 return True
536 finally:
537 _master_keys_lock.release()
538
539
540def close_ssh():
541 global _master_keys_lock
542
543 terminate_ssh_clients()
544
545 for p in _master_processes:
546 try:
547 os.kill(p.pid, signal.SIGTERM)
548 p.wait()
549 except OSError:
550 pass
551 del _master_processes[:]
552 _master_keys.clear()
553
554 d = ssh_sock(create=False)
555 if d:
556 try:
557 platform_utils.rmdir(os.path.dirname(d))
558 except OSError:
559 pass
560
561 # We're done with the lock, so we can delete it.
562 _master_keys_lock = None
563
564
565URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
566URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/') 471URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
567 472
568 473
@@ -614,27 +519,6 @@ def GetUrlCookieFile(url, quiet):
614 yield cookiefile, None 519 yield cookiefile, None
615 520
616 521
617def _preconnect(url):
618 m = URI_ALL.match(url)
619 if m:
620 scheme = m.group(1)
621 host = m.group(2)
622 if ':' in host:
623 host, port = host.split(':')
624 else:
625 port = None
626 if scheme in ('ssh', 'git+ssh', 'ssh+git'):
627 return _open_ssh(host, port)
628 return False
629
630 m = URI_SCP.match(url)
631 if m:
632 host = m.group(1)
633 return _open_ssh(host)
634
635 return False
636
637
638class Remote(object): 522class Remote(object):
639 """Configuration options related to a remote. 523 """Configuration options related to a remote.
640 """ 524 """
@@ -671,9 +555,23 @@ class Remote(object):
671 555
672 return self.url.replace(longest, longestUrl, 1) 556 return self.url.replace(longest, longestUrl, 1)
673 557
674 def PreConnectFetch(self): 558 def PreConnectFetch(self, ssh_proxy):
559 """Run any setup for this remote before we connect to it.
560
561 In practice, if the remote is using SSH, we'll attempt to create a new
562 SSH master session to it for reuse across projects.
563
564 Args:
565 ssh_proxy: The SSH settings for managing master sessions.
566
567 Returns:
568 Whether the preconnect phase for this remote was successful.
569 """
570 if not ssh_proxy:
571 return True
572
675 connectionUrl = self._InsteadOf() 573 connectionUrl = self._InsteadOf()
676 return _preconnect(connectionUrl) 574 return ssh_proxy.preconnect(connectionUrl)
677 575
678 def ReviewUrl(self, userEmail, validate_certs): 576 def ReviewUrl(self, userEmail, validate_certs):
679 if self._review_url is None: 577 if self._review_url is None:
@@ -844,3 +742,70 @@ class Branch(object):
844 def _Get(self, key, all_keys=False): 742 def _Get(self, key, all_keys=False):
845 key = 'branch.%s.%s' % (self.name, key) 743 key = 'branch.%s.%s' % (self.name, key)
846 return self._config.GetString(key, all_keys=all_keys) 744 return self._config.GetString(key, all_keys=all_keys)
745
746
747class SyncAnalysisState:
748 """Configuration options related to logging of sync state for analysis.
749
750 This object is versioned.
751 """
752 def __init__(self, config, options, superproject_logging_data):
753 """Initializes SyncAnalysisState.
754
755 Saves the following data into the |config| object.
756 - sys.argv, options, superproject's logging data.
757 - repo.*, branch.* and remote.* parameters from config object.
758 - Current time as synctime.
759 - Version number of the object.
760
761 All the keys saved by this object are prepended with SYNC_STATE_PREFIX.
762
763 Args:
764 config: GitConfig object to store all options.
765 options: Options passed to sync returned from optparse. See _Options().
766 superproject_logging_data: A dictionary of superproject data that is to be logged.
767 """
768 self._config = config
769 now = datetime.datetime.utcnow()
770 self._Set('main.synctime', now.isoformat() + 'Z')
771 self._Set('main.version', '1')
772 self._Set('sys.argv', sys.argv)
773 for key, value in superproject_logging_data.items():
774 self._Set(f'superproject.{key}', value)
775 for key, value in options.__dict__.items():
776 self._Set(f'options.{key}', value)
777 config_items = config.DumpConfigDict().items()
778 EXTRACT_NAMESPACES = {'repo', 'branch', 'remote'}
779 self._SetDictionary({k: v for k, v in config_items
780 if not k.startswith(SYNC_STATE_PREFIX) and
781 k.split('.', 1)[0] in EXTRACT_NAMESPACES})
782
783 def _SetDictionary(self, data):
784 """Save all key/value pairs of |data| dictionary.
785
786 Args:
787 data: A dictionary whose key/value are to be saved.
788 """
789 for key, value in data.items():
790 self._Set(key, value)
791
792 def _Set(self, key, value):
793 """Set the |value| for a |key| in the |_config| member.
794
795 |key| is prepended with the value of SYNC_STATE_PREFIX constant.
796
797 Args:
798 key: Name of the key.
799 value: |value| could be of any type. If it is 'bool', it will be saved
800 as a Boolean and for all other types, it will be saved as a String.
801 """
802 if value is None:
803 return
804 sync_key = f'{SYNC_STATE_PREFIX}{key}'
805 sync_key = sync_key.replace('_', '')
806 if isinstance(value, str):
807 self._config.SetString(sync_key, value)
808 elif isinstance(value, bool):
809 self._config.SetBoolean(sync_key, value)
810 else:
811 self._config.SetString(sync_key, str(value))
diff --git a/git_superproject.py b/git_superproject.py
index 89320971..4ca84a58 100644
--- a/git_superproject.py
+++ b/git_superproject.py
@@ -19,21 +19,52 @@ https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
19 19
20Examples: 20Examples:
21 superproject = Superproject() 21 superproject = Superproject()
22 project_commit_ids = superproject.UpdateProjectsRevisionId(projects) 22 UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects)
23""" 23"""
24 24
25import hashlib 25import hashlib
26import functools
26import os 27import os
27import sys 28import sys
29import time
30from typing import NamedTuple
28 31
29from error import BUG_REPORT_URL 32from git_command import git_require, GitCommand
30from git_command import GitCommand 33from git_config import RepoConfig
31from git_refs import R_HEADS 34from git_refs import R_HEADS
35from manifest_xml import LOCAL_MANIFEST_GROUP_PREFIX
32 36
33_SUPERPROJECT_GIT_NAME = 'superproject.git' 37_SUPERPROJECT_GIT_NAME = 'superproject.git'
34_SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml' 38_SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml'
35 39
36 40
41class SyncResult(NamedTuple):
42 """Return the status of sync and whether caller should exit."""
43
44 # Whether the superproject sync was successful.
45 success: bool
46 # Whether the caller should exit.
47 fatal: bool
48
49
50class CommitIdsResult(NamedTuple):
51 """Return the commit ids and whether caller should exit."""
52
53 # A dictionary with the projects/commit ids on success, otherwise None.
54 commit_ids: dict
55 # Whether the caller should exit.
56 fatal: bool
57
58
59class UpdateProjectsResult(NamedTuple):
60 """Return the overriding manifest file and whether caller should exit."""
61
62 # Path name of the overriding manifest file if successful, otherwise None.
63 manifest_path: str
64 # Whether the caller should exit.
65 fatal: bool
66
67
37class Superproject(object): 68class Superproject(object):
38 """Get commit ids from superproject. 69 """Get commit ids from superproject.
39 70
@@ -41,21 +72,25 @@ class Superproject(object):
41 lookup of commit ids for all projects. It contains _project_commit_ids which 72 lookup of commit ids for all projects. It contains _project_commit_ids which
42 is a dictionary with project/commit id entries. 73 is a dictionary with project/commit id entries.
43 """ 74 """
44 def __init__(self, manifest, repodir, superproject_dir='exp-superproject', 75 def __init__(self, manifest, repodir, git_event_log,
45 quiet=False): 76 superproject_dir='exp-superproject', quiet=False, print_messages=False):
46 """Initializes superproject. 77 """Initializes superproject.
47 78
48 Args: 79 Args:
49 manifest: A Manifest object that is to be written to a file. 80 manifest: A Manifest object that is to be written to a file.
50 repodir: Path to the .repo/ dir for holding all internal checkout state. 81 repodir: Path to the .repo/ dir for holding all internal checkout state.
51 It must be in the top directory of the repo client checkout. 82 It must be in the top directory of the repo client checkout.
83 git_event_log: A git trace2 event log to log events.
52 superproject_dir: Relative path under |repodir| to checkout superproject. 84 superproject_dir: Relative path under |repodir| to checkout superproject.
53 quiet: If True then only print the progress messages. 85 quiet: If True then only print the progress messages.
86 print_messages: if True then print error/warning messages.
54 """ 87 """
55 self._project_commit_ids = None 88 self._project_commit_ids = None
56 self._manifest = manifest 89 self._manifest = manifest
90 self._git_event_log = git_event_log
57 self._quiet = quiet 91 self._quiet = quiet
58 self._branch = self._GetBranch() 92 self._print_messages = print_messages
93 self._branch = manifest.branch
59 self._repodir = os.path.abspath(repodir) 94 self._repodir = os.path.abspath(repodir)
60 self._superproject_dir = superproject_dir 95 self._superproject_dir = superproject_dir
61 self._superproject_path = os.path.join(self._repodir, superproject_dir) 96 self._superproject_path = os.path.join(self._repodir, superproject_dir)
@@ -63,8 +98,12 @@ class Superproject(object):
63 _SUPERPROJECT_MANIFEST_NAME) 98 _SUPERPROJECT_MANIFEST_NAME)
64 git_name = '' 99 git_name = ''
65 if self._manifest.superproject: 100 if self._manifest.superproject:
66 remote_name = self._manifest.superproject['remote'].name 101 remote = self._manifest.superproject['remote']
67 git_name = hashlib.md5(remote_name.encode('utf8')).hexdigest() + '-' 102 git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-'
103 self._branch = self._manifest.superproject['revision']
104 self._remote_url = remote.url
105 else:
106 self._remote_url = None
68 self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME 107 self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
69 self._work_git = os.path.join(self._superproject_path, self._work_git_name) 108 self._work_git = os.path.join(self._superproject_path, self._work_git_name)
70 109
@@ -73,16 +112,28 @@ class Superproject(object):
73 """Returns a dictionary of projects and their commit ids.""" 112 """Returns a dictionary of projects and their commit ids."""
74 return self._project_commit_ids 113 return self._project_commit_ids
75 114
76 def _GetBranch(self): 115 @property
77 """Returns the branch name for getting the approved manifest.""" 116 def manifest_path(self):
78 p = self._manifest.manifestProject 117 """Returns the manifest path if the path exists or None."""
79 b = p.GetBranch(p.CurrentBranch) 118 return self._manifest_path if os.path.exists(self._manifest_path) else None
80 if not b: 119
81 return None 120 def _LogMessage(self, message):
82 branch = b.merge 121 """Logs message to stderr and _git_event_log."""
83 if branch and branch.startswith(R_HEADS): 122 if self._print_messages:
84 branch = branch[len(R_HEADS):] 123 print(message, file=sys.stderr)
85 return branch 124 self._git_event_log.ErrorEvent(message, f'{message}')
125
126 def _LogMessagePrefix(self):
127 """Returns the prefix string to be logged in each log message"""
128 return f'repo superproject branch: {self._branch} url: {self._remote_url}'
129
130 def _LogError(self, message):
131 """Logs error message to stderr and _git_event_log."""
132 self._LogMessage(f'{self._LogMessagePrefix()} error: {message}')
133
134 def _LogWarning(self, message):
135 """Logs warning message to stderr and _git_event_log."""
136 self._LogMessage(f'{self._LogMessagePrefix()} warning: {message}')
86 137
87 def _Init(self): 138 def _Init(self):
88 """Sets up a local Git repository to get a copy of a superproject. 139 """Sets up a local Git repository to get a copy of a superproject.
@@ -103,25 +154,25 @@ class Superproject(object):
103 capture_stderr=True) 154 capture_stderr=True)
104 retval = p.Wait() 155 retval = p.Wait()
105 if retval: 156 if retval:
106 print('repo: error: git init call failed with return code: %r, stderr: %r' % 157 self._LogWarning(f'git init call failed, command: git {cmd}, '
107 (retval, p.stderr), file=sys.stderr) 158 f'return code: {retval}, stderr: {p.stderr}')
108 return False 159 return False
109 return True 160 return True
110 161
111 def _Fetch(self, url): 162 def _Fetch(self):
112 """Fetches a local copy of a superproject for the manifest based on url. 163 """Fetches a local copy of a superproject for the manifest based on |_remote_url|.
113
114 Args:
115 url: superproject's url.
116 164
117 Returns: 165 Returns:
118 True if fetch is successful, or False. 166 True if fetch is successful, or False.
119 """ 167 """
120 if not os.path.exists(self._work_git): 168 if not os.path.exists(self._work_git):
121 print('git fetch missing drectory: %s' % self._work_git, 169 self._LogWarning(f'git fetch missing directory: {self._work_git}')
122 file=sys.stderr)
123 return False 170 return False
124 cmd = ['fetch', url, '--depth', '1', '--force', '--no-tags', '--filter', 'blob:none'] 171 if not git_require((2, 28, 0)):
172 self._LogWarning('superproject requires a git version 2.28 or later')
173 return False
174 cmd = ['fetch', self._remote_url, '--depth', '1', '--force', '--no-tags',
175 '--filter', 'blob:none']
125 if self._branch: 176 if self._branch:
126 cmd += [self._branch + ':' + self._branch] 177 cmd += [self._branch + ':' + self._branch]
127 p = GitCommand(None, 178 p = GitCommand(None,
@@ -131,8 +182,8 @@ class Superproject(object):
131 capture_stderr=True) 182 capture_stderr=True)
132 retval = p.Wait() 183 retval = p.Wait()
133 if retval: 184 if retval:
134 print('repo: error: git fetch call failed with return code: %r, stderr: %r' % 185 self._LogWarning(f'git fetch call failed, command: git {cmd}, '
135 (retval, p.stderr), file=sys.stderr) 186 f'return code: {retval}, stderr: {p.stderr}')
136 return False 187 return False
137 return True 188 return True
138 189
@@ -145,8 +196,7 @@ class Superproject(object):
145 data: data returned from 'git ls-tree ...' instead of None. 196 data: data returned from 'git ls-tree ...' instead of None.
146 """ 197 """
147 if not os.path.exists(self._work_git): 198 if not os.path.exists(self._work_git):
148 print('git ls-tree missing drectory: %s' % self._work_git, 199 self._LogWarning(f'git ls-tree missing directory: {self._work_git}')
149 file=sys.stderr)
150 return None 200 return None
151 data = None 201 data = None
152 branch = 'HEAD' if not self._branch else self._branch 202 branch = 'HEAD' if not self._branch else self._branch
@@ -161,52 +211,52 @@ class Superproject(object):
161 if retval == 0: 211 if retval == 0:
162 data = p.stdout 212 data = p.stdout
163 else: 213 else:
164 print('repo: error: git ls-tree call failed with return code: %r, stderr: %r' % ( 214 self._LogWarning(f'git ls-tree call failed, command: git {cmd}, '
165 retval, p.stderr), file=sys.stderr) 215 f'return code: {retval}, stderr: {p.stderr}')
166 return data 216 return data
167 217
168 def Sync(self): 218 def Sync(self):
169 """Gets a local copy of a superproject for the manifest. 219 """Gets a local copy of a superproject for the manifest.
170 220
171 Returns: 221 Returns:
172 True if sync of superproject is successful, or False. 222 SyncResult
173 """ 223 """
174 print('WARNING: --use-superproject is experimental and not '
175 'for general use', file=sys.stderr)
176
177 if not self._manifest.superproject: 224 if not self._manifest.superproject:
178 print('error: superproject tag is not defined in manifest', 225 self._LogWarning(f'superproject tag is not defined in manifest: '
179 file=sys.stderr) 226 f'{self._manifest.manifestFile}')
180 return False 227 return SyncResult(False, False)
181 228
182 url = self._manifest.superproject['remote'].url 229 print('NOTICE: --use-superproject is in beta; report any issues to the '
183 if not url: 230 'address described in `repo version`', file=sys.stderr)
184 print('error: superproject URL is not defined in manifest', 231 should_exit = True
185 file=sys.stderr) 232 if not self._remote_url:
186 return False 233 self._LogWarning(f'superproject URL is not defined in manifest: '
234 f'{self._manifest.manifestFile}')
235 return SyncResult(False, should_exit)
187 236
188 if not self._Init(): 237 if not self._Init():
189 return False 238 return SyncResult(False, should_exit)
190 if not self._Fetch(url): 239 if not self._Fetch():
191 return False 240 return SyncResult(False, should_exit)
192 if not self._quiet: 241 if not self._quiet:
193 print('%s: Initial setup for superproject completed.' % self._work_git) 242 print('%s: Initial setup for superproject completed.' % self._work_git)
194 return True 243 return SyncResult(True, False)
195 244
196 def _GetAllProjectsCommitIds(self): 245 def _GetAllProjectsCommitIds(self):
197 """Get commit ids for all projects from superproject and save them in _project_commit_ids. 246 """Get commit ids for all projects from superproject and save them in _project_commit_ids.
198 247
199 Returns: 248 Returns:
200 A dictionary with the projects/commit ids on success, otherwise None. 249 CommitIdsResult
201 """ 250 """
202 if not self.Sync(): 251 sync_result = self.Sync()
203 return None 252 if not sync_result.success:
253 return CommitIdsResult(None, sync_result.fatal)
204 254
205 data = self._LsTree() 255 data = self._LsTree()
206 if not data: 256 if not data:
207 print('error: git ls-tree failed to return data for superproject', 257 self._LogWarning(f'git ls-tree failed to return data for manifest: '
208 file=sys.stderr) 258 f'{self._manifest.manifestFile}')
209 return None 259 return CommitIdsResult(None, True)
210 260
211 # Parse lines like the following to select lines starting with '160000' and 261 # Parse lines like the following to select lines starting with '160000' and
212 # build a dictionary with project path (last element) and its commit id (3rd element). 262 # build a dictionary with project path (last element) and its commit id (3rd element).
@@ -222,18 +272,16 @@ class Superproject(object):
222 commit_ids[ls_data[3]] = ls_data[2] 272 commit_ids[ls_data[3]] = ls_data[2]
223 273
224 self._project_commit_ids = commit_ids 274 self._project_commit_ids = commit_ids
225 return commit_ids 275 return CommitIdsResult(commit_ids, False)
226 276
227 def _WriteManfiestFile(self): 277 def _WriteManifestFile(self):
228 """Writes manifest to a file. 278 """Writes manifest to a file.
229 279
230 Returns: 280 Returns:
231 manifest_path: Path name of the file into which manifest is written instead of None. 281 manifest_path: Path name of the file into which manifest is written instead of None.
232 """ 282 """
233 if not os.path.exists(self._superproject_path): 283 if not os.path.exists(self._superproject_path):
234 print('error: missing superproject directory %s' % 284 self._LogWarning(f'missing superproject directory: {self._superproject_path}')
235 self._superproject_path,
236 file=sys.stderr)
237 return None 285 return None
238 manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr()).toxml() 286 manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr()).toxml()
239 manifest_path = self._manifest_path 287 manifest_path = self._manifest_path
@@ -241,12 +289,30 @@ class Superproject(object):
241 with open(manifest_path, 'w', encoding='utf-8') as fp: 289 with open(manifest_path, 'w', encoding='utf-8') as fp:
242 fp.write(manifest_str) 290 fp.write(manifest_str)
243 except IOError as e: 291 except IOError as e:
244 print('error: cannot write manifest to %s:\n%s' 292 self._LogError(f'cannot write manifest to : {manifest_path} {e}')
245 % (manifest_path, e),
246 file=sys.stderr)
247 return None 293 return None
248 return manifest_path 294 return manifest_path
249 295
296 def _SkipUpdatingProjectRevisionId(self, project):
297 """Checks if a project's revision id needs to be updated or not.
298
299 Revision id for projects from local manifest will not be updated.
300
301 Args:
302 project: project whose revision id is being updated.
303
304 Returns:
305 True if a project's revision id should not be updated, or False,
306 """
307 path = project.relpath
308 if not path:
309 return True
310 # Skip the project with revisionId.
311 if project.revisionId:
312 return True
313 # Skip the project if it comes from the local manifest.
314 return any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups)
315
250 def UpdateProjectsRevisionId(self, projects): 316 def UpdateProjectsRevisionId(self, projects):
251 """Update revisionId of every project in projects with the commit id. 317 """Update revisionId of every project in projects with the commit id.
252 318
@@ -254,27 +320,96 @@ class Superproject(object):
254 projects: List of projects whose revisionId needs to be updated. 320 projects: List of projects whose revisionId needs to be updated.
255 321
256 Returns: 322 Returns:
257 manifest_path: Path name of the overriding manfiest file instead of None. 323 UpdateProjectsResult
258 """ 324 """
259 commit_ids = self._GetAllProjectsCommitIds() 325 commit_ids_result = self._GetAllProjectsCommitIds()
326 commit_ids = commit_ids_result.commit_ids
260 if not commit_ids: 327 if not commit_ids:
261 print('error: Cannot get project commit ids from manifest', file=sys.stderr) 328 return UpdateProjectsResult(None, commit_ids_result.fatal)
262 return None
263 329
264 projects_missing_commit_ids = [] 330 projects_missing_commit_ids = []
265 for project in projects: 331 for project in projects:
266 path = project.relpath 332 if self._SkipUpdatingProjectRevisionId(project):
267 if not path:
268 continue 333 continue
334 path = project.relpath
269 commit_id = commit_ids.get(path) 335 commit_id = commit_ids.get(path)
270 if commit_id: 336 if not commit_id:
271 project.SetRevisionId(commit_id)
272 else:
273 projects_missing_commit_ids.append(path) 337 projects_missing_commit_ids.append(path)
338
339 # If superproject doesn't have a commit id for a project, then report an
340 # error event and continue as if do not use superproject is specified.
274 if projects_missing_commit_ids: 341 if projects_missing_commit_ids:
275 print('error: please file a bug using %s to report missing commit_ids for: %s' % 342 self._LogWarning(f'please file a bug using {self._manifest.contactinfo.bugurl} '
276 (BUG_REPORT_URL, projects_missing_commit_ids), file=sys.stderr) 343 f'to report missing commit_ids for: {projects_missing_commit_ids}')
277 return None 344 return UpdateProjectsResult(None, False)
278 345
279 manifest_path = self._WriteManfiestFile() 346 for project in projects:
280 return manifest_path 347 if not self._SkipUpdatingProjectRevisionId(project):
348 project.SetRevisionId(commit_ids.get(project.relpath))
349
350 manifest_path = self._WriteManifestFile()
351 return UpdateProjectsResult(manifest_path, False)
352
353
354@functools.lru_cache(maxsize=None)
355def _UseSuperprojectFromConfiguration():
356 """Returns the user choice of whether to use superproject."""
357 user_cfg = RepoConfig.ForUser()
358 time_now = int(time.time())
359
360 user_value = user_cfg.GetBoolean('repo.superprojectChoice')
361 if user_value is not None:
362 user_expiration = user_cfg.GetInt('repo.superprojectChoiceExpire')
363 if user_expiration is None or user_expiration <= 0 or user_expiration >= time_now:
364 # TODO(b/190688390) - Remove prompt when we are comfortable with the new
365 # default value.
366 if user_value:
367 print(('You are currently enrolled in Git submodules experiment '
368 '(go/android-submodules-quickstart). Use --no-use-superproject '
369 'to override.\n'), file=sys.stderr)
370 else:
371 print(('You are not currently enrolled in Git submodules experiment '
372 '(go/android-submodules-quickstart). Use --use-superproject '
373 'to override.\n'), file=sys.stderr)
374 return user_value
375
376 # We don't have an unexpired choice, ask for one.
377 system_cfg = RepoConfig.ForSystem()
378 system_value = system_cfg.GetBoolean('repo.superprojectChoice')
379 if system_value:
380 # The system configuration is proposing that we should enable the
381 # use of superproject. Treat the user as enrolled for two weeks.
382 #
383 # TODO(b/190688390) - Remove prompt when we are comfortable with the new
384 # default value.
385 userchoice = True
386 time_choiceexpire = time_now + (86400 * 14)
387 user_cfg.SetString('repo.superprojectChoiceExpire', str(time_choiceexpire))
388 user_cfg.SetBoolean('repo.superprojectChoice', userchoice)
389 print('You are automatically enrolled in Git submodules experiment '
390 '(go/android-submodules-quickstart) for another two weeks.\n',
391 file=sys.stderr)
392 return True
393
394 # For all other cases, we would not use superproject by default.
395 return False
396
397
398def PrintMessages(opt, manifest):
399 """Returns a boolean if error/warning messages are to be printed."""
400 return opt.use_superproject is not None or manifest.superproject
401
402
403def UseSuperproject(opt, manifest):
404 """Returns a boolean if use-superproject option is enabled."""
405
406 if opt.use_superproject is not None:
407 return opt.use_superproject
408 else:
409 client_value = manifest.manifestProject.config.GetBoolean('repo.superproject')
410 if client_value is not None:
411 return client_value
412 else:
413 if not manifest.superproject:
414 return False
415 return _UseSuperprojectFromConfiguration()
diff --git a/git_trace2_event_log.py b/git_trace2_event_log.py
index 8f12d1a9..0e5e9089 100644
--- a/git_trace2_event_log.py
+++ b/git_trace2_event_log.py
@@ -144,6 +144,19 @@ class EventLog(object):
144 command_event['subcommands'] = subcommands 144 command_event['subcommands'] = subcommands
145 self._log.append(command_event) 145 self._log.append(command_event)
146 146
147 def LogConfigEvents(self, config, event_dict_name):
148 """Append a |event_dict_name| event for each config key in |config|.
149
150 Args:
151 config: Configuration dictionary.
152 event_dict_name: Name of the event dictionary for items to be logged under.
153 """
154 for param, value in config.items():
155 event = self._CreateEventDict(event_dict_name)
156 event['param'] = param
157 event['value'] = value
158 self._log.append(event)
159
147 def DefParamRepoEvents(self, config): 160 def DefParamRepoEvents(self, config):
148 """Append a 'def_param' event for each repo.* config key to the current log. 161 """Append a 'def_param' event for each repo.* config key to the current log.
149 162
@@ -152,12 +165,34 @@ class EventLog(object):
152 """ 165 """
153 # Only output the repo.* config parameters. 166 # Only output the repo.* config parameters.
154 repo_config = {k: v for k, v in config.items() if k.startswith('repo.')} 167 repo_config = {k: v for k, v in config.items() if k.startswith('repo.')}
168 self.LogConfigEvents(repo_config, 'def_param')
169
170 def GetDataEventName(self, value):
171 """Returns 'data-json' if the value is an array else returns 'data'."""
172 return 'data-json' if value[0] == '[' and value[-1] == ']' else 'data'
155 173
156 for param, value in repo_config.items(): 174 def LogDataConfigEvents(self, config, prefix):
157 def_param_event = self._CreateEventDict('def_param') 175 """Append a 'data' event for each config key/value in |config| to the current log.
158 def_param_event['param'] = param 176
159 def_param_event['value'] = value 177 For each keyX and valueX of the config, "key" field of the event is '|prefix|/keyX'
160 self._log.append(def_param_event) 178 and the "value" of the "key" field is valueX.
179
180 Args:
181 config: Configuration dictionary.
182 prefix: Prefix for each key that is logged.
183 """
184 for key, value in config.items():
185 event = self._CreateEventDict(self.GetDataEventName(value))
186 event['key'] = f'{prefix}/{key}'
187 event['value'] = value
188 self._log.append(event)
189
190 def ErrorEvent(self, msg, fmt):
191 """Append a 'error' event to the current log."""
192 error_event = self._CreateEventDict('error')
193 error_event['msg'] = msg
194 error_event['fmt'] = fmt
195 self._log.append(error_event)
161 196
162 def _GetEventTargetPath(self): 197 def _GetEventTargetPath(self):
163 """Get the 'trace2.eventtarget' path from git configuration. 198 """Get the 'trace2.eventtarget' path from git configuration.
diff --git a/main.py b/main.py
index 8aba2ec2..2050cabb 100755
--- a/main.py
+++ b/main.py
@@ -39,7 +39,7 @@ from color import SetDefaultColoring
39import event_log 39import event_log
40from repo_trace import SetTrace 40from repo_trace import SetTrace
41from git_command import user_agent 41from git_command import user_agent
42from git_config import init_ssh, close_ssh, RepoConfig 42from git_config import RepoConfig
43from git_trace2_event_log import EventLog 43from git_trace2_event_log import EventLog
44from command import InteractiveCommand 44from command import InteractiveCommand
45from command import MirrorSafeCommand 45from command import MirrorSafeCommand
@@ -71,7 +71,7 @@ from subcmds import all_commands
71# 71#
72# python-3.6 is in Ubuntu Bionic. 72# python-3.6 is in Ubuntu Bionic.
73MIN_PYTHON_VERSION_SOFT = (3, 6) 73MIN_PYTHON_VERSION_SOFT = (3, 6)
74MIN_PYTHON_VERSION_HARD = (3, 5) 74MIN_PYTHON_VERSION_HARD = (3, 6)
75 75
76if sys.version_info.major < 3: 76if sys.version_info.major < 3:
77 print('repo: error: Python 2 is no longer supported; ' 77 print('repo: error: Python 2 is no longer supported; '
@@ -95,6 +95,8 @@ global_options = optparse.OptionParser(
95 add_help_option=False) 95 add_help_option=False)
96global_options.add_option('-h', '--help', action='store_true', 96global_options.add_option('-h', '--help', action='store_true',
97 help='show this help message and exit') 97 help='show this help message and exit')
98global_options.add_option('--help-all', action='store_true',
99 help='show this help message with all subcommands and exit')
98global_options.add_option('-p', '--paginate', 100global_options.add_option('-p', '--paginate',
99 dest='pager', action='store_true', 101 dest='pager', action='store_true',
100 help='display command output in the pager') 102 help='display command output in the pager')
@@ -116,6 +118,10 @@ global_options.add_option('--time',
116global_options.add_option('--version', 118global_options.add_option('--version',
117 dest='show_version', action='store_true', 119 dest='show_version', action='store_true',
118 help='display this version of repo') 120 help='display this version of repo')
121global_options.add_option('--show-toplevel',
122 action='store_true',
123 help='display the path of the top-level directory of '
124 'the repo client checkout')
119global_options.add_option('--event-log', 125global_options.add_option('--event-log',
120 dest='event_log', action='store', 126 dest='event_log', action='store',
121 help='filename of event log to append timeline to') 127 help='filename of event log to append timeline to')
@@ -128,34 +134,40 @@ class _Repo(object):
128 self.repodir = repodir 134 self.repodir = repodir
129 self.commands = all_commands 135 self.commands = all_commands
130 136
137 def _PrintHelp(self, short: bool = False, all_commands: bool = False):
138 """Show --help screen."""
139 global_options.print_help()
140 print()
141 if short:
142 commands = ' '.join(sorted(self.commands))
143 wrapped_commands = textwrap.wrap(commands, width=77)
144 print('Available commands:\n %s' % ('\n '.join(wrapped_commands),))
145 print('\nRun `repo help <command>` for command-specific details.')
146 print('Bug reports:', Wrapper().BUG_URL)
147 else:
148 cmd = self.commands['help']()
149 if all_commands:
150 cmd.PrintAllCommandsBody()
151 else:
152 cmd.PrintCommonCommandsBody()
153
131 def _ParseArgs(self, argv): 154 def _ParseArgs(self, argv):
132 """Parse the main `repo` command line options.""" 155 """Parse the main `repo` command line options."""
133 name = None 156 for i, arg in enumerate(argv):
134 glob = [] 157 if not arg.startswith('-'):
135 158 name = arg
136 for i in range(len(argv)): 159 glob = argv[:i]
137 if not argv[i].startswith('-'):
138 name = argv[i]
139 if i > 0:
140 glob = argv[:i]
141 argv = argv[i + 1:] 160 argv = argv[i + 1:]
142 break 161 break
143 if not name: 162 else:
163 name = None
144 glob = argv 164 glob = argv
145 name = 'help'
146 argv = [] 165 argv = []
147 gopts, _gargs = global_options.parse_args(glob) 166 gopts, _gargs = global_options.parse_args(glob)
148 167
149 name, alias_args = self._ExpandAlias(name) 168 if name:
150 argv = alias_args + argv 169 name, alias_args = self._ExpandAlias(name)
151 170 argv = alias_args + argv
152 if gopts.help:
153 global_options.print_help()
154 commands = ' '.join(sorted(self.commands))
155 wrapped_commands = textwrap.wrap(commands, width=77)
156 print('\nAvailable commands:\n %s' % ('\n '.join(wrapped_commands),))
157 print('\nRun `repo help <command>` for command-specific details.')
158 global_options.exit()
159 171
160 return (name, gopts, argv) 172 return (name, gopts, argv)
161 173
@@ -186,32 +198,44 @@ class _Repo(object):
186 198
187 if gopts.trace: 199 if gopts.trace:
188 SetTrace() 200 SetTrace()
189 if gopts.show_version: 201
190 if name == 'help': 202 # Handle options that terminate quickly first.
191 name = 'version' 203 if gopts.help or gopts.help_all:
192 else: 204 self._PrintHelp(short=False, all_commands=gopts.help_all)
193 print('fatal: invalid usage of --version', file=sys.stderr) 205 return 0
194 return 1 206 elif gopts.show_version:
207 # Always allow global --version regardless of subcommand validity.
208 name = 'version'
209 elif gopts.show_toplevel:
210 print(os.path.dirname(self.repodir))
211 return 0
212 elif not name:
213 # No subcommand specified, so show the help/subcommand.
214 self._PrintHelp(short=True)
215 return 1
195 216
196 SetDefaultColoring(gopts.color) 217 SetDefaultColoring(gopts.color)
197 218
219 git_trace2_event_log = EventLog()
220 repo_client = RepoClient(self.repodir)
221 gitc_manifest = None
222 gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
223 if gitc_client_name:
224 gitc_manifest = GitcClient(self.repodir, gitc_client_name)
225 repo_client.isGitcClient = True
226
198 try: 227 try:
199 cmd = self.commands[name]() 228 cmd = self.commands[name](
229 repodir=self.repodir,
230 client=repo_client,
231 manifest=repo_client.manifest,
232 gitc_manifest=gitc_manifest,
233 git_event_log=git_trace2_event_log)
200 except KeyError: 234 except KeyError:
201 print("repo: '%s' is not a repo command. See 'repo help'." % name, 235 print("repo: '%s' is not a repo command. See 'repo help'." % name,
202 file=sys.stderr) 236 file=sys.stderr)
203 return 1 237 return 1
204 238
205 git_trace2_event_log = EventLog()
206 cmd.repodir = self.repodir
207 cmd.client = RepoClient(cmd.repodir)
208 cmd.manifest = cmd.client.manifest
209 cmd.gitc_manifest = None
210 gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
211 if gitc_client_name:
212 cmd.gitc_manifest = GitcClient(cmd.repodir, gitc_client_name)
213 cmd.client.isGitcClient = True
214
215 Editor.globalConfig = cmd.client.globalConfig 239 Editor.globalConfig = cmd.client.globalConfig
216 240
217 if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror: 241 if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
@@ -591,20 +615,16 @@ def _Main(argv):
591 615
592 repo = _Repo(opt.repodir) 616 repo = _Repo(opt.repodir)
593 try: 617 try:
594 try: 618 init_http()
595 init_ssh() 619 name, gopts, argv = repo._ParseArgs(argv)
596 init_http() 620 run = lambda: repo._Run(name, gopts, argv) or 0
597 name, gopts, argv = repo._ParseArgs(argv) 621 if gopts.trace_python:
598 run = lambda: repo._Run(name, gopts, argv) or 0 622 import trace
599 if gopts.trace_python: 623 tracer = trace.Trace(count=False, trace=True, timing=True,
600 import trace 624 ignoredirs=set(sys.path[1:]))
601 tracer = trace.Trace(count=False, trace=True, timing=True, 625 result = tracer.runfunc(run)
602 ignoredirs=set(sys.path[1:])) 626 else:
603 result = tracer.runfunc(run) 627 result = run()
604 else:
605 result = run()
606 finally:
607 close_ssh()
608 except KeyboardInterrupt: 628 except KeyboardInterrupt:
609 print('aborted by user', file=sys.stderr) 629 print('aborted by user', file=sys.stderr)
610 result = 1 630 result = 1
diff --git a/man/repo-abandon.1 b/man/repo-abandon.1
new file mode 100644
index 00000000..b3c0422f
--- /dev/null
+++ b/man/repo-abandon.1
@@ -0,0 +1,36 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo abandon" "Repo Manual"
3.SH NAME
4repo \- repo abandon - manual page for repo abandon
5.SH SYNOPSIS
6.B repo
7\fI\,abandon \/\fR[\fI\,--all | <branchname>\/\fR] [\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Permanently abandon a development branch
12.PP
13This subcommand permanently abandons a development branch by
14deleting it (and all its history) from your local repository.
15.PP
16It is equivalent to "git branch \fB\-D\fR <branchname>".
17.SH OPTIONS
18.TP
19\fB\-h\fR, \fB\-\-help\fR
20show this help message and exit
21.TP
22\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
23number of jobs to run in parallel (default: based on
24number of CPU cores)
25.TP
26\fB\-\-all\fR
27delete all branches in all projects
28.SS Logging options:
29.TP
30\fB\-v\fR, \fB\-\-verbose\fR
31show all output
32.TP
33\fB\-q\fR, \fB\-\-quiet\fR
34only show errors
35.PP
36Run `repo help abandon` to view the detailed manual.
diff --git a/man/repo-branch.1 b/man/repo-branch.1
new file mode 100644
index 00000000..854ee98b
--- /dev/null
+++ b/man/repo-branch.1
@@ -0,0 +1 @@
.so man1/repo-branches.1 \ No newline at end of file
diff --git a/man/repo-branches.1 b/man/repo-branches.1
new file mode 100644
index 00000000..7fe0b02d
--- /dev/null
+++ b/man/repo-branches.1
@@ -0,0 +1,59 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo branches" "Repo Manual"
3.SH NAME
4repo \- repo branches - manual page for repo branches
5.SH SYNOPSIS
6.B repo
7\fI\,branches \/\fR[\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11View current topic branches
12.PP
13Summarizes the currently available topic branches.
14.PP
15# Branch Display
16.PP
17The branch display output by this command is organized into four
18columns of information; for example:
19.TP
20*P nocolor
21| in repo
22.TP
23repo2
24|
25.PP
26The first column contains a * if the branch is the currently
27checked out branch in any of the specified projects, or a blank
28if no project has the branch checked out.
29.PP
30The second column contains either blank, p or P, depending upon
31the upload status of the branch.
32.IP
33(blank): branch not yet published by repo upload
34.IP
35P: all commits were published by repo upload
36p: only some commits were published by repo upload
37.PP
38The third column contains the branch name.
39.PP
40The fourth column (after the | separator) lists the projects that
41the branch appears in, or does not appear in. If no project list
42is shown, then the branch appears in all projects.
43.SH OPTIONS
44.TP
45\fB\-h\fR, \fB\-\-help\fR
46show this help message and exit
47.TP
48\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
49number of jobs to run in parallel (default: based on
50number of CPU cores)
51.SS Logging options:
52.TP
53\fB\-v\fR, \fB\-\-verbose\fR
54show all output
55.TP
56\fB\-q\fR, \fB\-\-quiet\fR
57only show errors
58.PP
59Run `repo help branches` to view the detailed manual.
diff --git a/man/repo-checkout.1 b/man/repo-checkout.1
new file mode 100644
index 00000000..6dd3e6ca
--- /dev/null
+++ b/man/repo-checkout.1
@@ -0,0 +1,36 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo checkout" "Repo Manual"
3.SH NAME
4repo \- repo checkout - manual page for repo checkout
5.SH SYNOPSIS
6.B repo
7\fI\,checkout <branchname> \/\fR[\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Checkout a branch for development
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
18number of jobs to run in parallel (default: based on
19number of CPU cores)
20.SS Logging options:
21.TP
22\fB\-v\fR, \fB\-\-verbose\fR
23show all output
24.TP
25\fB\-q\fR, \fB\-\-quiet\fR
26only show errors
27.PP
28Run `repo help checkout` to view the detailed manual.
29.SH DETAILS
30.PP
31The 'repo checkout' command checks out an existing branch that was previously
32created by 'repo start'.
33.PP
34The command is equivalent to:
35.IP
36repo forall [<project>...] \fB\-c\fR git checkout <branchname>
diff --git a/man/repo-cherry-pick.1 b/man/repo-cherry-pick.1
new file mode 100644
index 00000000..e7716c55
--- /dev/null
+++ b/man/repo-cherry-pick.1
@@ -0,0 +1,28 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo cherry-pick" "Repo Manual"
3.SH NAME
4repo \- repo cherry-pick - manual page for repo cherry-pick
5.SH SYNOPSIS
6.B repo
7\fI\,cherry-pick <sha1>\/\fR
8.SH DESCRIPTION
9Summary
10.PP
11Cherry\-pick a change.
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.SS Logging options:
17.TP
18\fB\-v\fR, \fB\-\-verbose\fR
19show all output
20.TP
21\fB\-q\fR, \fB\-\-quiet\fR
22only show errors
23.PP
24Run `repo help cherry\-pick` to view the detailed manual.
25.SH DETAILS
26.PP
27\&'repo cherry\-pick' cherry\-picks a change from one branch to another. The change
28id will be updated, and a reference to the old change id will be added.
diff --git a/man/repo-diff.1 b/man/repo-diff.1
new file mode 100644
index 00000000..890f8d22
--- /dev/null
+++ b/man/repo-diff.1
@@ -0,0 +1,35 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo diff" "Repo Manual"
3.SH NAME
4repo \- repo diff - manual page for repo diff
5.SH SYNOPSIS
6.B repo
7\fI\,diff \/\fR[\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Show changes between commit and working tree
12.PP
13The \fB\-u\fR option causes 'repo diff' to generate diff output with file paths
14relative to the repository root, so the output can be applied
15to the Unix 'patch' command.
16.SH OPTIONS
17.TP
18\fB\-h\fR, \fB\-\-help\fR
19show this help message and exit
20.TP
21\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
22number of jobs to run in parallel (default: based on
23number of CPU cores)
24.TP
25\fB\-u\fR, \fB\-\-absolute\fR
26paths are relative to the repository root
27.SS Logging options:
28.TP
29\fB\-v\fR, \fB\-\-verbose\fR
30show all output
31.TP
32\fB\-q\fR, \fB\-\-quiet\fR
33only show errors
34.PP
35Run `repo help diff` to view the detailed manual.
diff --git a/man/repo-diffmanifests.1 b/man/repo-diffmanifests.1
new file mode 100644
index 00000000..add50f17
--- /dev/null
+++ b/man/repo-diffmanifests.1
@@ -0,0 +1,61 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo diffmanifests" "Repo Manual"
3.SH NAME
4repo \- repo diffmanifests - manual page for repo diffmanifests
5.SH SYNOPSIS
6.B repo
7\fI\,diffmanifests manifest1.xml \/\fR[\fI\,manifest2.xml\/\fR] [\fI\,options\/\fR]
8.SH DESCRIPTION
9Summary
10.PP
11Manifest diff utility
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-\-raw\fR
18display raw diff
19.TP
20\fB\-\-no\-color\fR
21does not display the diff in color
22.TP
23\fB\-\-pretty\-format=\fR<FORMAT>
24print the log using a custom git pretty format string
25.SS Logging options:
26.TP
27\fB\-v\fR, \fB\-\-verbose\fR
28show all output
29.TP
30\fB\-q\fR, \fB\-\-quiet\fR
31only show errors
32.PP
33Run `repo help diffmanifests` to view the detailed manual.
34.SH DETAILS
35.PP
36The repo diffmanifests command shows differences between project revisions of
37manifest1 and manifest2. if manifest2 is not specified, current manifest.xml
38will be used instead. Both absolute and relative paths may be used for
39manifests. Relative paths start from project's ".repo/manifests" folder.
40.PP
41The \fB\-\-raw\fR option Displays the diff in a way that facilitates parsing, the
42project pattern will be <status> <path> <revision from> [<revision to>] and the
43commit pattern will be <status> <onelined log> with status values respectively :
44.IP
45A = Added project
46R = Removed project
47C = Changed project
48U = Project with unreachable revision(s) (revision(s) not found)
49.PP
50for project, and
51.IP
52A = Added commit
53R = Removed commit
54.PP
55for a commit.
56.PP
57Only changed projects may contain commits, and commit status always starts with
58a space, and are part of last printed project. Unreachable revisions may occur
59if project is not up to date or if repo has not been initialized with all the
60groups, in which case some projects won't be synced and their revisions won't be
61found.
diff --git a/man/repo-download.1 b/man/repo-download.1
new file mode 100644
index 00000000..cf7f767d
--- /dev/null
+++ b/man/repo-download.1
@@ -0,0 +1,44 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo download" "Repo Manual"
3.SH NAME
4repo \- repo download - manual page for repo download
5.SH SYNOPSIS
6.B repo
7\fI\,download {\/\fR[\fI\,project\/\fR] \fI\,change\/\fR[\fI\,/patchset\/\fR]\fI\,}\/\fR...
8.SH DESCRIPTION
9Summary
10.PP
11Download and checkout a change
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-b\fR BRANCH, \fB\-\-branch\fR=\fI\,BRANCH\/\fR
18create a new branch first
19.TP
20\fB\-c\fR, \fB\-\-cherry\-pick\fR
21cherry\-pick instead of checkout
22.TP
23\fB\-x\fR, \fB\-\-record\-origin\fR
24pass \fB\-x\fR when cherry\-picking
25.TP
26\fB\-r\fR, \fB\-\-revert\fR
27revert instead of checkout
28.TP
29\fB\-f\fR, \fB\-\-ff\-only\fR
30force fast\-forward merge
31.SS Logging options:
32.TP
33\fB\-v\fR, \fB\-\-verbose\fR
34show all output
35.TP
36\fB\-q\fR, \fB\-\-quiet\fR
37only show errors
38.PP
39Run `repo help download` to view the detailed manual.
40.SH DETAILS
41.PP
42The 'repo download' command downloads a change from the review system and makes
43it available in your project's local working directory. If no project is
44specified try to use current directory as a project.
diff --git a/man/repo-forall.1 b/man/repo-forall.1
new file mode 100644
index 00000000..eb2ad57b
--- /dev/null
+++ b/man/repo-forall.1
@@ -0,0 +1,128 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo forall" "Repo Manual"
3.SH NAME
4repo \- repo forall - manual page for repo forall
5.SH SYNOPSIS
6.B repo
7\fI\,forall \/\fR[\fI\,<project>\/\fR...] \fI\,-c <command> \/\fR[\fI\,<arg>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Run a shell command in each project
12.PP
13repo forall \fB\-r\fR str1 [str2] ... \fB\-c\fR <command> [<arg>...]
14.SH OPTIONS
15.TP
16\fB\-h\fR, \fB\-\-help\fR
17show this help message and exit
18.TP
19\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
20number of jobs to run in parallel (default: based on
21number of CPU cores)
22.TP
23\fB\-r\fR, \fB\-\-regex\fR
24execute the command only on projects matching regex or
25wildcard expression
26.TP
27\fB\-i\fR, \fB\-\-inverse\-regex\fR
28execute the command only on projects not matching
29regex or wildcard expression
30.TP
31\fB\-g\fR GROUPS, \fB\-\-groups\fR=\fI\,GROUPS\/\fR
32execute the command only on projects matching the
33specified groups
34.TP
35\fB\-c\fR, \fB\-\-command\fR
36command (and arguments) to execute
37.TP
38\fB\-e\fR, \fB\-\-abort\-on\-errors\fR
39abort if a command exits unsuccessfully
40.TP
41\fB\-\-ignore\-missing\fR
42silently skip & do not exit non\-zero due missing
43checkouts
44.TP
45\fB\-\-interactive\fR
46force interactive usage
47.SS Logging options:
48.TP
49\fB\-v\fR, \fB\-\-verbose\fR
50show all output
51.TP
52\fB\-q\fR, \fB\-\-quiet\fR
53only show errors
54.TP
55\fB\-p\fR
56show project headers before output
57.PP
58Run `repo help forall` to view the detailed manual.
59.SH DETAILS
60.PP
61Executes the same shell command in each project.
62.PP
63The \fB\-r\fR option allows running the command only on projects matching regex or
64wildcard expression.
65.PP
66By default, projects are processed non\-interactively in parallel. If you want to
67run interactive commands, make sure to pass \fB\-\-interactive\fR to force \fB\-\-jobs\fR 1.
68While the processing order of projects is not guaranteed, the order of project
69output is stable.
70.PP
71Output Formatting
72.PP
73The \fB\-p\fR option causes 'repo forall' to bind pipes to the command's stdin, stdout
74and stderr streams, and pipe all output into a continuous stream that is
75displayed in a single pager session. Project headings are inserted before the
76output of each command is displayed. If the command produces no output in a
77project, no heading is displayed.
78.PP
79The formatting convention used by \fB\-p\fR is very suitable for some types of
80searching, e.g. `repo forall \fB\-p\fR \fB\-c\fR git log \fB\-SFoo\fR` will print all commits that
81add or remove references to Foo.
82.PP
83The \fB\-v\fR option causes 'repo forall' to display stderr messages if a command
84produces output only on stderr. Normally the \fB\-p\fR option causes command output to
85be suppressed until the command produces at least one byte of output on stdout.
86.PP
87Environment
88.PP
89pwd is the project's working directory. If the current client is a mirror
90client, then pwd is the Git repository.
91.PP
92REPO_PROJECT is set to the unique name of the project.
93.PP
94REPO_PATH is the path relative the the root of the client.
95.PP
96REPO_REMOTE is the name of the remote system from the manifest.
97.PP
98REPO_LREV is the name of the revision from the manifest, translated to a local
99tracking branch. If you need to pass the manifest revision to a locally executed
100git command, use REPO_LREV.
101.PP
102REPO_RREV is the name of the revision from the manifest, exactly as written in
103the manifest.
104.PP
105REPO_COUNT is the total number of projects being iterated.
106.PP
107REPO_I is the current (1\-based) iteration count. Can be used in conjunction with
108REPO_COUNT to add a simple progress indicator to your command.
109.PP
110REPO__* are any extra environment variables, specified by the "annotation"
111element under any project element. This can be useful for differentiating trees
112based on user\-specific criteria, or simply annotating tree details.
113.PP
114shell positional arguments ($1, $2, .., $#) are set to any arguments following
115<command>.
116.PP
117Example: to list projects:
118.IP
119repo forall \fB\-c\fR 'echo $REPO_PROJECT'
120.PP
121Notice that $REPO_PROJECT is quoted to ensure it is expanded in the context of
122running <command> instead of in the calling shell.
123.PP
124Unless \fB\-p\fR is used, stdin, stdout, stderr are inherited from the terminal and are
125not redirected.
126.PP
127If \fB\-e\fR is used, when a command exits unsuccessfully, 'repo forall' will abort
128without iterating through the remaining projects.
diff --git a/man/repo-gitc-delete.1 b/man/repo-gitc-delete.1
new file mode 100644
index 00000000..c84c6e45
--- /dev/null
+++ b/man/repo-gitc-delete.1
@@ -0,0 +1,31 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo gitc-delete" "Repo Manual"
3.SH NAME
4repo \- repo gitc-delete - manual page for repo gitc-delete
5.SH SYNOPSIS
6.B repo
7\fI\,gitc-delete\/\fR
8.SH DESCRIPTION
9Summary
10.PP
11Delete a GITC Client.
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-f\fR, \fB\-\-force\fR
18force the deletion (no prompt)
19.SS Logging options:
20.TP
21\fB\-v\fR, \fB\-\-verbose\fR
22show all output
23.TP
24\fB\-q\fR, \fB\-\-quiet\fR
25only show errors
26.PP
27Run `repo help gitc\-delete` to view the detailed manual.
28.SH DETAILS
29.PP
30This subcommand deletes the current GITC client, deleting the GITC manifest and
31all locally downloaded sources.
diff --git a/man/repo-gitc-init.1 b/man/repo-gitc-init.1
new file mode 100644
index 00000000..9b61866e
--- /dev/null
+++ b/man/repo-gitc-init.1
@@ -0,0 +1,150 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "September 2021" "repo gitc-init" "Repo Manual"
3.SH NAME
4repo \- repo gitc-init - manual page for repo gitc-init
5.SH SYNOPSIS
6.B repo
7\fI\,gitc-init \/\fR[\fI\,options\/\fR] [\fI\,client name\/\fR]
8.SH DESCRIPTION
9Summary
10.PP
11Initialize a GITC Client.
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.SS Logging options:
17.TP
18\fB\-v\fR, \fB\-\-verbose\fR
19show all output
20.TP
21\fB\-q\fR, \fB\-\-quiet\fR
22only show errors
23.SS Manifest options:
24.TP
25\fB\-u\fR URL, \fB\-\-manifest\-url\fR=\fI\,URL\/\fR
26manifest repository location
27.TP
28\fB\-b\fR REVISION, \fB\-\-manifest\-branch\fR=\fI\,REVISION\/\fR
29manifest branch or revision (use HEAD for default)
30.TP
31\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
32initial manifest file
33.TP
34\fB\-\-standalone\-manifest\fR
35download the manifest as a static file rather then
36create a git checkout of the manifest repo
37.TP
38\fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
39restrict manifest projects to ones with specified
40group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
41.TP
42\fB\-p\fR PLATFORM, \fB\-\-platform\fR=\fI\,PLATFORM\/\fR
43restrict manifest projects to ones with a specified
44platform group [auto|all|none|linux|darwin|...]
45.TP
46\fB\-\-submodules\fR
47sync any submodules associated with the manifest repo
48.SS Manifest (only) checkout options:
49.TP
50\fB\-\-current\-branch\fR
51fetch only current manifest branch from server
52.TP
53\fB\-\-no\-current\-branch\fR
54fetch all manifest branches from server
55.TP
56\fB\-\-tags\fR
57fetch tags in the manifest
58.TP
59\fB\-\-no\-tags\fR
60don't fetch tags in the manifest
61.SS Checkout modes:
62.TP
63\fB\-\-mirror\fR
64create a replica of the remote repositories rather
65than a client working directory
66.TP
67\fB\-\-archive\fR
68checkout an archive instead of a git repository for
69each project. See git archive.
70.TP
71\fB\-\-worktree\fR
72use git\-worktree to manage projects
73.SS Project checkout optimizations:
74.TP
75\fB\-\-reference\fR=\fI\,DIR\/\fR
76location of mirror directory
77.TP
78\fB\-\-dissociate\fR
79dissociate from reference mirrors after clone
80.TP
81\fB\-\-depth\fR=\fI\,DEPTH\/\fR
82create a shallow clone with given depth; see git clone
83.TP
84\fB\-\-partial\-clone\fR
85perform partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
86.TP
87\fB\-\-no\-partial\-clone\fR
88disable use of partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
89.TP
90\fB\-\-partial\-clone\-exclude\fR=\fI\,PARTIAL_CLONE_EXCLUDE\/\fR
91exclude the specified projects (a comma\-delimited
92project names) from partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
93.TP
94\fB\-\-clone\-filter\fR=\fI\,CLONE_FILTER\/\fR
95filter for use with \fB\-\-partial\-clone\fR [default:
96blob:none]
97.TP
98\fB\-\-use\-superproject\fR
99use the manifest superproject to sync projects
100.TP
101\fB\-\-no\-use\-superproject\fR
102disable use of manifest superprojects
103.TP
104\fB\-\-clone\-bundle\fR
105enable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
106not \fB\-\-partial\-clone\fR)
107.TP
108\fB\-\-no\-clone\-bundle\fR
109disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
110\fB\-\-partial\-clone\fR)
111.SS repo Version options:
112.TP
113\fB\-\-repo\-url\fR=\fI\,URL\/\fR
114repo repository location ($REPO_URL)
115.TP
116\fB\-\-repo\-rev\fR=\fI\,REV\/\fR
117repo branch or revision ($REPO_REV)
118.TP
119\fB\-\-no\-repo\-verify\fR
120do not verify repo source code
121.SS Other options:
122.TP
123\fB\-\-config\-name\fR
124Always prompt for name/e\-mail
125.SS GITC options:
126.TP
127\fB\-f\fR MANIFEST_FILE, \fB\-\-manifest\-file\fR=\fI\,MANIFEST_FILE\/\fR
128Optional manifest file to use for this GITC client.
129.TP
130\fB\-c\fR GITC_CLIENT, \fB\-\-gitc\-client\fR=\fI\,GITC_CLIENT\/\fR
131Name of the gitc_client instance to create or modify.
132.PP
133Run `repo help gitc\-init` to view the detailed manual.
134.SH DETAILS
135.PP
136The 'repo gitc\-init' command is ran to initialize a new GITC client for use with
137the GITC file system.
138.PP
139This command will setup the client directory, initialize repo, just like repo
140init does, and then downloads the manifest collection and installs it in the
141\&.repo/directory of the GITC client.
142.PP
143Once this is done, a GITC manifest is generated by pulling the HEAD SHA for each
144project and generates the properly formatted XML file and installs it as
145\&.manifest in the GITC client directory.
146.PP
147The \fB\-c\fR argument is required to specify the GITC client name.
148.PP
149The optional \fB\-f\fR argument can be used to specify the manifest file to use for
150this GITC client.
diff --git a/man/repo-grep.1 b/man/repo-grep.1
new file mode 100644
index 00000000..be410588
--- /dev/null
+++ b/man/repo-grep.1
@@ -0,0 +1,119 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo grep" "Repo Manual"
3.SH NAME
4repo \- repo grep - manual page for repo grep
5.SH SYNOPSIS
6.B repo
7\fI\,grep {pattern | -e pattern} \/\fR[\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Print lines matching a pattern
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
18number of jobs to run in parallel (default: based on
19number of CPU cores)
20.SS Logging options:
21.TP
22\fB\-\-verbose\fR
23show all output
24.TP
25\fB\-q\fR, \fB\-\-quiet\fR
26only show errors
27.SS Sources:
28.TP
29\fB\-\-cached\fR
30Search the index, instead of the work tree
31.TP
32\fB\-r\fR TREEish, \fB\-\-revision\fR=\fI\,TREEish\/\fR
33Search TREEish, instead of the work tree
34.SS Pattern:
35.TP
36\fB\-e\fR PATTERN
37Pattern to search for
38.TP
39\fB\-i\fR, \fB\-\-ignore\-case\fR
40Ignore case differences
41.TP
42\fB\-a\fR, \fB\-\-text\fR
43Process binary files as if they were text
44.TP
45\fB\-I\fR
46Don't match the pattern in binary files
47.TP
48\fB\-w\fR, \fB\-\-word\-regexp\fR
49Match the pattern only at word boundaries
50.TP
51\fB\-v\fR, \fB\-\-invert\-match\fR
52Select non\-matching lines
53.TP
54\fB\-G\fR, \fB\-\-basic\-regexp\fR
55Use POSIX basic regexp for patterns (default)
56.TP
57\fB\-E\fR, \fB\-\-extended\-regexp\fR
58Use POSIX extended regexp for patterns
59.TP
60\fB\-F\fR, \fB\-\-fixed\-strings\fR
61Use fixed strings (not regexp) for pattern
62.SS Pattern Grouping:
63.TP
64\fB\-\-all\-match\fR
65Limit match to lines that have all patterns
66.TP
67\fB\-\-and\fR, \fB\-\-or\fR, \fB\-\-not\fR
68Boolean operators to combine patterns
69.TP
70\-(, \-)
71Boolean operator grouping
72.SS Output:
73.TP
74\fB\-n\fR
75Prefix the line number to matching lines
76.TP
77\fB\-C\fR CONTEXT
78Show CONTEXT lines around match
79.TP
80\fB\-B\fR CONTEXT
81Show CONTEXT lines before match
82.TP
83\fB\-A\fR CONTEXT
84Show CONTEXT lines after match
85.TP
86\fB\-l\fR, \fB\-\-name\-only\fR, \fB\-\-files\-with\-matches\fR
87Show only file names containing matching lines
88.TP
89\fB\-L\fR, \fB\-\-files\-without\-match\fR
90Show only file names not containing matching lines
91.PP
92Run `repo help grep` to view the detailed manual.
93.SH DETAILS
94.PP
95Search for the specified patterns in all project files.
96.PP
97Boolean Options
98.PP
99The following options can appear as often as necessary to express the pattern to
100locate:
101.HP
102\fB\-e\fR PATTERN
103.HP
104\fB\-\-and\fR, \fB\-\-or\fR, \fB\-\-not\fR, \-(, \-)
105.PP
106Further, the \fB\-r\fR/\-\-revision option may be specified multiple times in order to
107scan multiple trees. If the same file matches in more than one tree, only the
108first result is reported, prefixed by the revision name it was found under.
109.PP
110Examples
111.PP
112Look for a line that has '#define' and either 'MAX_PATH or 'PATH_MAX':
113.IP
114repo grep \fB\-e\fR '#define' \fB\-\-and\fR \-\e( \fB\-e\fR MAX_PATH \fB\-e\fR PATH_MAX \e)
115.PP
116Look for a line that has 'NODE' or 'Unexpected' in files that contain a line
117that matches both expressions:
118.IP
119repo grep \fB\-\-all\-match\fR \fB\-e\fR NODE \fB\-e\fR Unexpected
diff --git a/man/repo-help.1 b/man/repo-help.1
new file mode 100644
index 00000000..d6da3c51
--- /dev/null
+++ b/man/repo-help.1
@@ -0,0 +1,33 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo help" "Repo Manual"
3.SH NAME
4repo \- repo help - manual page for repo help
5.SH SYNOPSIS
6.B repo
7\fI\,help \/\fR[\fI\,--all|command\/\fR]
8.SH DESCRIPTION
9Summary
10.PP
11Display detailed help on a command
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-a\fR, \fB\-\-all\fR
18show the complete list of commands
19.TP
20\fB\-\-help\-all\fR
21show the \fB\-\-help\fR of all commands
22.SS Logging options:
23.TP
24\fB\-v\fR, \fB\-\-verbose\fR
25show all output
26.TP
27\fB\-q\fR, \fB\-\-quiet\fR
28only show errors
29.PP
30Run `repo help help` to view the detailed manual.
31.SH DETAILS
32.PP
33Displays detailed usage information about a command.
diff --git a/man/repo-info.1 b/man/repo-info.1
new file mode 100644
index 00000000..cf7c17b8
--- /dev/null
+++ b/man/repo-info.1
@@ -0,0 +1,40 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo info" "Repo Manual"
3.SH NAME
4repo \- repo info - manual page for repo info
5.SH SYNOPSIS
6.B repo
7\fI\,info \/\fR[\fI\,-dl\/\fR] [\fI\,-o \/\fR[\fI\,-c\/\fR]] [\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Get info on the manifest branch, current branch or unmerged branches
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-d\fR, \fB\-\-diff\fR
18show full info and commit diff including remote
19branches
20.TP
21\fB\-o\fR, \fB\-\-overview\fR
22show overview of all local commits
23.TP
24\fB\-c\fR, \fB\-\-current\-branch\fR
25consider only checked out branches
26.TP
27\fB\-\-no\-current\-branch\fR
28consider all local branches
29.TP
30\fB\-l\fR, \fB\-\-local\-only\fR
31disable all remote operations
32.SS Logging options:
33.TP
34\fB\-v\fR, \fB\-\-verbose\fR
35show all output
36.TP
37\fB\-q\fR, \fB\-\-quiet\fR
38only show errors
39.PP
40Run `repo help info` to view the detailed manual.
diff --git a/man/repo-init.1 b/man/repo-init.1
new file mode 100644
index 00000000..9957b64d
--- /dev/null
+++ b/man/repo-init.1
@@ -0,0 +1,170 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "September 2021" "repo init" "Repo Manual"
3.SH NAME
4repo \- repo init - manual page for repo init
5.SH SYNOPSIS
6.B repo
7\fI\,init \/\fR[\fI\,options\/\fR] [\fI\,manifest url\/\fR]
8.SH DESCRIPTION
9Summary
10.PP
11Initialize a repo client checkout in the current directory
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.SS Logging options:
17.TP
18\fB\-v\fR, \fB\-\-verbose\fR
19show all output
20.TP
21\fB\-q\fR, \fB\-\-quiet\fR
22only show errors
23.SS Manifest options:
24.TP
25\fB\-u\fR URL, \fB\-\-manifest\-url\fR=\fI\,URL\/\fR
26manifest repository location
27.TP
28\fB\-b\fR REVISION, \fB\-\-manifest\-branch\fR=\fI\,REVISION\/\fR
29manifest branch or revision (use HEAD for default)
30.TP
31\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
32initial manifest file
33.TP
34\fB\-\-standalone\-manifest\fR
35download the manifest as a static file rather then
36create a git checkout of the manifest repo
37.TP
38\fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
39restrict manifest projects to ones with specified
40group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
41.TP
42\fB\-p\fR PLATFORM, \fB\-\-platform\fR=\fI\,PLATFORM\/\fR
43restrict manifest projects to ones with a specified
44platform group [auto|all|none|linux|darwin|...]
45.TP
46\fB\-\-submodules\fR
47sync any submodules associated with the manifest repo
48.SS Manifest (only) checkout options:
49.TP
50\fB\-c\fR, \fB\-\-current\-branch\fR
51fetch only current manifest branch from server
52.TP
53\fB\-\-no\-current\-branch\fR
54fetch all manifest branches from server
55.TP
56\fB\-\-tags\fR
57fetch tags in the manifest
58.TP
59\fB\-\-no\-tags\fR
60don't fetch tags in the manifest
61.SS Checkout modes:
62.TP
63\fB\-\-mirror\fR
64create a replica of the remote repositories rather
65than a client working directory
66.TP
67\fB\-\-archive\fR
68checkout an archive instead of a git repository for
69each project. See git archive.
70.TP
71\fB\-\-worktree\fR
72use git\-worktree to manage projects
73.SS Project checkout optimizations:
74.TP
75\fB\-\-reference\fR=\fI\,DIR\/\fR
76location of mirror directory
77.TP
78\fB\-\-dissociate\fR
79dissociate from reference mirrors after clone
80.TP
81\fB\-\-depth\fR=\fI\,DEPTH\/\fR
82create a shallow clone with given depth; see git clone
83.TP
84\fB\-\-partial\-clone\fR
85perform partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
86.TP
87\fB\-\-no\-partial\-clone\fR
88disable use of partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
89.TP
90\fB\-\-partial\-clone\-exclude\fR=\fI\,PARTIAL_CLONE_EXCLUDE\/\fR
91exclude the specified projects (a comma\-delimited
92project names) from partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
93.TP
94\fB\-\-clone\-filter\fR=\fI\,CLONE_FILTER\/\fR
95filter for use with \fB\-\-partial\-clone\fR [default:
96blob:none]
97.TP
98\fB\-\-use\-superproject\fR
99use the manifest superproject to sync projects
100.TP
101\fB\-\-no\-use\-superproject\fR
102disable use of manifest superprojects
103.TP
104\fB\-\-clone\-bundle\fR
105enable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
106not \fB\-\-partial\-clone\fR)
107.TP
108\fB\-\-no\-clone\-bundle\fR
109disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
110\fB\-\-partial\-clone\fR)
111.SS repo Version options:
112.TP
113\fB\-\-repo\-url\fR=\fI\,URL\/\fR
114repo repository location ($REPO_URL)
115.TP
116\fB\-\-repo\-rev\fR=\fI\,REV\/\fR
117repo branch or revision ($REPO_REV)
118.TP
119\fB\-\-no\-repo\-verify\fR
120do not verify repo source code
121.SS Other options:
122.TP
123\fB\-\-config\-name\fR
124Always prompt for name/e\-mail
125.PP
126Run `repo help init` to view the detailed manual.
127.SH DETAILS
128.PP
129The 'repo init' command is run once to install and initialize repo. The latest
130repo source code and manifest collection is downloaded from the server and is
131installed in the .repo/ directory in the current working directory.
132.PP
133When creating a new checkout, the manifest URL is the only required setting. It
134may be specified using the \fB\-\-manifest\-url\fR option, or as the first optional
135argument.
136.PP
137The optional \fB\-b\fR argument can be used to select the manifest branch to checkout
138and use. If no branch is specified, the remote's default branch is used. This is
139equivalent to using \fB\-b\fR HEAD.
140.PP
141The optional \fB\-m\fR argument can be used to specify an alternate manifest to be
142used. If no manifest is specified, the manifest default.xml will be used.
143.PP
144If the \fB\-\-standalone\-manifest\fR argument is set, the manifest will be downloaded
145directly from the specified \fB\-\-manifest\-url\fR as a static file (rather than setting
146up a manifest git checkout). With \fB\-\-standalone\-manifest\fR, the manifest will be
147fully static and will not be re\-downloaded during subsesquent `repo init` and
148`repo sync` calls.
149.PP
150The \fB\-\-reference\fR option can be used to point to a directory that has the content
151of a \fB\-\-mirror\fR sync. This will make the working directory use as much data as
152possible from the local reference directory when fetching from the server. This
153will make the sync go a lot faster by reducing data traffic on the network.
154.PP
155The \fB\-\-dissociate\fR option can be used to borrow the objects from the directory
156specified with the \fB\-\-reference\fR option only to reduce network transfer, and stop
157borrowing from them after a first clone is made by making necessary local copies
158of borrowed objects.
159.PP
160The \fB\-\-no\-clone\-bundle\fR option disables any attempt to use \fI\,$URL/clone.bundle\/\fP to
161bootstrap a new Git repository from a resumeable bundle file on a content
162delivery network. This may be necessary if there are problems with the local
163Python HTTP client or proxy configuration, but the Git binary works.
164.PP
165Switching Manifest Branches
166.PP
167To switch to another manifest branch, `repo init \fB\-b\fR otherbranch` may be used in
168an existing client. However, as this only updates the manifest, a subsequent
169`repo sync` (or `repo sync \fB\-d\fR`) is necessary to update the working directory
170files.
diff --git a/man/repo-list.1 b/man/repo-list.1
new file mode 100644
index 00000000..7f85e612
--- /dev/null
+++ b/man/repo-list.1
@@ -0,0 +1,61 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo list" "Repo Manual"
3.SH NAME
4repo \- repo list - manual page for repo list
5.SH SYNOPSIS
6.B repo
7\fI\,list \/\fR[\fI\,-f\/\fR] [\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11List projects and their associated directories
12.PP
13repo list [\-f] \fB\-r\fR str1 [str2]...
14.SH OPTIONS
15.TP
16\fB\-h\fR, \fB\-\-help\fR
17show this help message and exit
18.TP
19\fB\-r\fR, \fB\-\-regex\fR
20filter the project list based on regex or wildcard
21matching of strings
22.TP
23\fB\-g\fR GROUPS, \fB\-\-groups\fR=\fI\,GROUPS\/\fR
24filter the project list based on the groups the
25project is in
26.TP
27\fB\-a\fR, \fB\-\-all\fR
28show projects regardless of checkout state
29.TP
30\fB\-n\fR, \fB\-\-name\-only\fR
31display only the name of the repository
32.TP
33\fB\-p\fR, \fB\-\-path\-only\fR
34display only the path of the repository
35.TP
36\fB\-f\fR, \fB\-\-fullpath\fR
37display the full work tree path instead of the
38relative path
39.TP
40\fB\-\-relative\-to\fR=\fI\,PATH\/\fR
41display paths relative to this one (default: top of
42repo client checkout)
43.SS Logging options:
44.TP
45\fB\-v\fR, \fB\-\-verbose\fR
46show all output
47.TP
48\fB\-q\fR, \fB\-\-quiet\fR
49only show errors
50.PP
51Run `repo help list` to view the detailed manual.
52.SH DETAILS
53.PP
54List all projects; pass '.' to list the project for the cwd.
55.PP
56By default, only projects that currently exist in the checkout are shown. If you
57want to list all projects (using the specified filter settings), use the \fB\-\-all\fR
58option. If you want to show all projects regardless of the manifest groups, then
59also pass \fB\-\-groups\fR all.
60.PP
61This is similar to running: repo forall \fB\-c\fR 'echo "$REPO_PATH : $REPO_PROJECT"'.
diff --git a/man/repo-manifest.1 b/man/repo-manifest.1
new file mode 100644
index 00000000..be467607
--- /dev/null
+++ b/man/repo-manifest.1
@@ -0,0 +1,548 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo manifest" "Repo Manual"
3.SH NAME
4repo \- repo manifest - manual page for repo manifest
5.SH SYNOPSIS
6.B repo
7\fI\,manifest \/\fR[\fI\,-o {-|NAME.xml}\/\fR] [\fI\,-m MANIFEST.xml\/\fR] [\fI\,-r\/\fR]
8.SH DESCRIPTION
9Summary
10.PP
11Manifest inspection utility
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-r\fR, \fB\-\-revision\-as\-HEAD\fR
18save revisions as current HEAD
19.TP
20\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
21temporary manifest to use for this sync
22.TP
23\fB\-\-suppress\-upstream\-revision\fR
24if in \fB\-r\fR mode, do not write the upstream field (only
25of use if the branch names for a sha1 manifest are
26sensitive)
27.TP
28\fB\-\-suppress\-dest\-branch\fR
29if in \fB\-r\fR mode, do not write the dest\-branch field
30(only of use if the branch names for a sha1 manifest
31are sensitive)
32.TP
33\fB\-\-json\fR
34output manifest in JSON format (experimental)
35.TP
36\fB\-\-pretty\fR
37format output for humans to read
38.TP
39\fB\-\-no\-local\-manifests\fR
40ignore local manifests
41.TP
42\fB\-o\fR \-|NAME.xml, \fB\-\-output\-file\fR=\fI\,\-\/\fR|NAME.xml
43file to save the manifest to
44.SS Logging options:
45.TP
46\fB\-v\fR, \fB\-\-verbose\fR
47show all output
48.TP
49\fB\-q\fR, \fB\-\-quiet\fR
50only show errors
51.PP
52Run `repo help manifest` to view the detailed manual.
53.SH DETAILS
54.PP
55With the \fB\-o\fR option, exports the current manifest for inspection. The manifest
56and (if present) local_manifests/ are combined together to produce a single
57manifest file. This file can be stored in a Git repository for use during future
58\&'repo init' invocations.
59.PP
60The \fB\-r\fR option can be used to generate a manifest file with project revisions set
61to the current commit hash. These are known as "revision locked manifests", as
62they don't follow a particular branch. In this case, the 'upstream' attribute is
63set to the ref we were on when the manifest was generated. The 'dest\-branch'
64attribute is set to indicate the remote ref to push changes to via 'repo
65upload'.
66.PP
67repo Manifest Format
68.PP
69A repo manifest describes the structure of a repo client; that is the
70directories that are visible and where they should be obtained from with git.
71.PP
72The basic structure of a manifest is a bare Git repository holding a single
73`default.xml` XML file in the top level directory.
74.PP
75Manifests are inherently version controlled, since they are kept within a Git
76repository. Updates to manifests are automatically obtained by clients during
77`repo sync`.
78.PP
79[TOC]
80.PP
81XML File Format
82.PP
83A manifest XML file (e.g. `default.xml`) roughly conforms to the following DTD:
84.PP
85```xml <!DOCTYPE manifest [
86.TP
87<!ELEMENT manifest (notice?,
88remote*,
89default?,
90manifest\-server?,
91remove\-project*,
92project*,
93extend\-project*,
94repo\-hooks?,
95superproject?,
96contactinfo?,
97include*)>
98.IP
99<!ELEMENT notice (#PCDATA)>
100.IP
101<!ELEMENT remote (annotation*)>
102<!ATTLIST remote name ID #REQUIRED>
103<!ATTLIST remote alias CDATA #IMPLIED>
104<!ATTLIST remote fetch CDATA #REQUIRED>
105<!ATTLIST remote pushurl CDATA #IMPLIED>
106<!ATTLIST remote review CDATA #IMPLIED>
107<!ATTLIST remote revision CDATA #IMPLIED>
108.IP
109<!ELEMENT default EMPTY>
110<!ATTLIST default remote IDREF #IMPLIED>
111<!ATTLIST default revision CDATA #IMPLIED>
112<!ATTLIST default dest\-branch CDATA #IMPLIED>
113<!ATTLIST default upstream CDATA #IMPLIED>
114<!ATTLIST default sync\-j CDATA #IMPLIED>
115<!ATTLIST default sync\-c CDATA #IMPLIED>
116<!ATTLIST default sync\-s CDATA #IMPLIED>
117<!ATTLIST default sync\-tags CDATA #IMPLIED>
118.IP
119<!ELEMENT manifest\-server EMPTY>
120<!ATTLIST manifest\-server url CDATA #REQUIRED>
121.TP
122<!ELEMENT project (annotation*,
123project*,
124copyfile*,
125linkfile*)>
126.TP
127<!ATTLIST project name
128CDATA #REQUIRED>
129.TP
130<!ATTLIST project path
131CDATA #IMPLIED>
132.TP
133<!ATTLIST project remote
134IDREF #IMPLIED>
135.TP
136<!ATTLIST project revision
137CDATA #IMPLIED>
138.IP
139<!ATTLIST project dest\-branch CDATA #IMPLIED>
140<!ATTLIST project groups CDATA #IMPLIED>
141<!ATTLIST project sync\-c CDATA #IMPLIED>
142<!ATTLIST project sync\-s CDATA #IMPLIED>
143<!ATTLIST project sync\-tags CDATA #IMPLIED>
144<!ATTLIST project upstream CDATA #IMPLIED>
145<!ATTLIST project clone\-depth CDATA #IMPLIED>
146<!ATTLIST project force\-path CDATA #IMPLIED>
147.IP
148<!ELEMENT annotation EMPTY>
149<!ATTLIST annotation name CDATA #REQUIRED>
150<!ATTLIST annotation value CDATA #REQUIRED>
151<!ATTLIST annotation keep CDATA "true">
152.IP
153<!ELEMENT copyfile EMPTY>
154<!ATTLIST copyfile src CDATA #REQUIRED>
155<!ATTLIST copyfile dest CDATA #REQUIRED>
156.IP
157<!ELEMENT linkfile EMPTY>
158<!ATTLIST linkfile src CDATA #REQUIRED>
159<!ATTLIST linkfile dest CDATA #REQUIRED>
160.IP
161<!ELEMENT extend\-project EMPTY>
162<!ATTLIST extend\-project name CDATA #REQUIRED>
163<!ATTLIST extend\-project path CDATA #IMPLIED>
164<!ATTLIST extend\-project groups CDATA #IMPLIED>
165<!ATTLIST extend\-project revision CDATA #IMPLIED>
166<!ATTLIST extend\-project remote CDATA #IMPLIED>
167.IP
168<!ELEMENT remove\-project EMPTY>
169<!ATTLIST remove\-project name CDATA #REQUIRED>
170<!ATTLIST remove\-project optional CDATA #IMPLIED>
171.IP
172<!ELEMENT repo\-hooks EMPTY>
173<!ATTLIST repo\-hooks in\-project CDATA #REQUIRED>
174<!ATTLIST repo\-hooks enabled\-list CDATA #REQUIRED>
175.IP
176<!ELEMENT superproject EMPTY>
177<!ATTLIST superproject name CDATA #REQUIRED>
178<!ATTLIST superproject remote IDREF #IMPLIED>
179.IP
180<!ELEMENT contactinfo EMPTY>
181<!ATTLIST contactinfo bugurl CDATA #REQUIRED>
182.IP
183<!ELEMENT include EMPTY>
184<!ATTLIST include name CDATA #REQUIRED>
185<!ATTLIST include groups CDATA #IMPLIED>
186.PP
187]>
188```
189.PP
190For compatibility purposes across repo releases, all unknown elements are
191silently ignored. However, repo reserves all possible names for itself for
192future use. If you want to use custom elements, the `x\-*` namespace is reserved
193for that purpose, and repo guarantees to never allocate any corresponding names.
194.PP
195A description of the elements and their attributes follows.
196.PP
197Element manifest
198.PP
199The root element of the file.
200.PP
201Element notice
202.PP
203Arbitrary text that is displayed to users whenever `repo sync` finishes. The
204content is simply passed through as it exists in the manifest.
205.PP
206Element remote
207.PP
208One or more remote elements may be specified. Each remote element specifies a
209Git URL shared by one or more projects and (optionally) the Gerrit review server
210those projects upload changes through.
211.PP
212Attribute `name`: A short name unique to this manifest file. The name specified
213here is used as the remote name in each project's .git/config, and is therefore
214automatically available to commands like `git fetch`, `git remote`, `git pull`
215and `git push`.
216.PP
217Attribute `alias`: The alias, if specified, is used to override `name` to be set
218as the remote name in each project's .git/config. Its value can be duplicated
219while attribute `name` has to be unique in the manifest file. This helps each
220project to be able to have same remote name which actually points to different
221remote url.
222.PP
223Attribute `fetch`: The Git URL prefix for all projects which use this remote.
224Each project's name is appended to this prefix to form the actual URL used to
225clone the project.
226.PP
227Attribute `pushurl`: The Git "push" URL prefix for all projects which use this
228remote. Each project's name is appended to this prefix to form the actual URL
229used to "git push" the project. This attribute is optional; if not specified
230then "git push" will use the same URL as the `fetch` attribute.
231.PP
232Attribute `review`: Hostname of the Gerrit server where reviews are uploaded to
233by `repo upload`. This attribute is optional; if not specified then `repo
234upload` will not function.
235.PP
236Attribute `revision`: Name of a Git branch (e.g. `main` or `refs/heads/main`).
237Remotes with their own revision will override the default revision.
238.PP
239Element default
240.PP
241At most one default element may be specified. Its remote and revision attributes
242are used when a project element does not specify its own remote or revision
243attribute.
244.PP
245Attribute `remote`: Name of a previously defined remote element. Project
246elements lacking a remote attribute of their own will use this remote.
247.PP
248Attribute `revision`: Name of a Git branch (e.g. `main` or `refs/heads/main`).
249Project elements lacking their own revision attribute will use this revision.
250.PP
251Attribute `dest\-branch`: Name of a Git branch (e.g. `main`). Project elements
252not setting their own `dest\-branch` will inherit this value. If this value is
253not set, projects will use `revision` by default instead.
254.PP
255Attribute `upstream`: Name of the Git ref in which a sha1 can be found. Used
256when syncing a revision locked manifest in \fB\-c\fR mode to avoid having to sync the
257entire ref space. Project elements not setting their own `upstream` will inherit
258this value.
259.PP
260Attribute `sync\-j`: Number of parallel jobs to use when synching.
261.PP
262Attribute `sync\-c`: Set to true to only sync the given Git branch (specified in
263the `revision` attribute) rather than the whole ref space. Project elements
264lacking a sync\-c element of their own will use this value.
265.PP
266Attribute `sync\-s`: Set to true to also sync sub\-projects.
267.PP
268Attribute `sync\-tags`: Set to false to only sync the given Git branch (specified
269in the `revision` attribute) rather than the other ref tags.
270.PP
271Element manifest\-server
272.PP
273At most one manifest\-server may be specified. The url attribute is used to
274specify the URL of a manifest server, which is an XML RPC service.
275.PP
276The manifest server should implement the following RPC methods:
277.IP
278GetApprovedManifest(branch, target)
279.PP
280Return a manifest in which each project is pegged to a known good revision for
281the current branch and target. This is used by repo sync when the \fB\-\-smart\-sync\fR
282option is given.
283.PP
284The target to use is defined by environment variables TARGET_PRODUCT and
285TARGET_BUILD_VARIANT. These variables are used to create a string of the form
286$TARGET_PRODUCT\-$TARGET_BUILD_VARIANT, e.g. passion\-userdebug. If one of those
287variables or both are not present, the program will call GetApprovedManifest
288without the target parameter and the manifest server should choose a reasonable
289default target.
290.IP
291GetManifest(tag)
292.PP
293Return a manifest in which each project is pegged to the revision at the
294specified tag. This is used by repo sync when the \fB\-\-smart\-tag\fR option is given.
295.PP
296Element project
297.PP
298One or more project elements may be specified. Each element describes a single
299Git repository to be cloned into the repo client workspace. You may specify
300Git\-submodules by creating a nested project. Git\-submodules will be
301automatically recognized and inherit their parent's attributes, but those may be
302overridden by an explicitly specified project element.
303.PP
304Attribute `name`: A unique name for this project. The project's name is appended
305onto its remote's fetch URL to generate the actual URL to configure the Git
306remote with. The URL gets formed as:
307.IP
308${remote_fetch}/${project_name}.git
309.PP
310where ${remote_fetch} is the remote's fetch attribute and ${project_name} is the
311project's name attribute. The suffix ".git" is always appended as repo assumes
312the upstream is a forest of bare Git repositories. If the project has a parent
313element, its name will be prefixed by the parent's.
314.PP
315The project name must match the name Gerrit knows, if Gerrit is being used for
316code reviews.
317.PP
318"name" must not be empty, and may not be an absolute path or use "." or ".."
319path components. It is always interpreted relative to the remote's fetch
320settings, so if a different base path is needed, declare a different remote with
321the new settings needed. These restrictions are not enforced for [Local
322Manifests].
323.PP
324Attribute `path`: An optional path relative to the top directory of the repo
325client where the Git working directory for this project should be placed. If not
326supplied the project "name" is used. If the project has a parent element, its
327path will be prefixed by the parent's.
328.PP
329"path" may not be an absolute path or use "." or ".." path components. These
330restrictions are not enforced for [Local Manifests].
331.PP
332If you want to place files into the root of the checkout (e.g. a README or
333Makefile or another build script), use the [copyfile] or [linkfile] elements
334instead.
335.PP
336Attribute `remote`: Name of a previously defined remote element. If not supplied
337the remote given by the default element is used.
338.PP
339Attribute `revision`: Name of the Git branch the manifest wants to track for
340this project. Names can be relative to refs/heads (e.g. just "main") or absolute
341(e.g. "refs/heads/main"). Tags and/or explicit SHA\-1s should work in theory, but
342have not been extensively tested. If not supplied the revision given by the
343remote element is used if applicable, else the default element is used.
344.PP
345Attribute `dest\-branch`: Name of a Git branch (e.g. `main`). When using `repo
346upload`, changes will be submitted for code review on this branch. If
347unspecified both here and in the default element, `revision` is used instead.
348.PP
349Attribute `groups`: List of groups to which this project belongs, whitespace or
350comma separated. All projects belong to the group "all", and each project
351automatically belongs to a group of its name:`name` and path:`path`. E.g. for
352`<project name="monkeys" path="barrel\-of"/>`, that project definition is
353implicitly in the following manifest groups: default, name:monkeys, and
354path:barrel\-of. If you place a project in the group "notdefault", it will not be
355automatically downloaded by repo. If the project has a parent element, the
356`name` and `path` here are the prefixed ones.
357.PP
358Attribute `sync\-c`: Set to true to only sync the given Git branch (specified in
359the `revision` attribute) rather than the whole ref space.
360.PP
361Attribute `sync\-s`: Set to true to also sync sub\-projects.
362.PP
363Attribute `upstream`: Name of the Git ref in which a sha1 can be found. Used
364when syncing a revision locked manifest in \fB\-c\fR mode to avoid having to sync the
365entire ref space.
366.PP
367Attribute `clone\-depth`: Set the depth to use when fetching this project. If
368specified, this value will override any value given to repo init with the
369\fB\-\-depth\fR option on the command line.
370.PP
371Attribute `force\-path`: Set to true to force this project to create the local
372mirror repository according to its `path` attribute (if supplied) rather than
373the `name` attribute. This attribute only applies to the local mirrors syncing,
374it will be ignored when syncing the projects in a client working directory.
375.PP
376Element extend\-project
377.PP
378Modify the attributes of the named project.
379.PP
380This element is mostly useful in a local manifest file, to modify the attributes
381of an existing project without completely replacing the existing project
382definition. This makes the local manifest more robust against changes to the
383original manifest.
384.PP
385Attribute `path`: If specified, limit the change to projects checked out at the
386specified path, rather than all projects with the given name.
387.PP
388Attribute `groups`: List of additional groups to which this project belongs.
389Same syntax as the corresponding element of `project`.
390.PP
391Attribute `revision`: If specified, overrides the revision of the original
392project. Same syntax as the corresponding element of `project`.
393.PP
394Attribute `remote`: If specified, overrides the remote of the original project.
395Same syntax as the corresponding element of `project`.
396.PP
397Element annotation
398.PP
399Zero or more annotation elements may be specified as children of a project or
400remote element. Each element describes a name\-value pair. For projects, this
401name\-value pair will be exported into each project's environment during a
402\&'forall' command, prefixed with `REPO__`. In addition, there is an optional
403attribute "keep" which accepts the case insensitive values "true" (default) or
404"false". This attribute determines whether or not the annotation will be kept
405when exported with the manifest subcommand.
406.PP
407Element copyfile
408.PP
409Zero or more copyfile elements may be specified as children of a project
410element. Each element describes a src\-dest pair of files; the "src" file will be
411copied to the "dest" place during `repo sync` command.
412.PP
413"src" is project relative, "dest" is relative to the top of the tree. Copying
414from paths outside of the project or to paths outside of the repo client is not
415allowed.
416.PP
417"src" and "dest" must be files. Directories or symlinks are not allowed.
418Intermediate paths must not be symlinks either.
419.PP
420Parent directories of "dest" will be automatically created if missing.
421.PP
422Element linkfile
423.PP
424It's just like copyfile and runs at the same time as copyfile but instead of
425copying it creates a symlink.
426.PP
427The symlink is created at "dest" (relative to the top of the tree) and points to
428the path specified by "src" which is a path in the project.
429.PP
430Parent directories of "dest" will be automatically created if missing.
431.PP
432The symlink target may be a file or directory, but it may not point outside of
433the repo client.
434.PP
435Element remove\-project
436.PP
437Deletes the named project from the internal manifest table, possibly allowing a
438subsequent project element in the same manifest file to replace the project with
439a different source.
440.PP
441This element is mostly useful in a local manifest file, where the user can
442remove a project, and possibly replace it with their own definition.
443.PP
444Attribute `optional`: Set to true to ignore remove\-project elements with no
445matching `project` element.
446.PP
447Element repo\-hooks
448.PP
449NB: See the [practical documentation](./repo\-hooks.md) for using repo hooks.
450.PP
451Only one repo\-hooks element may be specified at a time. Attempting to redefine
452it will fail to parse.
453.PP
454Attribute `in\-project`: The project where the hooks are defined. The value must
455match the `name` attribute (**not** the `path` attribute) of a previously
456defined `project` element.
457.PP
458Attribute `enabled\-list`: List of hooks to use, whitespace or comma separated.
459.PP
460Element superproject
461.PP
462*** *Note*: This is currently a WIP. ***
463.PP
464NB: See the [git superprojects documentation](
465https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects) for background
466information.
467.PP
468This element is used to specify the URL of the superproject. It has "name" and
469"remote" as atrributes. Only "name" is required while the others have reasonable
470defaults. At most one superproject may be specified. Attempting to redefine it
471will fail to parse.
472.PP
473Attribute `name`: A unique name for the superproject. This attribute has the
474same meaning as project's name attribute. See the [element
475project](#element\-project) for more information.
476.PP
477Attribute `remote`: Name of a previously defined remote element. If not supplied
478the remote given by the default element is used.
479.PP
480Element contactinfo
481.PP
482*** *Note*: This is currently a WIP. ***
483.PP
484This element is used to let manifest authors self\-register contact info. It has
485"bugurl" as a required atrribute. This element can be repeated, and any later
486entries will clobber earlier ones. This would allow manifest authors who extend
487manifests to specify their own contact info.
488.PP
489Attribute `bugurl`: The URL to file a bug against the manifest owner.
490.PP
491Element include
492.PP
493This element provides the capability of including another manifest file into the
494originating manifest. Normal rules apply for the target manifest to include \- it
495must be a usable manifest on its own.
496.PP
497Attribute `name`: the manifest to include, specified relative to the manifest
498repository's root.
499.PP
500"name" may not be an absolute path or use "." or ".." path components. These
501restrictions are not enforced for [Local Manifests].
502.PP
503Attribute `groups`: List of additional groups to which all projects in the
504included manifest belong. This appends and recurses, meaning all projects in
505sub\-manifests carry all parent include groups. Same syntax as the corresponding
506element of `project`.
507.PP
508Local Manifests
509.PP
510Additional remotes and projects may be added through local manifest files stored
511in `$TOP_DIR/.repo/local_manifests/*.xml`.
512.PP
513For example:
514.IP
515\f(CW$ ls .repo/local_manifests\fR
516.IP
517local_manifest.xml
518another_local_manifest.xml
519.IP
520\f(CW$ cat .repo/local_manifests/local_manifest.xml\fR
521.IP
522<?xml version="1.0" encoding="UTF\-8"?>
523<manifest>
524.IP
525<project path="manifest"
526.IP
527name="tools/manifest" />
528.IP
529<project path="platform\-manifest"
530.IP
531name="platform/manifest" />
532.IP
533</manifest>
534.PP
535Users may add projects to the local manifest(s) prior to a `repo sync`
536invocation, instructing repo to automatically download and manage these extra
537projects.
538.PP
539Manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml` will be loaded
540in alphabetical order.
541.PP
542Projects from local manifest files are added into local::<local manifest
543filename> group.
544.PP
545The legacy `$TOP_DIR/.repo/local_manifest.xml` path is no longer supported.
546.SS [copyfile]: #Element\-copyfile [linkfile]: #Element\-linkfile [Local Manifests]:
547.PP
548#local\-manifests
diff --git a/man/repo-overview.1 b/man/repo-overview.1
new file mode 100644
index 00000000..a12c7640
--- /dev/null
+++ b/man/repo-overview.1
@@ -0,0 +1,39 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo overview" "Repo Manual"
3.SH NAME
4repo \- repo overview - manual page for repo overview
5.SH SYNOPSIS
6.B repo
7\fI\,overview \/\fR[\fI\,--current-branch\/\fR] [\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Display overview of unmerged project branches
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-c\fR, \fB\-\-current\-branch\fR
18consider only checked out branches
19.TP
20\fB\-\-no\-current\-branch\fR
21consider all local branches
22.SS Logging options:
23.TP
24\fB\-v\fR, \fB\-\-verbose\fR
25show all output
26.TP
27\fB\-q\fR, \fB\-\-quiet\fR
28only show errors
29.PP
30Run `repo help overview` to view the detailed manual.
31.SH DETAILS
32.PP
33The 'repo overview' command is used to display an overview of the projects
34branches, and list any local commits that have not yet been merged into the
35project.
36.PP
37The \fB\-c\fR/\-\-current\-branch option can be used to restrict the output to only
38branches currently checked out in each project. By default, all branches are
39displayed.
diff --git a/man/repo-prune.1 b/man/repo-prune.1
new file mode 100644
index 00000000..bd68a373
--- /dev/null
+++ b/man/repo-prune.1
@@ -0,0 +1,28 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo prune" "Repo Manual"
3.SH NAME
4repo \- repo prune - manual page for repo prune
5.SH SYNOPSIS
6.B repo
7\fI\,prune \/\fR[\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Prune (delete) already merged topics
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
18number of jobs to run in parallel (default: based on
19number of CPU cores)
20.SS Logging options:
21.TP
22\fB\-v\fR, \fB\-\-verbose\fR
23show all output
24.TP
25\fB\-q\fR, \fB\-\-quiet\fR
26only show errors
27.PP
28Run `repo help prune` to view the detailed manual.
diff --git a/man/repo-rebase.1 b/man/repo-rebase.1
new file mode 100644
index 00000000..aa261036
--- /dev/null
+++ b/man/repo-rebase.1
@@ -0,0 +1,55 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo rebase" "Repo Manual"
3.SH NAME
4repo \- repo rebase - manual page for repo rebase
5.SH SYNOPSIS
6.B repo
7\fI\,rebase {\/\fR[\fI\,<project>\/\fR...] \fI\,| -i <project>\/\fR...\fI\,}\/\fR
8.SH DESCRIPTION
9Summary
10.PP
11Rebase local branches on upstream branch
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-\-fail\-fast\fR
18stop rebasing after first error is hit
19.TP
20\fB\-f\fR, \fB\-\-force\-rebase\fR
21pass \fB\-\-force\-rebase\fR to git rebase
22.TP
23\fB\-\-no\-ff\fR
24pass \fB\-\-no\-ff\fR to git rebase
25.TP
26\fB\-\-autosquash\fR
27pass \fB\-\-autosquash\fR to git rebase
28.TP
29\fB\-\-whitespace\fR=\fI\,WS\/\fR
30pass \fB\-\-whitespace\fR to git rebase
31.TP
32\fB\-\-auto\-stash\fR
33stash local modifications before starting
34.TP
35\fB\-m\fR, \fB\-\-onto\-manifest\fR
36rebase onto the manifest version instead of upstream
37HEAD (this helps to make sure the local tree stays
38consistent if you previously synced to a manifest)
39.SS Logging options:
40.TP
41\fB\-v\fR, \fB\-\-verbose\fR
42show all output
43.TP
44\fB\-q\fR, \fB\-\-quiet\fR
45only show errors
46.TP
47\fB\-i\fR, \fB\-\-interactive\fR
48interactive rebase (single project only)
49.PP
50Run `repo help rebase` to view the detailed manual.
51.SH DETAILS
52.PP
53\&'repo rebase' uses git rebase to move local changes in the current topic branch
54to the HEAD of the upstream history, useful when you have made commits in a
55topic branch but need to incorporate new upstream changes "underneath" them.
diff --git a/man/repo-selfupdate.1 b/man/repo-selfupdate.1
new file mode 100644
index 00000000..70c855ab
--- /dev/null
+++ b/man/repo-selfupdate.1
@@ -0,0 +1,35 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo selfupdate" "Repo Manual"
3.SH NAME
4repo \- repo selfupdate - manual page for repo selfupdate
5.SH SYNOPSIS
6.B repo
7\fI\,selfupdate\/\fR
8.SH DESCRIPTION
9Summary
10.PP
11Update repo to the latest version
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.SS Logging options:
17.TP
18\fB\-v\fR, \fB\-\-verbose\fR
19show all output
20.TP
21\fB\-q\fR, \fB\-\-quiet\fR
22only show errors
23.SS repo Version options:
24.TP
25\fB\-\-no\-repo\-verify\fR
26do not verify repo source code
27.PP
28Run `repo help selfupdate` to view the detailed manual.
29.SH DETAILS
30.PP
31The 'repo selfupdate' command upgrades repo to the latest version, if a newer
32version is available.
33.PP
34Normally this is done automatically by 'repo sync' and does not need to be
35performed by an end\-user.
diff --git a/man/repo-smartsync.1 b/man/repo-smartsync.1
new file mode 100644
index 00000000..5d939117
--- /dev/null
+++ b/man/repo-smartsync.1
@@ -0,0 +1,118 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo smartsync" "Repo Manual"
3.SH NAME
4repo \- repo smartsync - manual page for repo smartsync
5.SH SYNOPSIS
6.B repo
7\fI\,smartsync \/\fR[\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Update working tree to the latest known good revision
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
18number of jobs to run in parallel (default: based on
19number of CPU cores)
20.TP
21\fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
22number of network jobs to run in parallel (defaults to
23\fB\-\-jobs\fR)
24.TP
25\fB\-\-jobs\-checkout\fR=\fI\,JOBS\/\fR
26number of local checkout jobs to run in parallel
27(defaults to \fB\-\-jobs\fR)
28.TP
29\fB\-f\fR, \fB\-\-force\-broken\fR
30obsolete option (to be deleted in the future)
31.TP
32\fB\-\-fail\-fast\fR
33stop syncing after first error is hit
34.TP
35\fB\-\-force\-sync\fR
36overwrite an existing git directory if it needs to
37point to a different object directory. WARNING: this
38may cause loss of data
39.TP
40\fB\-\-force\-remove\-dirty\fR
41force remove projects with uncommitted modifications
42if projects no longer exist in the manifest. WARNING:
43this may cause loss of data
44.TP
45\fB\-l\fR, \fB\-\-local\-only\fR
46only update working tree, don't fetch
47.TP
48\fB\-\-no\-manifest\-update\fR, \fB\-\-nmu\fR
49use the existing manifest checkout as\-is. (do not
50update to the latest revision)
51.TP
52\fB\-n\fR, \fB\-\-network\-only\fR
53fetch only, don't update working tree
54.TP
55\fB\-d\fR, \fB\-\-detach\fR
56detach projects back to manifest revision
57.TP
58\fB\-c\fR, \fB\-\-current\-branch\fR
59fetch only current branch from server
60.TP
61\fB\-\-no\-current\-branch\fR
62fetch all branches from server
63.TP
64\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
65temporary manifest to use for this sync
66.TP
67\fB\-\-clone\-bundle\fR
68enable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS
69.TP
70\fB\-\-no\-clone\-bundle\fR
71disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS
72.TP
73\fB\-u\fR MANIFEST_SERVER_USERNAME, \fB\-\-manifest\-server\-username\fR=\fI\,MANIFEST_SERVER_USERNAME\/\fR
74username to authenticate with the manifest server
75.TP
76\fB\-p\fR MANIFEST_SERVER_PASSWORD, \fB\-\-manifest\-server\-password\fR=\fI\,MANIFEST_SERVER_PASSWORD\/\fR
77password to authenticate with the manifest server
78.TP
79\fB\-\-fetch\-submodules\fR
80fetch submodules from server
81.TP
82\fB\-\-use\-superproject\fR
83use the manifest superproject to sync projects
84.TP
85\fB\-\-no\-use\-superproject\fR
86disable use of manifest superprojects
87.TP
88\fB\-\-tags\fR
89fetch tags
90.TP
91\fB\-\-no\-tags\fR
92don't fetch tags
93.TP
94\fB\-\-optimized\-fetch\fR
95only fetch projects fixed to sha1 if revision does not
96exist locally
97.TP
98\fB\-\-retry\-fetches\fR=\fI\,RETRY_FETCHES\/\fR
99number of times to retry fetches on transient errors
100.TP
101\fB\-\-prune\fR
102delete refs that no longer exist on the remote
103.SS Logging options:
104.TP
105\fB\-v\fR, \fB\-\-verbose\fR
106show all output
107.TP
108\fB\-q\fR, \fB\-\-quiet\fR
109only show errors
110.SS repo Version options:
111.TP
112\fB\-\-no\-repo\-verify\fR
113do not verify repo source code
114.PP
115Run `repo help smartsync` to view the detailed manual.
116.SH DETAILS
117.PP
118The 'repo smartsync' command is a shortcut for sync \fB\-s\fR.
diff --git a/man/repo-stage.1 b/man/repo-stage.1
new file mode 100644
index 00000000..07e1cac6
--- /dev/null
+++ b/man/repo-stage.1
@@ -0,0 +1,30 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo stage" "Repo Manual"
3.SH NAME
4repo \- repo stage - manual page for repo stage
5.SH SYNOPSIS
6.B repo
7\fI\,stage -i \/\fR[\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Stage file(s) for commit
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.SS Logging options:
17.TP
18\fB\-v\fR, \fB\-\-verbose\fR
19show all output
20.TP
21\fB\-q\fR, \fB\-\-quiet\fR
22only show errors
23.TP
24\fB\-i\fR, \fB\-\-interactive\fR
25use interactive staging
26.PP
27Run `repo help stage` to view the detailed manual.
28.SH DETAILS
29.PP
30The 'repo stage' command stages files to prepare the next commit.
diff --git a/man/repo-start.1 b/man/repo-start.1
new file mode 100644
index 00000000..b00a31f4
--- /dev/null
+++ b/man/repo-start.1
@@ -0,0 +1,41 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo start" "Repo Manual"
3.SH NAME
4repo \- repo start - manual page for repo start
5.SH SYNOPSIS
6.B repo
7\fI\,start <newbranchname> \/\fR[\fI\,--all | <project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Start a new branch for development
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
18number of jobs to run in parallel (default: based on
19number of CPU cores)
20.TP
21\fB\-\-all\fR
22begin branch in all projects
23.TP
24\fB\-r\fR REVISION, \fB\-\-rev\fR=\fI\,REVISION\/\fR, \fB\-\-revision\fR=\fI\,REVISION\/\fR
25point branch at this revision instead of upstream
26.TP
27\fB\-\-head\fR, \fB\-\-HEAD\fR
28abbreviation for \fB\-\-rev\fR HEAD
29.SS Logging options:
30.TP
31\fB\-v\fR, \fB\-\-verbose\fR
32show all output
33.TP
34\fB\-q\fR, \fB\-\-quiet\fR
35only show errors
36.PP
37Run `repo help start` to view the detailed manual.
38.SH DETAILS
39.PP
40\&'repo start' begins a new branch of development, starting from the revision
41specified in the manifest.
diff --git a/man/repo-status.1 b/man/repo-status.1
new file mode 100644
index 00000000..fbae2c5d
--- /dev/null
+++ b/man/repo-status.1
@@ -0,0 +1,98 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo status" "Repo Manual"
3.SH NAME
4repo \- repo status - manual page for repo status
5.SH SYNOPSIS
6.B repo
7\fI\,status \/\fR[\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Show the working tree status
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
18number of jobs to run in parallel (default: based on
19number of CPU cores)
20.TP
21\fB\-o\fR, \fB\-\-orphans\fR
22include objects in working directory outside of repo
23projects
24.SS Logging options:
25.TP
26\fB\-v\fR, \fB\-\-verbose\fR
27show all output
28.TP
29\fB\-q\fR, \fB\-\-quiet\fR
30only show errors
31.PP
32Run `repo help status` to view the detailed manual.
33.SH DETAILS
34.PP
35\&'repo status' compares the working tree to the staging area (aka index), and the
36most recent commit on this branch (HEAD), in each project specified. A summary
37is displayed, one line per file where there is a difference between these three
38states.
39.PP
40The \fB\-j\fR/\-\-jobs option can be used to run multiple status queries in parallel.
41.PP
42The \fB\-o\fR/\-\-orphans option can be used to show objects that are in the working
43directory, but not associated with a repo project. This includes unmanaged
44top\-level files and directories, but also includes deeper items. For example, if
45dir/subdir/proj1 and dir/subdir/proj2 are repo projects, dir/subdir/proj3 will
46be shown if it is not known to repo.
47.PP
48Status Display
49.PP
50The status display is organized into three columns of information, for example
51if the file 'subcmds/status.py' is modified in the project 'repo' on branch
52\&'devwork':
53.TP
54project repo/
55branch devwork
56.TP
57\fB\-m\fR
58subcmds/status.py
59.PP
60The first column explains how the staging area (index) differs from the last
61commit (HEAD). Its values are always displayed in upper case and have the
62following meanings:
63.TP
64\-:
65no difference
66.TP
67A:
68added (not in HEAD, in index )
69.TP
70M:
71modified ( in HEAD, in index, different content )
72.TP
73D:
74deleted ( in HEAD, not in index )
75.TP
76R:
77renamed (not in HEAD, in index, path changed )
78.TP
79C:
80copied (not in HEAD, in index, copied from another)
81.TP
82T:
83mode changed ( in HEAD, in index, same content )
84.TP
85U:
86unmerged; conflict resolution required
87.PP
88The second column explains how the working directory differs from the index. Its
89values are always displayed in lower case and have the following meanings:
90.TP
91\-:
92new / unknown (not in index, in work tree )
93.TP
94m:
95modified ( in index, in work tree, modified )
96.TP
97d:
98deleted ( in index, not in work tree )
diff --git a/man/repo-sync.1 b/man/repo-sync.1
new file mode 100644
index 00000000..c87c9701
--- /dev/null
+++ b/man/repo-sync.1
@@ -0,0 +1,209 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo sync" "Repo Manual"
3.SH NAME
4repo \- repo sync - manual page for repo sync
5.SH SYNOPSIS
6.B repo
7\fI\,sync \/\fR[\fI\,<project>\/\fR...]
8.SH DESCRIPTION
9Summary
10.PP
11Update working tree to the latest revision
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
18number of jobs to run in parallel (default: based on
19number of CPU cores)
20.TP
21\fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
22number of network jobs to run in parallel (defaults to
23\fB\-\-jobs\fR)
24.TP
25\fB\-\-jobs\-checkout\fR=\fI\,JOBS\/\fR
26number of local checkout jobs to run in parallel
27(defaults to \fB\-\-jobs\fR)
28.TP
29\fB\-f\fR, \fB\-\-force\-broken\fR
30obsolete option (to be deleted in the future)
31.TP
32\fB\-\-fail\-fast\fR
33stop syncing after first error is hit
34.TP
35\fB\-\-force\-sync\fR
36overwrite an existing git directory if it needs to
37point to a different object directory. WARNING: this
38may cause loss of data
39.TP
40\fB\-\-force\-remove\-dirty\fR
41force remove projects with uncommitted modifications
42if projects no longer exist in the manifest. WARNING:
43this may cause loss of data
44.TP
45\fB\-l\fR, \fB\-\-local\-only\fR
46only update working tree, don't fetch
47.TP
48\fB\-\-no\-manifest\-update\fR, \fB\-\-nmu\fR
49use the existing manifest checkout as\-is. (do not
50update to the latest revision)
51.TP
52\fB\-n\fR, \fB\-\-network\-only\fR
53fetch only, don't update working tree
54.TP
55\fB\-d\fR, \fB\-\-detach\fR
56detach projects back to manifest revision
57.TP
58\fB\-c\fR, \fB\-\-current\-branch\fR
59fetch only current branch from server
60.TP
61\fB\-\-no\-current\-branch\fR
62fetch all branches from server
63.TP
64\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
65temporary manifest to use for this sync
66.TP
67\fB\-\-clone\-bundle\fR
68enable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS
69.TP
70\fB\-\-no\-clone\-bundle\fR
71disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS
72.TP
73\fB\-u\fR MANIFEST_SERVER_USERNAME, \fB\-\-manifest\-server\-username\fR=\fI\,MANIFEST_SERVER_USERNAME\/\fR
74username to authenticate with the manifest server
75.TP
76\fB\-p\fR MANIFEST_SERVER_PASSWORD, \fB\-\-manifest\-server\-password\fR=\fI\,MANIFEST_SERVER_PASSWORD\/\fR
77password to authenticate with the manifest server
78.TP
79\fB\-\-fetch\-submodules\fR
80fetch submodules from server
81.TP
82\fB\-\-use\-superproject\fR
83use the manifest superproject to sync projects
84.TP
85\fB\-\-no\-use\-superproject\fR
86disable use of manifest superprojects
87.TP
88\fB\-\-tags\fR
89fetch tags
90.TP
91\fB\-\-no\-tags\fR
92don't fetch tags
93.TP
94\fB\-\-optimized\-fetch\fR
95only fetch projects fixed to sha1 if revision does not
96exist locally
97.TP
98\fB\-\-retry\-fetches\fR=\fI\,RETRY_FETCHES\/\fR
99number of times to retry fetches on transient errors
100.TP
101\fB\-\-prune\fR
102delete refs that no longer exist on the remote
103.TP
104\fB\-s\fR, \fB\-\-smart\-sync\fR
105smart sync using manifest from the latest known good
106build
107.TP
108\fB\-t\fR SMART_TAG, \fB\-\-smart\-tag\fR=\fI\,SMART_TAG\/\fR
109smart sync using manifest from a known tag
110.SS Logging options:
111.TP
112\fB\-v\fR, \fB\-\-verbose\fR
113show all output
114.TP
115\fB\-q\fR, \fB\-\-quiet\fR
116only show errors
117.SS repo Version options:
118.TP
119\fB\-\-no\-repo\-verify\fR
120do not verify repo source code
121.PP
122Run `repo help sync` to view the detailed manual.
123.SH DETAILS
124.PP
125The 'repo sync' command synchronizes local project directories with the remote
126repositories specified in the manifest. If a local project does not yet exist,
127it will clone a new local directory from the remote repository and set up
128tracking branches as specified in the manifest. If the local project already
129exists, 'repo sync' will update the remote branches and rebase any new local
130changes on top of the new remote changes.
131.PP
132\&'repo sync' will synchronize all projects listed at the command line. Projects
133can be specified either by name, or by a relative or absolute path to the
134project's local directory. If no projects are specified, 'repo sync' will
135synchronize all projects listed in the manifest.
136.PP
137The \fB\-d\fR/\-\-detach option can be used to switch specified projects back to the
138manifest revision. This option is especially helpful if the project is currently
139on a topic branch, but the manifest revision is temporarily needed.
140.PP
141The \fB\-s\fR/\-\-smart\-sync option can be used to sync to a known good build as
142specified by the manifest\-server element in the current manifest. The
143\fB\-t\fR/\-\-smart\-tag option is similar and allows you to specify a custom tag/label.
144.PP
145The \fB\-u\fR/\-\-manifest\-server\-username and \fB\-p\fR/\-\-manifest\-server\-password options can
146be used to specify a username and password to authenticate with the manifest
147server when using the \fB\-s\fR or \fB\-t\fR option.
148.PP
149If \fB\-u\fR and \fB\-p\fR are not specified when using the \fB\-s\fR or \fB\-t\fR option, 'repo sync' will
150attempt to read authentication credentials for the manifest server from the
151user's .netrc file.
152.PP
153\&'repo sync' will not use authentication credentials from \fB\-u\fR/\-p or .netrc if the
154manifest server specified in the manifest file already includes credentials.
155.PP
156By default, all projects will be synced. The \fB\-\-fail\-fast\fR option can be used to
157halt syncing as soon as possible when the first project fails to sync.
158.PP
159The \fB\-\-force\-sync\fR option can be used to overwrite existing git directories if
160they have previously been linked to a different object directory. WARNING: This
161may cause data to be lost since refs may be removed when overwriting.
162.PP
163The \fB\-\-force\-remove\-dirty\fR option can be used to remove previously used projects
164with uncommitted changes. WARNING: This may cause data to be lost since
165uncommitted changes may be removed with projects that no longer exist in the
166manifest.
167.PP
168The \fB\-\-no\-clone\-bundle\fR option disables any attempt to use \fI\,$URL/clone.bundle\/\fP to
169bootstrap a new Git repository from a resumeable bundle file on a content
170delivery network. This may be necessary if there are problems with the local
171Python HTTP client or proxy configuration, but the Git binary works.
172.PP
173The \fB\-\-fetch\-submodules\fR option enables fetching Git submodules of a project from
174server.
175.PP
176The \fB\-c\fR/\-\-current\-branch option can be used to only fetch objects that are on the
177branch specified by a project's revision.
178.PP
179The \fB\-\-optimized\-fetch\fR option can be used to only fetch projects that are fixed
180to a sha1 revision if the sha1 revision does not already exist locally.
181.PP
182The \fB\-\-prune\fR option can be used to remove any refs that no longer exist on the
183remote.
184.PP
185SSH Connections
186.PP
187If at least one project remote URL uses an SSH connection (ssh://, git+ssh://,
188or user@host:path syntax) repo will automatically enable the SSH ControlMaster
189option when connecting to that host. This feature permits other projects in the
190same 'repo sync' session to reuse the same SSH tunnel, saving connection setup
191overheads.
192.PP
193To disable this behavior on UNIX platforms, set the GIT_SSH environment variable
194to 'ssh'. For example:
195.IP
196export GIT_SSH=ssh
197repo sync
198.PP
199Compatibility
200.PP
201This feature is automatically disabled on Windows, due to the lack of UNIX
202domain socket support.
203.PP
204This feature is not compatible with url.insteadof rewrites in the user's
205~/.gitconfig. 'repo sync' is currently not able to perform the rewrite early
206enough to establish the ControlMaster tunnel.
207.PP
208If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or later is
209required to fix a server side protocol bug.
diff --git a/man/repo-upload.1 b/man/repo-upload.1
new file mode 100644
index 00000000..36a0daca
--- /dev/null
+++ b/man/repo-upload.1
@@ -0,0 +1,175 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo upload" "Repo Manual"
3.SH NAME
4repo \- repo upload - manual page for repo upload
5.SH SYNOPSIS
6.B repo
7\fI\,upload \/\fR[\fI\,--re --cc\/\fR] [\fI\,<project>\/\fR]...
8.SH DESCRIPTION
9Summary
10.PP
11Upload changes for code review
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.TP
17\fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
18number of jobs to run in parallel (default: based on
19number of CPU cores)
20.TP
21\fB\-t\fR
22send local branch name to Gerrit Code Review
23.TP
24\fB\-\-hashtag\fR=\fI\,HASHTAGS\/\fR, \fB\-\-ht\fR=\fI\,HASHTAGS\/\fR
25add hashtags (comma delimited) to the review
26.TP
27\fB\-\-hashtag\-branch\fR, \fB\-\-htb\fR
28add local branch name as a hashtag
29.TP
30\fB\-l\fR LABELS, \fB\-\-label\fR=\fI\,LABELS\/\fR
31add a label when uploading
32.TP
33\fB\-\-re\fR=\fI\,REVIEWERS\/\fR, \fB\-\-reviewers\fR=\fI\,REVIEWERS\/\fR
34request reviews from these people
35.TP
36\fB\-\-cc\fR=\fI\,CC\/\fR
37also send email to these email addresses
38.TP
39\fB\-\-br\fR=\fI\,BRANCH\/\fR, \fB\-\-branch\fR=\fI\,BRANCH\/\fR
40(local) branch to upload
41.TP
42\fB\-c\fR, \fB\-\-current\-branch\fR
43upload current git branch
44.TP
45\fB\-\-no\-current\-branch\fR
46upload all git branches
47.TP
48\fB\-\-ne\fR, \fB\-\-no\-emails\fR
49do not send e\-mails on upload
50.TP
51\fB\-p\fR, \fB\-\-private\fR
52upload as a private change (deprecated; use \fB\-\-wip\fR)
53.TP
54\fB\-w\fR, \fB\-\-wip\fR
55upload as a work\-in\-progress change
56.TP
57\fB\-o\fR PUSH_OPTIONS, \fB\-\-push\-option\fR=\fI\,PUSH_OPTIONS\/\fR
58additional push options to transmit
59.TP
60\fB\-D\fR BRANCH, \fB\-\-destination\fR=\fI\,BRANCH\/\fR, \fB\-\-dest\fR=\fI\,BRANCH\/\fR
61submit for review on this target branch
62.TP
63\fB\-n\fR, \fB\-\-dry\-run\fR
64do everything except actually upload the CL
65.TP
66\fB\-y\fR, \fB\-\-yes\fR
67answer yes to all safe prompts
68.TP
69\fB\-\-no\-cert\-checks\fR
70disable verifying ssl certs (unsafe)
71.SS Logging options:
72.TP
73\fB\-v\fR, \fB\-\-verbose\fR
74show all output
75.TP
76\fB\-q\fR, \fB\-\-quiet\fR
77only show errors
78.SS pre\-upload hooks:
79.TP
80\fB\-\-no\-verify\fR
81Do not run the pre\-upload hook.
82.TP
83\fB\-\-verify\fR
84Run the pre\-upload hook without prompting.
85.TP
86\fB\-\-ignore\-hooks\fR
87Do not abort if pre\-upload hooks fail.
88.PP
89Run `repo help upload` to view the detailed manual.
90.SH DETAILS
91.PP
92The 'repo upload' command is used to send changes to the Gerrit Code Review
93system. It searches for topic branches in local projects that have not yet been
94published for review. If multiple topic branches are found, 'repo upload' opens
95an editor to allow the user to select which branches to upload.
96.PP
97\&'repo upload' searches for uploadable changes in all projects listed at the
98command line. Projects can be specified either by name, or by a relative or
99absolute path to the project's local directory. If no projects are specified,
100\&'repo upload' will search for uploadable changes in all projects listed in the
101manifest.
102.PP
103If the \fB\-\-reviewers\fR or \fB\-\-cc\fR options are passed, those emails are added to the
104respective list of users, and emails are sent to any new users. Users passed as
105\fB\-\-reviewers\fR must already be registered with the code review system, or the
106upload will fail.
107.PP
108Configuration
109.PP
110review.URL.autoupload:
111.PP
112To disable the "Upload ... (y/N)?" prompt, you can set a per\-project or global
113Git configuration option. If review.URL.autoupload is set to "true" then repo
114will assume you always answer "y" at the prompt, and will not prompt you
115further. If it is set to "false" then repo will assume you always answer "n",
116and will abort.
117.PP
118review.URL.autoreviewer:
119.PP
120To automatically append a user or mailing list to reviews, you can set a
121per\-project or global Git option to do so.
122.PP
123review.URL.autocopy:
124.PP
125To automatically copy a user or mailing list to all uploaded reviews, you can
126set a per\-project or global Git option to do so. Specifically,
127review.URL.autocopy can be set to a comma separated list of reviewers who you
128always want copied on all uploads with a non\-empty \fB\-\-re\fR argument.
129.PP
130review.URL.username:
131.PP
132Override the username used to connect to Gerrit Code Review. By default the
133local part of the email address is used.
134.PP
135The URL must match the review URL listed in the manifest XML file, or in the
136\&.git/config within the project. For example:
137.IP
138[remote "origin"]
139.IP
140url = git://git.example.com/project.git
141review = http://review.example.com/
142.IP
143[review "http://review.example.com/"]
144.IP
145autoupload = true
146autocopy = johndoe@company.com,my\-team\-alias@company.com
147.PP
148review.URL.uploadtopic:
149.PP
150To add a topic branch whenever uploading a commit, you can set a per\-project or
151global Git option to do so. If review.URL.uploadtopic is set to "true" then repo
152will assume you always want the equivalent of the \fB\-t\fR option to the repo command.
153If unset or set to "false" then repo will make use of only the command line
154option.
155.PP
156review.URL.uploadhashtags:
157.PP
158To add hashtags whenever uploading a commit, you can set a per\-project or global
159Git option to do so. The value of review.URL.uploadhashtags will be used as
160comma delimited hashtags like the \fB\-\-hashtag\fR option.
161.PP
162review.URL.uploadlabels:
163.PP
164To add labels whenever uploading a commit, you can set a per\-project or global
165Git option to do so. The value of review.URL.uploadlabels will be used as comma
166delimited labels like the \fB\-\-label\fR option.
167.PP
168review.URL.uploadnotify:
169.PP
170Control e\-mail notifications when uploading.
171https://gerrit\-review.googlesource.com/Documentation/user\-upload.html#notify
172.PP
173References
174.PP
175Gerrit Code Review: https://www.gerritcodereview.com/
diff --git a/man/repo-version.1 b/man/repo-version.1
new file mode 100644
index 00000000..cc703f61
--- /dev/null
+++ b/man/repo-version.1
@@ -0,0 +1,24 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo version" "Repo Manual"
3.SH NAME
4repo \- repo version - manual page for repo version
5.SH SYNOPSIS
6.B repo
7\fI\,version\/\fR
8.SH DESCRIPTION
9Summary
10.PP
11Display the version of repo
12.SH OPTIONS
13.TP
14\fB\-h\fR, \fB\-\-help\fR
15show this help message and exit
16.SS Logging options:
17.TP
18\fB\-v\fR, \fB\-\-verbose\fR
19show all output
20.TP
21\fB\-q\fR, \fB\-\-quiet\fR
22only show errors
23.PP
24Run `repo help version` to view the detailed manual.
diff --git a/man/repo.1 b/man/repo.1
new file mode 100644
index 00000000..4aa76380
--- /dev/null
+++ b/man/repo.1
@@ -0,0 +1,133 @@
1.\" DO NOT MODIFY THIS FILE! It was generated by help2man.
2.TH REPO "1" "July 2021" "repo" "Repo Manual"
3.SH NAME
4repo \- repository management tool built on top of git
5.SH SYNOPSIS
6.B repo
7[\fI\,-p|--paginate|--no-pager\/\fR] \fI\,COMMAND \/\fR[\fI\,ARGS\/\fR]
8.SH OPTIONS
9.TP
10\fB\-h\fR, \fB\-\-help\fR
11show this help message and exit
12.TP
13\fB\-\-help\-all\fR
14show this help message with all subcommands and exit
15.TP
16\fB\-p\fR, \fB\-\-paginate\fR
17display command output in the pager
18.TP
19\fB\-\-no\-pager\fR
20disable the pager
21.TP
22\fB\-\-color\fR=\fI\,COLOR\/\fR
23control color usage: auto, always, never
24.TP
25\fB\-\-trace\fR
26trace git command execution (REPO_TRACE=1)
27.TP
28\fB\-\-trace\-python\fR
29trace python command execution
30.TP
31\fB\-\-time\fR
32time repo command execution
33.TP
34\fB\-\-version\fR
35display this version of repo
36.TP
37\fB\-\-show\-toplevel\fR
38display the path of the top\-level directory of the
39repo client checkout
40.TP
41\fB\-\-event\-log\fR=\fI\,EVENT_LOG\/\fR
42filename of event log to append timeline to
43.TP
44\fB\-\-git\-trace2\-event\-log\fR=\fI\,GIT_TRACE2_EVENT_LOG\/\fR
45directory to write git trace2 event log to
46.SS "The complete list of recognized repo commands are:"
47.TP
48abandon
49Permanently abandon a development branch
50.TP
51branch
52View current topic branches
53.TP
54branches
55View current topic branches
56.TP
57checkout
58Checkout a branch for development
59.TP
60cherry\-pick
61Cherry\-pick a change.
62.TP
63diff
64Show changes between commit and working tree
65.TP
66diffmanifests
67Manifest diff utility
68.TP
69download
70Download and checkout a change
71.TP
72forall
73Run a shell command in each project
74.TP
75gitc\-delete
76Delete a GITC Client.
77.TP
78gitc\-init
79Initialize a GITC Client.
80.TP
81grep
82Print lines matching a pattern
83.TP
84help
85Display detailed help on a command
86.TP
87info
88Get info on the manifest branch, current branch or unmerged branches
89.TP
90init
91Initialize a repo client checkout in the current directory
92.TP
93list
94List projects and their associated directories
95.TP
96manifest
97Manifest inspection utility
98.TP
99overview
100Display overview of unmerged project branches
101.TP
102prune
103Prune (delete) already merged topics
104.TP
105rebase
106Rebase local branches on upstream branch
107.TP
108selfupdate
109Update repo to the latest version
110.TP
111smartsync
112Update working tree to the latest known good revision
113.TP
114stage
115Stage file(s) for commit
116.TP
117start
118Start a new branch for development
119.TP
120status
121Show the working tree status
122.TP
123sync
124Update working tree to the latest revision
125.TP
126upload
127Upload changes for code review
128.TP
129version
130Display the version of repo
131.PP
132See 'repo help <command>' for more information on a specific command.
133Bug reports: https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue
diff --git a/manifest_xml.py b/manifest_xml.py
index 0c2b45e5..68ead53c 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -12,6 +12,7 @@
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14 14
15import collections
15import itertools 16import itertools
16import os 17import os
17import platform 18import platform
@@ -24,14 +25,21 @@ import gitc_utils
24from git_config import GitConfig, IsId 25from git_config import GitConfig, IsId
25from git_refs import R_HEADS, HEAD 26from git_refs import R_HEADS, HEAD
26import platform_utils 27import platform_utils
27from project import RemoteSpec, Project, MetaProject 28from project import Annotation, RemoteSpec, Project, MetaProject
28from error import (ManifestParseError, ManifestInvalidPathError, 29from error import (ManifestParseError, ManifestInvalidPathError,
29 ManifestInvalidRevisionError) 30 ManifestInvalidRevisionError)
31from wrapper import Wrapper
30 32
31MANIFEST_FILE_NAME = 'manifest.xml' 33MANIFEST_FILE_NAME = 'manifest.xml'
32LOCAL_MANIFEST_NAME = 'local_manifest.xml' 34LOCAL_MANIFEST_NAME = 'local_manifest.xml'
33LOCAL_MANIFESTS_DIR_NAME = 'local_manifests' 35LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
34 36
37# Add all projects from local manifest into a group.
38LOCAL_MANIFEST_GROUP_PREFIX = 'local:'
39
40# ContactInfo has the self-registered bug url, supplied by the manifest authors.
41ContactInfo = collections.namedtuple('ContactInfo', 'bugurl')
42
35# urljoin gets confused if the scheme is not known. 43# urljoin gets confused if the scheme is not known.
36urllib.parse.uses_relative.extend([ 44urllib.parse.uses_relative.extend([
37 'ssh', 45 'ssh',
@@ -114,9 +122,13 @@ class _Default(object):
114 sync_tags = True 122 sync_tags = True
115 123
116 def __eq__(self, other): 124 def __eq__(self, other):
125 if not isinstance(other, _Default):
126 return False
117 return self.__dict__ == other.__dict__ 127 return self.__dict__ == other.__dict__
118 128
119 def __ne__(self, other): 129 def __ne__(self, other):
130 if not isinstance(other, _Default):
131 return True
120 return self.__dict__ != other.__dict__ 132 return self.__dict__ != other.__dict__
121 133
122 134
@@ -137,14 +149,22 @@ class _XmlRemote(object):
137 self.reviewUrl = review 149 self.reviewUrl = review
138 self.revision = revision 150 self.revision = revision
139 self.resolvedFetchUrl = self._resolveFetchUrl() 151 self.resolvedFetchUrl = self._resolveFetchUrl()
152 self.annotations = []
140 153
141 def __eq__(self, other): 154 def __eq__(self, other):
142 return self.__dict__ == other.__dict__ 155 if not isinstance(other, _XmlRemote):
156 return False
157 return (sorted(self.annotations) == sorted(other.annotations) and
158 self.name == other.name and self.fetchUrl == other.fetchUrl and
159 self.pushUrl == other.pushUrl and self.remoteAlias == other.remoteAlias
160 and self.reviewUrl == other.reviewUrl and self.revision == other.revision)
143 161
144 def __ne__(self, other): 162 def __ne__(self, other):
145 return self.__dict__ != other.__dict__ 163 return not self.__eq__(other)
146 164
147 def _resolveFetchUrl(self): 165 def _resolveFetchUrl(self):
166 if self.fetchUrl is None:
167 return ''
148 url = self.fetchUrl.rstrip('/') 168 url = self.fetchUrl.rstrip('/')
149 manifestUrl = self.manifestUrl.rstrip('/') 169 manifestUrl = self.manifestUrl.rstrip('/')
150 # urljoin will gets confused over quite a few things. The ones we care 170 # urljoin will gets confused over quite a few things. The ones we care
@@ -173,6 +193,9 @@ class _XmlRemote(object):
173 orig_name=self.name, 193 orig_name=self.name,
174 fetchUrl=self.fetchUrl) 194 fetchUrl=self.fetchUrl)
175 195
196 def AddAnnotation(self, name, value, keep):
197 self.annotations.append(Annotation(name, value, keep))
198
176 199
177class XmlManifest(object): 200class XmlManifest(object):
178 """manages the repo configuration file""" 201 """manages the repo configuration file"""
@@ -247,8 +270,7 @@ class XmlManifest(object):
247 self.Override(name) 270 self.Override(name)
248 271
249 # Old versions of repo would generate symlinks we need to clean up. 272 # Old versions of repo would generate symlinks we need to clean up.
250 if os.path.lexists(self.manifestFile): 273 platform_utils.remove(self.manifestFile, missing_ok=True)
251 platform_utils.remove(self.manifestFile)
252 # This file is interpreted as if it existed inside the manifest repo. 274 # This file is interpreted as if it existed inside the manifest repo.
253 # That allows us to use <include> with the relative file name. 275 # That allows us to use <include> with the relative file name.
254 with open(self.manifestFile, 'w') as fp: 276 with open(self.manifestFile, 'w') as fp:
@@ -282,6 +304,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
282 if r.revision is not None: 304 if r.revision is not None:
283 e.setAttribute('revision', r.revision) 305 e.setAttribute('revision', r.revision)
284 306
307 for a in r.annotations:
308 if a.keep == 'true':
309 ae = doc.createElement('annotation')
310 ae.setAttribute('name', a.name)
311 ae.setAttribute('value', a.value)
312 e.appendChild(ae)
313
285 def _ParseList(self, field): 314 def _ParseList(self, field):
286 """Parse fields that contain flattened lists. 315 """Parse fields that contain flattened lists.
287 316
@@ -477,6 +506,15 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
477 if not d.remote or remote.orig_name != remoteName: 506 if not d.remote or remote.orig_name != remoteName:
478 remoteName = remote.orig_name 507 remoteName = remote.orig_name
479 e.setAttribute('remote', remoteName) 508 e.setAttribute('remote', remoteName)
509 revision = remote.revision or d.revisionExpr
510 if not revision or revision != self._superproject['revision']:
511 e.setAttribute('revision', self._superproject['revision'])
512 root.appendChild(e)
513
514 if self._contactinfo.bugurl != Wrapper().BUG_URL:
515 root.appendChild(doc.createTextNode(''))
516 e = doc.createElement('contactinfo')
517 e.setAttribute('bugurl', self._contactinfo.bugurl)
480 root.appendChild(e) 518 root.appendChild(e)
481 519
482 return doc 520 return doc
@@ -490,6 +528,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
490 'manifest-server', 528 'manifest-server',
491 'repo-hooks', 529 'repo-hooks',
492 'superproject', 530 'superproject',
531 'contactinfo',
493 } 532 }
494 # Elements that may be repeated. 533 # Elements that may be repeated.
495 MULTI_ELEMENTS = { 534 MULTI_ELEMENTS = {
@@ -566,6 +605,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
566 return self._superproject 605 return self._superproject
567 606
568 @property 607 @property
608 def contactinfo(self):
609 self._Load()
610 return self._contactinfo
611
612 @property
569 def notice(self): 613 def notice(self):
570 self._Load() 614 self._Load()
571 return self._notice 615 return self._notice
@@ -596,6 +640,17 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
596 return set(x.strip() for x in exclude.split(',')) 640 return set(x.strip() for x in exclude.split(','))
597 641
598 @property 642 @property
643 def UseLocalManifests(self):
644 return self._load_local_manifests
645
646 def SetUseLocalManifests(self, value):
647 self._load_local_manifests = value
648
649 @property
650 def HasLocalManifests(self):
651 return self._load_local_manifests and self.local_manifests
652
653 @property
599 def IsMirror(self): 654 def IsMirror(self):
600 return self.manifestProject.config.GetBoolean('repo.mirror') 655 return self.manifestProject.config.GetBoolean('repo.mirror')
601 656
@@ -630,6 +685,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
630 self._default = None 685 self._default = None
631 self._repo_hooks_project = None 686 self._repo_hooks_project = None
632 self._superproject = {} 687 self._superproject = {}
688 self._contactinfo = ContactInfo(Wrapper().BUG_URL)
633 self._notice = None 689 self._notice = None
634 self.branch = None 690 self.branch = None
635 self._manifest_server = None 691 self._manifest_server = None
@@ -657,7 +713,9 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
657 # Since local manifests are entirely managed by the user, allow 713 # Since local manifests are entirely managed by the user, allow
658 # them to point anywhere the user wants. 714 # them to point anywhere the user wants.
659 nodes.append(self._ParseManifestXml( 715 nodes.append(self._ParseManifestXml(
660 local, self.repodir, restrict_includes=False)) 716 local, self.repodir,
717 parent_groups=f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}',
718 restrict_includes=False))
661 except OSError: 719 except OSError:
662 pass 720 pass
663 721
@@ -754,9 +812,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
754 for node in itertools.chain(*node_list): 812 for node in itertools.chain(*node_list):
755 if node.nodeName == 'default': 813 if node.nodeName == 'default':
756 new_default = self._ParseDefault(node) 814 new_default = self._ParseDefault(node)
815 emptyDefault = not node.hasAttributes() and not node.hasChildNodes()
757 if self._default is None: 816 if self._default is None:
758 self._default = new_default 817 self._default = new_default
759 elif new_default != self._default: 818 elif not emptyDefault and new_default != self._default:
760 raise ManifestParseError('duplicate default in %s' % 819 raise ManifestParseError('duplicate default in %s' %
761 (self.manifestFile)) 820 (self.manifestFile))
762 821
@@ -795,6 +854,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
795 for subproject in project.subprojects: 854 for subproject in project.subprojects:
796 recursively_add_projects(subproject) 855 recursively_add_projects(subproject)
797 856
857 repo_hooks_project = None
858 enabled_repo_hooks = None
798 for node in itertools.chain(*node_list): 859 for node in itertools.chain(*node_list):
799 if node.nodeName == 'project': 860 if node.nodeName == 'project':
800 project = self._ParseProject(node) 861 project = self._ParseProject(node)
@@ -807,6 +868,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
807 'project: %s' % name) 868 'project: %s' % name)
808 869
809 path = node.getAttribute('path') 870 path = node.getAttribute('path')
871 dest_path = node.getAttribute('dest-path')
810 groups = node.getAttribute('groups') 872 groups = node.getAttribute('groups')
811 if groups: 873 if groups:
812 groups = self._ParseList(groups) 874 groups = self._ParseList(groups)
@@ -815,46 +877,37 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
815 if remote: 877 if remote:
816 remote = self._get_remote(node) 878 remote = self._get_remote(node)
817 879
880 named_projects = self._projects[name]
881 if dest_path and not path and len(named_projects) > 1:
882 raise ManifestParseError('extend-project cannot use dest-path when '
883 'matching multiple projects: %s' % name)
818 for p in self._projects[name]: 884 for p in self._projects[name]:
819 if path and p.relpath != path: 885 if path and p.relpath != path:
820 continue 886 continue
821 if groups: 887 if groups:
822 p.groups.extend(groups) 888 p.groups.extend(groups)
823 if revision: 889 if revision:
824 p.revisionExpr = revision 890 p.SetRevision(revision)
825 if IsId(revision): 891
826 p.revisionId = revision
827 else:
828 p.revisionId = None
829 if remote: 892 if remote:
830 p.remote = remote.ToRemoteSpec(name) 893 p.remote = remote.ToRemoteSpec(name)
831 if node.nodeName == 'repo-hooks':
832 # Get the name of the project and the (space-separated) list of enabled.
833 repo_hooks_project = self._reqatt(node, 'in-project')
834 enabled_repo_hooks = self._ParseList(self._reqatt(node, 'enabled-list'))
835 894
895 if dest_path:
896 del self._paths[p.relpath]
897 relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths(name, dest_path)
898 p.UpdatePaths(relpath, worktree, gitdir, objdir)
899 self._paths[p.relpath] = p
900
901 if node.nodeName == 'repo-hooks':
836 # Only one project can be the hooks project 902 # Only one project can be the hooks project
837 if self._repo_hooks_project is not None: 903 if repo_hooks_project is not None:
838 raise ManifestParseError( 904 raise ManifestParseError(
839 'duplicate repo-hooks in %s' % 905 'duplicate repo-hooks in %s' %
840 (self.manifestFile)) 906 (self.manifestFile))
841 907
842 # Store a reference to the Project. 908 # Get the name of the project and the (space-separated) list of enabled.
843 try: 909 repo_hooks_project = self._reqatt(node, 'in-project')
844 repo_hooks_projects = self._projects[repo_hooks_project] 910 enabled_repo_hooks = self._ParseList(self._reqatt(node, 'enabled-list'))
845 except KeyError:
846 raise ManifestParseError(
847 'project %s not found for repo-hooks' %
848 (repo_hooks_project))
849
850 if len(repo_hooks_projects) != 1:
851 raise ManifestParseError(
852 'internal error parsing repo-hooks in %s' %
853 (self.manifestFile))
854 self._repo_hooks_project = repo_hooks_projects[0]
855
856 # Store the enabled hooks in the Project object.
857 self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
858 if node.nodeName == 'superproject': 911 if node.nodeName == 'superproject':
859 name = self._reqatt(node, 'name') 912 name = self._reqatt(node, 'name')
860 # There can only be one superproject. 913 # There can only be one superproject.
@@ -872,21 +925,51 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
872 raise ManifestParseError("no remote for superproject %s within %s" % 925 raise ManifestParseError("no remote for superproject %s within %s" %
873 (name, self.manifestFile)) 926 (name, self.manifestFile))
874 self._superproject['remote'] = remote.ToRemoteSpec(name) 927 self._superproject['remote'] = remote.ToRemoteSpec(name)
928 revision = node.getAttribute('revision') or remote.revision
929 if not revision:
930 revision = self._default.revisionExpr
931 if not revision:
932 raise ManifestParseError('no revision for superproject %s within %s' %
933 (name, self.manifestFile))
934 self._superproject['revision'] = revision
935 if node.nodeName == 'contactinfo':
936 bugurl = self._reqatt(node, 'bugurl')
937 # This element can be repeated, later entries will clobber earlier ones.
938 self._contactinfo = ContactInfo(bugurl)
939
875 if node.nodeName == 'remove-project': 940 if node.nodeName == 'remove-project':
876 name = self._reqatt(node, 'name') 941 name = self._reqatt(node, 'name')
877 942
878 if name not in self._projects: 943 if name in self._projects:
944 for p in self._projects[name]:
945 del self._paths[p.relpath]
946 del self._projects[name]
947
948 # If the manifest removes the hooks project, treat it as if it deleted
949 # the repo-hooks element too.
950 if repo_hooks_project == name:
951 repo_hooks_project = None
952 elif not XmlBool(node, 'optional', False):
879 raise ManifestParseError('remove-project element specifies non-existent ' 953 raise ManifestParseError('remove-project element specifies non-existent '
880 'project: %s' % name) 954 'project: %s' % name)
881 955
882 for p in self._projects[name]: 956 # Store repo hooks project information.
883 del self._paths[p.relpath] 957 if repo_hooks_project:
884 del self._projects[name] 958 # Store a reference to the Project.
959 try:
960 repo_hooks_projects = self._projects[repo_hooks_project]
961 except KeyError:
962 raise ManifestParseError(
963 'project %s not found for repo-hooks' %
964 (repo_hooks_project))
885 965
886 # If the manifest removes the hooks project, treat it as if it deleted 966 if len(repo_hooks_projects) != 1:
887 # the repo-hooks element too. 967 raise ManifestParseError(
888 if self._repo_hooks_project and (self._repo_hooks_project.name == name): 968 'internal error parsing repo-hooks in %s' %
889 self._repo_hooks_project = None 969 (self.manifestFile))
970 self._repo_hooks_project = repo_hooks_projects[0]
971 # Store the enabled hooks in the Project object.
972 self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
890 973
891 def _AddMetaProjectMirror(self, m): 974 def _AddMetaProjectMirror(self, m):
892 name = None 975 name = None
@@ -945,7 +1028,14 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
945 if revision == '': 1028 if revision == '':
946 revision = None 1029 revision = None
947 manifestUrl = self.manifestProject.config.GetString('remote.origin.url') 1030 manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
948 return _XmlRemote(name, alias, fetch, pushUrl, manifestUrl, review, revision) 1031
1032 remote = _XmlRemote(name, alias, fetch, pushUrl, manifestUrl, review, revision)
1033
1034 for n in node.childNodes:
1035 if n.nodeName == 'annotation':
1036 self._ParseAnnotation(remote, n)
1037
1038 return remote
949 1039
950 def _ParseDefault(self, node): 1040 def _ParseDefault(self, node):
951 """ 1041 """
@@ -1199,6 +1289,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
1199 if '~' in path: 1289 if '~' in path:
1200 return '~ not allowed (due to 8.3 filenames on Windows filesystems)' 1290 return '~ not allowed (due to 8.3 filenames on Windows filesystems)'
1201 1291
1292 path_codepoints = set(path)
1293
1202 # Some filesystems (like Apple's HFS+) try to normalize Unicode codepoints 1294 # Some filesystems (like Apple's HFS+) try to normalize Unicode codepoints
1203 # which means there are alternative names for ".git". Reject paths with 1295 # which means there are alternative names for ".git". Reject paths with
1204 # these in it as there shouldn't be any reasonable need for them here. 1296 # these in it as there shouldn't be any reasonable need for them here.
@@ -1222,10 +1314,17 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
1222 u'\u206F', # NOMINAL DIGIT SHAPES 1314 u'\u206F', # NOMINAL DIGIT SHAPES
1223 u'\uFEFF', # ZERO WIDTH NO-BREAK SPACE 1315 u'\uFEFF', # ZERO WIDTH NO-BREAK SPACE
1224 } 1316 }
1225 if BAD_CODEPOINTS & set(path): 1317 if BAD_CODEPOINTS & path_codepoints:
1226 # This message is more expansive than reality, but should be fine. 1318 # This message is more expansive than reality, but should be fine.
1227 return 'Unicode combining characters not allowed' 1319 return 'Unicode combining characters not allowed'
1228 1320
1321 # Reject newlines as there shouldn't be any legitmate use for them, they'll
1322 # be confusing to users, and they can easily break tools that expect to be
1323 # able to iterate over newline delimited lists. This even applies to our
1324 # own code like .repo/project.list.
1325 if {'\r', '\n'} & path_codepoints:
1326 return 'Newlines not allowed'
1327
1229 # Assume paths might be used on case-insensitive filesystems. 1328 # Assume paths might be used on case-insensitive filesystems.
1230 path = path.lower() 1329 path = path.lower()
1231 1330
@@ -1303,7 +1402,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
1303 self._ValidateFilePaths('linkfile', src, dest) 1402 self._ValidateFilePaths('linkfile', src, dest)
1304 project.AddLinkFile(src, dest, self.topdir) 1403 project.AddLinkFile(src, dest, self.topdir)
1305 1404
1306 def _ParseAnnotation(self, project, node): 1405 def _ParseAnnotation(self, element, node):
1307 name = self._reqatt(node, 'name') 1406 name = self._reqatt(node, 'name')
1308 value = self._reqatt(node, 'value') 1407 value = self._reqatt(node, 'value')
1309 try: 1408 try:
@@ -1313,7 +1412,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
1313 if keep != "true" and keep != "false": 1412 if keep != "true" and keep != "false":
1314 raise ManifestParseError('optional "keep" attribute must be ' 1413 raise ManifestParseError('optional "keep" attribute must be '
1315 '"true" or "false"') 1414 '"true" or "false"')
1316 project.AddAnnotation(name, value, keep) 1415 element.AddAnnotation(name, value, keep)
1317 1416
1318 def _get_remote(self, node): 1417 def _get_remote(self, node):
1319 name = node.getAttribute('remote') 1418 name = node.getAttribute('remote')
diff --git a/platform_utils.py b/platform_utils.py
index 00c51d9b..0203249a 100644
--- a/platform_utils.py
+++ b/platform_utils.py
@@ -124,31 +124,30 @@ def rename(src, dst):
124 else: 124 else:
125 raise 125 raise
126 else: 126 else:
127 os.rename(src, dst) 127 shutil.move(src, dst)
128 128
129 129
130def remove(path): 130def remove(path, missing_ok=False):
131 """Remove (delete) the file path. This is a replacement for os.remove that 131 """Remove (delete) the file path. This is a replacement for os.remove that
132 allows deleting read-only files on Windows, with support for long paths and 132 allows deleting read-only files on Windows, with support for long paths and
133 for deleting directory symbolic links. 133 for deleting directory symbolic links.
134 134
135 Availability: Unix, Windows.""" 135 Availability: Unix, Windows."""
136 if isWindows(): 136 longpath = _makelongpath(path) if isWindows() else path
137 longpath = _makelongpath(path) 137 try:
138 try: 138 os.remove(longpath)
139 os.remove(longpath) 139 except OSError as e:
140 except OSError as e: 140 if e.errno == errno.EACCES:
141 if e.errno == errno.EACCES: 141 os.chmod(longpath, stat.S_IWRITE)
142 os.chmod(longpath, stat.S_IWRITE) 142 # Directory symbolic links must be deleted with 'rmdir'.
143 # Directory symbolic links must be deleted with 'rmdir'. 143 if islink(longpath) and isdir(longpath):
144 if islink(longpath) and isdir(longpath): 144 os.rmdir(longpath)
145 os.rmdir(longpath)
146 else:
147 os.remove(longpath)
148 else: 145 else:
149 raise 146 os.remove(longpath)
150 else: 147 elif missing_ok and e.errno == errno.ENOENT:
151 os.remove(path) 148 pass
149 else:
150 raise
152 151
153 152
154def walk(top, topdown=True, onerror=None, followlinks=False): 153def walk(top, topdown=True, onerror=None, followlinks=False):
diff --git a/project.py b/project.py
index 992a0c07..5b26b64c 100644
--- a/project.py
+++ b/project.py
@@ -251,13 +251,29 @@ class DiffColoring(Coloring):
251 self.fail = self.printer('fail', fg='red') 251 self.fail = self.printer('fail', fg='red')
252 252
253 253
254class _Annotation(object): 254class Annotation(object):
255 255
256 def __init__(self, name, value, keep): 256 def __init__(self, name, value, keep):
257 self.name = name 257 self.name = name
258 self.value = value 258 self.value = value
259 self.keep = keep 259 self.keep = keep
260 260
261 def __eq__(self, other):
262 if not isinstance(other, Annotation):
263 return False
264 return self.__dict__ == other.__dict__
265
266 def __lt__(self, other):
267 # This exists just so that lists of Annotation objects can be sorted, for
268 # use in comparisons.
269 if not isinstance(other, Annotation):
270 raise ValueError('comparison is not between two Annotation objects')
271 if self.name == other.name:
272 if self.value == other.value:
273 return self.keep < other.keep
274 return self.value < other.value
275 return self.name < other.name
276
261 277
262def _SafeExpandPath(base, subpath, skipfinal=False): 278def _SafeExpandPath(base, subpath, skipfinal=False):
263 """Make sure |subpath| is completely safe under |base|. 279 """Make sure |subpath| is completely safe under |base|.
@@ -503,21 +519,8 @@ class Project(object):
503 self.client = self.manifest = manifest 519 self.client = self.manifest = manifest
504 self.name = name 520 self.name = name
505 self.remote = remote 521 self.remote = remote
506 self.gitdir = gitdir.replace('\\', '/') 522 self.UpdatePaths(relpath, worktree, gitdir, objdir)
507 self.objdir = objdir.replace('\\', '/') 523 self.SetRevision(revisionExpr, revisionId=revisionId)
508 if worktree:
509 self.worktree = os.path.normpath(worktree).replace('\\', '/')
510 else:
511 self.worktree = None
512 self.relpath = relpath
513 self.revisionExpr = revisionExpr
514
515 if revisionId is None \
516 and revisionExpr \
517 and IsId(revisionExpr):
518 self.revisionId = revisionExpr
519 else:
520 self.revisionId = revisionId
521 524
522 self.rebase = rebase 525 self.rebase = rebase
523 self.groups = groups 526 self.groups = groups
@@ -540,16 +543,6 @@ class Project(object):
540 self.copyfiles = [] 543 self.copyfiles = []
541 self.linkfiles = [] 544 self.linkfiles = []
542 self.annotations = [] 545 self.annotations = []
543 self.config = GitConfig.ForRepository(gitdir=self.gitdir,
544 defaults=self.client.globalConfig)
545
546 if self.worktree:
547 self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
548 else:
549 self.work_git = None
550 self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
551 self.bare_ref = GitRefs(gitdir)
552 self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
553 self.dest_branch = dest_branch 546 self.dest_branch = dest_branch
554 self.old_revision = old_revision 547 self.old_revision = old_revision
555 548
@@ -557,6 +550,35 @@ class Project(object):
557 # project containing repo hooks. 550 # project containing repo hooks.
558 self.enabled_repo_hooks = [] 551 self.enabled_repo_hooks = []
559 552
553 def SetRevision(self, revisionExpr, revisionId=None):
554 """Set revisionId based on revision expression and id"""
555 self.revisionExpr = revisionExpr
556 if revisionId is None and revisionExpr and IsId(revisionExpr):
557 self.revisionId = self.revisionExpr
558 else:
559 self.revisionId = revisionId
560
561 def UpdatePaths(self, relpath, worktree, gitdir, objdir):
562 """Update paths used by this project"""
563 self.gitdir = gitdir.replace('\\', '/')
564 self.objdir = objdir.replace('\\', '/')
565 if worktree:
566 self.worktree = os.path.normpath(worktree).replace('\\', '/')
567 else:
568 self.worktree = None
569 self.relpath = relpath
570
571 self.config = GitConfig.ForRepository(gitdir=self.gitdir,
572 defaults=self.manifest.globalConfig)
573
574 if self.worktree:
575 self.work_git = self._GitGetByExec(self, bare=False, gitdir=self.gitdir)
576 else:
577 self.work_git = None
578 self.bare_git = self._GitGetByExec(self, bare=True, gitdir=self.gitdir)
579 self.bare_ref = GitRefs(self.gitdir)
580 self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=self.objdir)
581
560 @property 582 @property
561 def Derived(self): 583 def Derived(self):
562 return self.is_derived 584 return self.is_derived
@@ -1041,15 +1063,16 @@ class Project(object):
1041 verbose=False, 1063 verbose=False,
1042 output_redir=None, 1064 output_redir=None,
1043 is_new=None, 1065 is_new=None,
1044 current_branch_only=False, 1066 current_branch_only=None,
1045 force_sync=False, 1067 force_sync=False,
1046 clone_bundle=True, 1068 clone_bundle=True,
1047 tags=True, 1069 tags=None,
1048 archive=False, 1070 archive=False,
1049 optimized_fetch=False, 1071 optimized_fetch=False,
1050 retry_fetches=0, 1072 retry_fetches=0,
1051 prune=False, 1073 prune=False,
1052 submodules=False, 1074 submodules=False,
1075 ssh_proxy=None,
1053 clone_filter=None, 1076 clone_filter=None,
1054 partial_clone_exclude=set()): 1077 partial_clone_exclude=set()):
1055 """Perform only the network IO portion of the sync process. 1078 """Perform only the network IO portion of the sync process.
@@ -1116,7 +1139,7 @@ class Project(object):
1116 and self._ApplyCloneBundle(initial=is_new, quiet=quiet, verbose=verbose)): 1139 and self._ApplyCloneBundle(initial=is_new, quiet=quiet, verbose=verbose)):
1117 is_new = False 1140 is_new = False
1118 1141
1119 if not current_branch_only: 1142 if current_branch_only is None:
1120 if self.sync_c: 1143 if self.sync_c:
1121 current_branch_only = True 1144 current_branch_only = True
1122 elif not self.manifest._loaded: 1145 elif not self.manifest._loaded:
@@ -1125,8 +1148,8 @@ class Project(object):
1125 elif self.manifest.default.sync_c: 1148 elif self.manifest.default.sync_c:
1126 current_branch_only = True 1149 current_branch_only = True
1127 1150
1128 if not self.sync_tags: 1151 if tags is None:
1129 tags = False 1152 tags = self.sync_tags
1130 1153
1131 if self.clone_depth: 1154 if self.clone_depth:
1132 depth = self.clone_depth 1155 depth = self.clone_depth
@@ -1143,6 +1166,7 @@ class Project(object):
1143 alt_dir=alt_dir, current_branch_only=current_branch_only, 1166 alt_dir=alt_dir, current_branch_only=current_branch_only,
1144 tags=tags, prune=prune, depth=depth, 1167 tags=tags, prune=prune, depth=depth,
1145 submodules=submodules, force_sync=force_sync, 1168 submodules=submodules, force_sync=force_sync,
1169 ssh_proxy=ssh_proxy,
1146 clone_filter=clone_filter, retry_fetches=retry_fetches): 1170 clone_filter=clone_filter, retry_fetches=retry_fetches):
1147 return False 1171 return False
1148 1172
@@ -1164,10 +1188,8 @@ class Project(object):
1164 self._InitMRef() 1188 self._InitMRef()
1165 else: 1189 else:
1166 self._InitMirrorHead() 1190 self._InitMirrorHead()
1167 try: 1191 platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'),
1168 platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD')) 1192 missing_ok=True)
1169 except OSError:
1170 pass
1171 return True 1193 return True
1172 1194
1173 def PostRepoUpgrade(self): 1195 def PostRepoUpgrade(self):
@@ -1214,6 +1236,9 @@ class Project(object):
1214 (self.revisionExpr, self.name)) 1236 (self.revisionExpr, self.name))
1215 1237
1216 def SetRevisionId(self, revisionId): 1238 def SetRevisionId(self, revisionId):
1239 if self.revisionExpr:
1240 self.upstream = self.revisionExpr
1241
1217 self.revisionId = revisionId 1242 self.revisionId = revisionId
1218 1243
1219 def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False): 1244 def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False):
@@ -1443,7 +1468,7 @@ class Project(object):
1443 self.linkfiles.append(_LinkFile(self.worktree, src, topdir, dest)) 1468 self.linkfiles.append(_LinkFile(self.worktree, src, topdir, dest))
1444 1469
1445 def AddAnnotation(self, name, value, keep): 1470 def AddAnnotation(self, name, value, keep):
1446 self.annotations.append(_Annotation(name, value, keep)) 1471 self.annotations.append(Annotation(name, value, keep))
1447 1472
1448 def DownloadPatchSet(self, change_id, patch_id): 1473 def DownloadPatchSet(self, change_id, patch_id):
1449 """Download a single patch set of a single change to FETCH_HEAD. 1474 """Download a single patch set of a single change to FETCH_HEAD.
@@ -1962,6 +1987,11 @@ class Project(object):
1962 # throws an error. 1987 # throws an error.
1963 self.bare_git.rev_list('-1', '--missing=allow-any', 1988 self.bare_git.rev_list('-1', '--missing=allow-any',
1964 '%s^0' % self.revisionExpr, '--') 1989 '%s^0' % self.revisionExpr, '--')
1990 if self.upstream:
1991 rev = self.GetRemote(self.remote.name).ToLocal(self.upstream)
1992 self.bare_git.rev_list('-1', '--missing=allow-any',
1993 '%s^0' % rev, '--')
1994 self.bare_git.merge_base('--is-ancestor', self.revisionExpr, rev)
1965 return True 1995 return True
1966 except GitError: 1996 except GitError:
1967 # There is no such persistent revision. We have to fetch it. 1997 # There is no such persistent revision. We have to fetch it.
@@ -1991,6 +2021,7 @@ class Project(object):
1991 prune=False, 2021 prune=False,
1992 depth=None, 2022 depth=None,
1993 submodules=False, 2023 submodules=False,
2024 ssh_proxy=None,
1994 force_sync=False, 2025 force_sync=False,
1995 clone_filter=None, 2026 clone_filter=None,
1996 retry_fetches=2, 2027 retry_fetches=2,
@@ -2038,16 +2069,14 @@ class Project(object):
2038 if not name: 2069 if not name:
2039 name = self.remote.name 2070 name = self.remote.name
2040 2071
2041 ssh_proxy = False
2042 remote = self.GetRemote(name) 2072 remote = self.GetRemote(name)
2043 if remote.PreConnectFetch(): 2073 if not remote.PreConnectFetch(ssh_proxy):
2044 ssh_proxy = True 2074 ssh_proxy = None
2045 2075
2046 if initial: 2076 if initial:
2047 if alt_dir and 'objects' == os.path.basename(alt_dir): 2077 if alt_dir and 'objects' == os.path.basename(alt_dir):
2048 ref_dir = os.path.dirname(alt_dir) 2078 ref_dir = os.path.dirname(alt_dir)
2049 packed_refs = os.path.join(self.gitdir, 'packed-refs') 2079 packed_refs = os.path.join(self.gitdir, 'packed-refs')
2050 remote = self.GetRemote(name)
2051 2080
2052 all_refs = self.bare_ref.all 2081 all_refs = self.bare_ref.all
2053 ids = set(all_refs.values()) 2082 ids = set(all_refs.values())
@@ -2134,6 +2163,8 @@ class Project(object):
2134 # Shallow checkout of a specific commit, fetch from that commit and not 2163 # Shallow checkout of a specific commit, fetch from that commit and not
2135 # the heads only as the commit might be deeper in the history. 2164 # the heads only as the commit might be deeper in the history.
2136 spec.append(branch) 2165 spec.append(branch)
2166 if self.upstream:
2167 spec.append(self.upstream)
2137 else: 2168 else:
2138 if is_sha1: 2169 if is_sha1:
2139 branch = self.upstream 2170 branch = self.upstream
@@ -2191,7 +2222,7 @@ class Project(object):
2191 ret = prunecmd.Wait() 2222 ret = prunecmd.Wait()
2192 if ret: 2223 if ret:
2193 break 2224 break
2194 output_redir.write('retrying fetch after pruning remote branches') 2225 print('retrying fetch after pruning remote branches', file=output_redir)
2195 # Continue right away so we don't sleep as we shouldn't need to. 2226 # Continue right away so we don't sleep as we shouldn't need to.
2196 continue 2227 continue
2197 elif current_branch_only and is_sha1 and ret == 128: 2228 elif current_branch_only and is_sha1 and ret == 128:
@@ -2204,10 +2235,11 @@ class Project(object):
2204 break 2235 break
2205 2236
2206 # Figure out how long to sleep before the next attempt, if there is one. 2237 # Figure out how long to sleep before the next attempt, if there is one.
2207 if not verbose: 2238 if not verbose and gitcmd.stdout:
2208 output_redir.write('\n%s:\n%s' % (self.name, gitcmd.stdout)) 2239 print('\n%s:\n%s' % (self.name, gitcmd.stdout), end='', file=output_redir)
2209 if try_n < retry_fetches - 1: 2240 if try_n < retry_fetches - 1:
2210 output_redir.write('sleeping %s seconds before retrying' % retry_cur_sleep) 2241 print('%s: sleeping %s seconds before retrying' % (self.name, retry_cur_sleep),
2242 file=output_redir)
2211 time.sleep(retry_cur_sleep) 2243 time.sleep(retry_cur_sleep)
2212 retry_cur_sleep = min(retry_exp_factor * retry_cur_sleep, 2244 retry_cur_sleep = min(retry_exp_factor * retry_cur_sleep,
2213 MAXIMUM_RETRY_SLEEP_SEC) 2245 MAXIMUM_RETRY_SLEEP_SEC)
@@ -2233,7 +2265,7 @@ class Project(object):
2233 name=name, quiet=quiet, verbose=verbose, output_redir=output_redir, 2265 name=name, quiet=quiet, verbose=verbose, output_redir=output_redir,
2234 current_branch_only=current_branch_only and depth, 2266 current_branch_only=current_branch_only and depth,
2235 initial=False, alt_dir=alt_dir, 2267 initial=False, alt_dir=alt_dir,
2236 depth=None, clone_filter=clone_filter) 2268 depth=None, ssh_proxy=ssh_proxy, clone_filter=clone_filter)
2237 2269
2238 return ok 2270 return ok
2239 2271
@@ -2279,15 +2311,12 @@ class Project(object):
2279 cmd.append('+refs/tags/*:refs/tags/*') 2311 cmd.append('+refs/tags/*:refs/tags/*')
2280 2312
2281 ok = GitCommand(self, cmd, bare=True).Wait() == 0 2313 ok = GitCommand(self, cmd, bare=True).Wait() == 0
2282 if os.path.exists(bundle_dst): 2314 platform_utils.remove(bundle_dst, missing_ok=True)
2283 platform_utils.remove(bundle_dst) 2315 platform_utils.remove(bundle_tmp, missing_ok=True)
2284 if os.path.exists(bundle_tmp):
2285 platform_utils.remove(bundle_tmp)
2286 return ok 2316 return ok
2287 2317
2288 def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet, verbose): 2318 def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet, verbose):
2289 if os.path.exists(dstPath): 2319 platform_utils.remove(dstPath, missing_ok=True)
2290 platform_utils.remove(dstPath)
2291 2320
2292 cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location'] 2321 cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
2293 if quiet: 2322 if quiet:
@@ -2438,14 +2467,6 @@ class Project(object):
2438 self.bare_objdir.init() 2467 self.bare_objdir.init()
2439 2468
2440 if self.use_git_worktrees: 2469 if self.use_git_worktrees:
2441 # Set up the m/ space to point to the worktree-specific ref space.
2442 # We'll update the worktree-specific ref space on each checkout.
2443 if self.manifest.branch:
2444 self.bare_git.symbolic_ref(
2445 '-m', 'redirecting to worktree scope',
2446 R_M + self.manifest.branch,
2447 R_WORKTREE_M + self.manifest.branch)
2448
2449 # Enable per-worktree config file support if possible. This is more a 2470 # Enable per-worktree config file support if possible. This is more a
2450 # nice-to-have feature for users rather than a hard requirement. 2471 # nice-to-have feature for users rather than a hard requirement.
2451 if git_require((2, 20, 0)): 2472 if git_require((2, 20, 0)):
@@ -2582,6 +2603,14 @@ class Project(object):
2582 def _InitMRef(self): 2603 def _InitMRef(self):
2583 if self.manifest.branch: 2604 if self.manifest.branch:
2584 if self.use_git_worktrees: 2605 if self.use_git_worktrees:
2606 # Set up the m/ space to point to the worktree-specific ref space.
2607 # We'll update the worktree-specific ref space on each checkout.
2608 ref = R_M + self.manifest.branch
2609 if not self.bare_ref.symref(ref):
2610 self.bare_git.symbolic_ref(
2611 '-m', 'redirecting to worktree scope',
2612 ref, R_WORKTREE_M + self.manifest.branch)
2613
2585 # We can't update this ref with git worktrees until it exists. 2614 # We can't update this ref with git worktrees until it exists.
2586 # We'll wait until the initial checkout to set it. 2615 # We'll wait until the initial checkout to set it.
2587 if not os.path.exists(self.worktree): 2616 if not os.path.exists(self.worktree):
@@ -2711,10 +2740,7 @@ class Project(object):
2711 # If the source file doesn't exist, ensure the destination 2740 # If the source file doesn't exist, ensure the destination
2712 # file doesn't either. 2741 # file doesn't either.
2713 if name in symlink_files and not os.path.lexists(src): 2742 if name in symlink_files and not os.path.lexists(src):
2714 try: 2743 platform_utils.remove(dst, missing_ok=True)
2715 platform_utils.remove(dst)
2716 except OSError:
2717 pass
2718 2744
2719 except OSError as e: 2745 except OSError as e:
2720 if e.errno == errno.EPERM: 2746 if e.errno == errno.EPERM:
diff --git a/release/update-manpages b/release/update-manpages
new file mode 100755
index 00000000..ddbce0cc
--- /dev/null
+++ b/release/update-manpages
@@ -0,0 +1,102 @@
1#!/usr/bin/env python3
2# Copyright (C) 2021 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16"""Helper tool for generating manual page for all repo commands.
17
18This is intended to be run before every official Repo release.
19"""
20
21from pathlib import Path
22from functools import partial
23import argparse
24import multiprocessing
25import os
26import re
27import shutil
28import subprocess
29import sys
30import tempfile
31
32TOPDIR = Path(__file__).resolve().parent.parent
33MANDIR = TOPDIR.joinpath('man')
34
35# Load repo local modules.
36sys.path.insert(0, str(TOPDIR))
37from git_command import RepoSourceVersion
38import subcmds
39
40def worker(cmd, **kwargs):
41 subprocess.run(cmd, **kwargs)
42
43def main(argv):
44 parser = argparse.ArgumentParser(description=__doc__)
45 opts = parser.parse_args(argv)
46
47 if not shutil.which('help2man'):
48 sys.exit('Please install help2man to continue.')
49
50 # Let repo know we're generating man pages so it can avoid some dynamic
51 # behavior (like probing active number of CPUs). We use a weird name &
52 # value to make it less likely for users to set this var themselves.
53 os.environ['_REPO_GENERATE_MANPAGES_'] = ' indeed! '
54
55 # "repo branch" is an alias for "repo branches".
56 del subcmds.all_commands['branch']
57 (MANDIR / 'repo-branch.1').write_text('.so man1/repo-branches.1')
58
59 version = RepoSourceVersion()
60 cmdlist = [['help2man', '-N', '-n', f'repo {cmd} - manual page for repo {cmd}',
61 '-S', f'repo {cmd}', '-m', 'Repo Manual', f'--version-string={version}',
62 '-o', MANDIR.joinpath(f'repo-{cmd}.1.tmp'), TOPDIR.joinpath('repo'),
63 '-h', f'help {cmd}'] for cmd in subcmds.all_commands]
64 cmdlist.append(['help2man', '-N', '-n', 'repository management tool built on top of git',
65 '-S', 'repo', '-m', 'Repo Manual', f'--version-string={version}',
66 '-o', MANDIR.joinpath('repo.1.tmp'), TOPDIR.joinpath('repo'),
67 '-h', '--help-all'])
68
69 with tempfile.TemporaryDirectory() as tempdir:
70 repo_dir = Path(tempdir) / '.repo'
71 repo_dir.mkdir()
72 (repo_dir / 'repo').symlink_to(TOPDIR)
73
74 # Run all cmd in parallel, and wait for them to finish.
75 with multiprocessing.Pool() as pool:
76 pool.map(partial(worker, cwd=tempdir, check=True), cmdlist)
77
78 regex = (
79 (r'(It was generated by help2man) [0-9.]+', '\g<1>.'),
80 (r'^\.IP\n(.*:)\n', '.SS \g<1>\n'),
81 (r'^\.PP\nDescription', '.SH DETAILS'),
82 )
83 for tmp_path in MANDIR.glob('*.1.tmp'):
84 path = tmp_path.parent / tmp_path.stem
85 old_data = path.read_text() if path.exists() else ''
86
87 data = tmp_path.read_text()
88 tmp_path.unlink()
89
90 for pattern, replacement in regex:
91 data = re.sub(pattern, replacement, data, flags=re.M)
92
93 # If the only thing that changed was the date, don't refresh. This avoids
94 # a lot of noise when only one file actually updates.
95 old_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', old_data, flags=re.M)
96 new_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', data, flags=re.M)
97 if old_data != new_data:
98 path.write_text(data)
99
100
101if __name__ == '__main__':
102 sys.exit(main(sys.argv[1:]))
diff --git a/repo b/repo
index d9c97de1..4cddbf1e 100755
--- a/repo
+++ b/repo
@@ -117,7 +117,7 @@ def check_python_version():
117 117
118 # If the python3 version looks like it's new enough, give it a try. 118 # If the python3 version looks like it's new enough, give it a try.
119 if (python3_ver and python3_ver >= MIN_PYTHON_VERSION_HARD 119 if (python3_ver and python3_ver >= MIN_PYTHON_VERSION_HARD
120 and python3_ver != (major, minor)): 120 and python3_ver != (major, minor)):
121 reexec('python3') 121 reexec('python3')
122 122
123 # We're still here, so diagnose things for the user. 123 # We're still here, so diagnose things for the user.
@@ -145,9 +145,11 @@ if not REPO_URL:
145REPO_REV = os.environ.get('REPO_REV') 145REPO_REV = os.environ.get('REPO_REV')
146if not REPO_REV: 146if not REPO_REV:
147 REPO_REV = 'stable' 147 REPO_REV = 'stable'
148# URL to file bug reports for repo tool issues.
149BUG_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue'
148 150
149# increment this whenever we make important changes to this script 151# increment this whenever we make important changes to this script
150VERSION = (2, 14) 152VERSION = (2, 17)
151 153
152# increment this if the MAINTAINER_KEYS block is modified 154# increment this if the MAINTAINER_KEYS block is modified
153KEYRING_VERSION = (2, 3) 155KEYRING_VERSION = (2, 3)
@@ -310,6 +312,10 @@ def InitParser(parser, gitc_init=False):
310 metavar='PLATFORM') 312 metavar='PLATFORM')
311 group.add_option('--submodules', action='store_true', 313 group.add_option('--submodules', action='store_true',
312 help='sync any submodules associated with the manifest repo') 314 help='sync any submodules associated with the manifest repo')
315 group.add_option('--standalone-manifest', action='store_true',
316 help='download the manifest as a static file '
317 'rather then create a git checkout of '
318 'the manifest repo')
313 319
314 # Options that only affect manifest project, and not any of the projects 320 # Options that only affect manifest project, and not any of the projects
315 # specified in the manifest itself. 321 # specified in the manifest itself.
@@ -322,8 +328,14 @@ def InitParser(parser, gitc_init=False):
322 group.add_option(*cbr_opts, 328 group.add_option(*cbr_opts,
323 dest='current_branch_only', action='store_true', 329 dest='current_branch_only', action='store_true',
324 help='fetch only current manifest branch from server') 330 help='fetch only current manifest branch from server')
331 group.add_option('--no-current-branch',
332 dest='current_branch_only', action='store_false',
333 help='fetch all manifest branches from server')
334 group.add_option('--tags',
335 action='store_true',
336 help='fetch tags in the manifest')
325 group.add_option('--no-tags', 337 group.add_option('--no-tags',
326 dest='tags', default=True, action='store_false', 338 dest='tags', action='store_false',
327 help="don't fetch tags in the manifest") 339 help="don't fetch tags in the manifest")
328 340
329 # These are fundamentally different ways of structuring the checkout. 341 # These are fundamentally different ways of structuring the checkout.
@@ -851,11 +863,10 @@ def _DownloadBundle(url, cwd, quiet, verbose):
851 try: 863 try:
852 r = urllib.request.urlopen(url) 864 r = urllib.request.urlopen(url)
853 except urllib.error.HTTPError as e: 865 except urllib.error.HTTPError as e:
854 if e.code in [401, 403, 404, 501]: 866 if e.code not in [400, 401, 403, 404, 501]:
855 return False 867 print('warning: Cannot get %s' % url, file=sys.stderr)
856 print('fatal: Cannot get %s' % url, file=sys.stderr) 868 print('warning: HTTP error %s' % e.code, file=sys.stderr)
857 print('fatal: HTTP error %s' % e.code, file=sys.stderr) 869 return False
858 raise CloneFailure()
859 except urllib.error.URLError as e: 870 except urllib.error.URLError as e:
860 print('fatal: Cannot get %s' % url, file=sys.stderr) 871 print('fatal: Cannot get %s' % url, file=sys.stderr)
861 print('fatal: error %s' % e.reason, file=sys.stderr) 872 print('fatal: error %s' % e.reason, file=sys.stderr)
@@ -1171,6 +1182,7 @@ The most commonly used repo commands are:
1171 1182
1172For access to the full online help, install repo ("repo init"). 1183For access to the full online help, install repo ("repo init").
1173""") 1184""")
1185 print('Bug reports:', BUG_URL)
1174 sys.exit(0) 1186 sys.exit(0)
1175 1187
1176 1188
@@ -1204,6 +1216,7 @@ def _Version():
1204 print('OS %s %s (%s)' % (uname.system, uname.release, uname.version)) 1216 print('OS %s %s (%s)' % (uname.system, uname.release, uname.version))
1205 print('CPU %s (%s)' % 1217 print('CPU %s (%s)' %
1206 (uname.machine, uname.processor if uname.processor else 'unknown')) 1218 (uname.machine, uname.processor if uname.processor else 'unknown'))
1219 print('Bug reports:', BUG_URL)
1207 sys.exit(0) 1220 sys.exit(0)
1208 1221
1209 1222
diff --git a/requirements.json b/requirements.json
index 86b9a46c..cb55cd25 100644
--- a/requirements.json
+++ b/requirements.json
@@ -38,9 +38,9 @@
38 # Supported Python versions. 38 # Supported Python versions.
39 # 39 #
40 # python-3.6 is in Ubuntu Bionic. 40 # python-3.6 is in Ubuntu Bionic.
41 # python-3.5 is in Debian Stretch. 41 # python-3.7 is in Debian Buster.
42 "python": { 42 "python": {
43 "hard": [3, 5], 43 "hard": [3, 6],
44 "soft": [3, 6] 44 "soft": [3, 6]
45 }, 45 },
46 46
diff --git a/run_tests b/run_tests
index 6c6f8594..573dd446 100755
--- a/run_tests
+++ b/run_tests
@@ -24,6 +24,10 @@ import sys
24 24
25def find_pytest(): 25def find_pytest():
26 """Try to locate a good version of pytest.""" 26 """Try to locate a good version of pytest."""
27 # If we're in a virtualenv, assume that it's provided the right pytest.
28 if 'VIRTUAL_ENV' in os.environ:
29 return 'pytest'
30
27 # Use the Python 3 version if available. 31 # Use the Python 3 version if available.
28 ret = shutil.which('pytest-3') 32 ret = shutil.which('pytest-3')
29 if ret: 33 if ret:
diff --git a/setup.py b/setup.py
index 9d0ff5f9..17aeae22 100755
--- a/setup.py
+++ b/setup.py
@@ -56,6 +56,6 @@ setuptools.setup(
56 'Programming Language :: Python :: 3 :: Only', 56 'Programming Language :: Python :: 3 :: Only',
57 'Topic :: Software Development :: Version Control :: Git', 57 'Topic :: Software Development :: Version Control :: Git',
58 ], 58 ],
59 python_requires='>=3.5', 59 python_requires='>=3.6',
60 packages=['subcmds'], 60 packages=['subcmds'],
61) 61)
diff --git a/ssh.py b/ssh.py
new file mode 100644
index 00000000..0ae8d120
--- /dev/null
+++ b/ssh.py
@@ -0,0 +1,277 @@
1# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""Common SSH management logic."""
16
17import functools
18import multiprocessing
19import os
20import re
21import signal
22import subprocess
23import sys
24import tempfile
25import time
26
27import platform_utils
28from repo_trace import Trace
29
30
31PROXY_PATH = os.path.join(os.path.dirname(__file__), 'git_ssh')
32
33
34def _run_ssh_version():
35 """run ssh -V to display the version number"""
36 return subprocess.check_output(['ssh', '-V'], stderr=subprocess.STDOUT).decode()
37
38
39def _parse_ssh_version(ver_str=None):
40 """parse a ssh version string into a tuple"""
41 if ver_str is None:
42 ver_str = _run_ssh_version()
43 m = re.match(r'^OpenSSH_([0-9.]+)(p[0-9]+)?\s', ver_str)
44 if m:
45 return tuple(int(x) for x in m.group(1).split('.'))
46 else:
47 return ()
48
49
50@functools.lru_cache(maxsize=None)
51def version():
52 """return ssh version as a tuple"""
53 try:
54 return _parse_ssh_version()
55 except subprocess.CalledProcessError:
56 print('fatal: unable to detect ssh version', file=sys.stderr)
57 sys.exit(1)
58
59
60URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
61URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
62
63
64class ProxyManager:
65 """Manage various ssh clients & masters that we spawn.
66
67 This will take care of sharing state between multiprocessing children, and
68 make sure that if we crash, we don't leak any of the ssh sessions.
69
70 The code should work with a single-process scenario too, and not add too much
71 overhead due to the manager.
72 """
73
74 # Path to the ssh program to run which will pass our master settings along.
75 # Set here more as a convenience API.
76 proxy = PROXY_PATH
77
78 def __init__(self, manager):
79 # Protect access to the list of active masters.
80 self._lock = multiprocessing.Lock()
81 # List of active masters (pid). These will be spawned on demand, and we are
82 # responsible for shutting them all down at the end.
83 self._masters = manager.list()
84 # Set of active masters indexed by "host:port" information.
85 # The value isn't used, but multiprocessing doesn't provide a set class.
86 self._master_keys = manager.dict()
87 # Whether ssh masters are known to be broken, so we give up entirely.
88 self._master_broken = manager.Value('b', False)
89 # List of active ssh sesssions. Clients will be added & removed as
90 # connections finish, so this list is just for safety & cleanup if we crash.
91 self._clients = manager.list()
92 # Path to directory for holding master sockets.
93 self._sock_path = None
94
95 def __enter__(self):
96 """Enter a new context."""
97 return self
98
99 def __exit__(self, exc_type, exc_value, traceback):
100 """Exit a context & clean up all resources."""
101 self.close()
102
103 def add_client(self, proc):
104 """Track a new ssh session."""
105 self._clients.append(proc.pid)
106
107 def remove_client(self, proc):
108 """Remove a completed ssh session."""
109 try:
110 self._clients.remove(proc.pid)
111 except ValueError:
112 pass
113
114 def add_master(self, proc):
115 """Track a new master connection."""
116 self._masters.append(proc.pid)
117
118 def _terminate(self, procs):
119 """Kill all |procs|."""
120 for pid in procs:
121 try:
122 os.kill(pid, signal.SIGTERM)
123 os.waitpid(pid, 0)
124 except OSError:
125 pass
126
127 # The multiprocessing.list() API doesn't provide many standard list()
128 # methods, so we have to manually clear the list.
129 while True:
130 try:
131 procs.pop(0)
132 except:
133 break
134
135 def close(self):
136 """Close this active ssh session.
137
138 Kill all ssh clients & masters we created, and nuke the socket dir.
139 """
140 self._terminate(self._clients)
141 self._terminate(self._masters)
142
143 d = self.sock(create=False)
144 if d:
145 try:
146 platform_utils.rmdir(os.path.dirname(d))
147 except OSError:
148 pass
149
150 def _open_unlocked(self, host, port=None):
151 """Make sure a ssh master session exists for |host| & |port|.
152
153 If one doesn't exist already, we'll create it.
154
155 We won't grab any locks, so the caller has to do that. This helps keep the
156 business logic of actually creating the master separate from grabbing locks.
157 """
158 # Check to see whether we already think that the master is running; if we
159 # think it's already running, return right away.
160 if port is not None:
161 key = '%s:%s' % (host, port)
162 else:
163 key = host
164
165 if key in self._master_keys:
166 return True
167
168 if self._master_broken.value or 'GIT_SSH' in os.environ:
169 # Failed earlier, so don't retry.
170 return False
171
172 # We will make two calls to ssh; this is the common part of both calls.
173 command_base = ['ssh', '-o', 'ControlPath %s' % self.sock(), host]
174 if port is not None:
175 command_base[1:1] = ['-p', str(port)]
176
177 # Since the key wasn't in _master_keys, we think that master isn't running.
178 # ...but before actually starting a master, we'll double-check. This can
179 # be important because we can't tell that that 'git@myhost.com' is the same
180 # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
181 check_command = command_base + ['-O', 'check']
182 try:
183 Trace(': %s', ' '.join(check_command))
184 check_process = subprocess.Popen(check_command,
185 stdout=subprocess.PIPE,
186 stderr=subprocess.PIPE)
187 check_process.communicate() # read output, but ignore it...
188 isnt_running = check_process.wait()
189
190 if not isnt_running:
191 # Our double-check found that the master _was_ infact running. Add to
192 # the list of keys.
193 self._master_keys[key] = True
194 return True
195 except Exception:
196 # Ignore excpetions. We we will fall back to the normal command and print
197 # to the log there.
198 pass
199
200 command = command_base[:1] + ['-M', '-N'] + command_base[1:]
201 try:
202 Trace(': %s', ' '.join(command))
203 p = subprocess.Popen(command)
204 except Exception as e:
205 self._master_broken.value = True
206 print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
207 % (host, port, str(e)), file=sys.stderr)
208 return False
209
210 time.sleep(1)
211 ssh_died = (p.poll() is not None)
212 if ssh_died:
213 return False
214
215 self.add_master(p)
216 self._master_keys[key] = True
217 return True
218
219 def _open(self, host, port=None):
220 """Make sure a ssh master session exists for |host| & |port|.
221
222 If one doesn't exist already, we'll create it.
223
224 This will obtain any necessary locks to avoid inter-process races.
225 """
226 # Bail before grabbing the lock if we already know that we aren't going to
227 # try creating new masters below.
228 if sys.platform in ('win32', 'cygwin'):
229 return False
230
231 # Acquire the lock. This is needed to prevent opening multiple masters for
232 # the same host when we're running "repo sync -jN" (for N > 1) _and_ the
233 # manifest <remote fetch="ssh://xyz"> specifies a different host from the
234 # one that was passed to repo init.
235 with self._lock:
236 return self._open_unlocked(host, port)
237
238 def preconnect(self, url):
239 """If |uri| will create a ssh connection, setup the ssh master for it."""
240 m = URI_ALL.match(url)
241 if m:
242 scheme = m.group(1)
243 host = m.group(2)
244 if ':' in host:
245 host, port = host.split(':')
246 else:
247 port = None
248 if scheme in ('ssh', 'git+ssh', 'ssh+git'):
249 return self._open(host, port)
250 return False
251
252 m = URI_SCP.match(url)
253 if m:
254 host = m.group(1)
255 return self._open(host)
256
257 return False
258
259 def sock(self, create=True):
260 """Return the path to the ssh socket dir.
261
262 This has all the master sockets so clients can talk to them.
263 """
264 if self._sock_path is None:
265 if not create:
266 return None
267 tmp_dir = '/tmp'
268 if not os.path.exists(tmp_dir):
269 tmp_dir = tempfile.gettempdir()
270 if version() < (6, 7):
271 tokens = '%r@%h:%p'
272 else:
273 tokens = '%C' # hash of %l%h%p%r
274 self._sock_path = os.path.join(
275 tempfile.mkdtemp('', 'ssh-', tmp_dir),
276 'master-' + tokens)
277 return self._sock_path
diff --git a/subcmds/abandon.py b/subcmds/abandon.py
index c7c127d6..85d85f5a 100644
--- a/subcmds/abandon.py
+++ b/subcmds/abandon.py
@@ -23,7 +23,7 @@ from progress import Progress
23 23
24 24
25class Abandon(Command): 25class Abandon(Command):
26 common = True 26 COMMON = True
27 helpSummary = "Permanently abandon a development branch" 27 helpSummary = "Permanently abandon a development branch"
28 helpUsage = """ 28 helpUsage = """
29%prog [--all | <branchname>] [<project>...] 29%prog [--all | <branchname>] [<project>...]
diff --git a/subcmds/branches.py b/subcmds/branches.py
index 2dc102bb..6d975ed4 100644
--- a/subcmds/branches.py
+++ b/subcmds/branches.py
@@ -62,7 +62,7 @@ class BranchInfo(object):
62 62
63 63
64class Branches(Command): 64class Branches(Command):
65 common = True 65 COMMON = True
66 helpSummary = "View current topic branches" 66 helpSummary = "View current topic branches"
67 helpUsage = """ 67 helpUsage = """
68%prog [<project>...] 68%prog [<project>...]
diff --git a/subcmds/checkout.py b/subcmds/checkout.py
index 4d8009b1..9b429489 100644
--- a/subcmds/checkout.py
+++ b/subcmds/checkout.py
@@ -20,7 +20,7 @@ from progress import Progress
20 20
21 21
22class Checkout(Command): 22class Checkout(Command):
23 common = True 23 COMMON = True
24 helpSummary = "Checkout a branch for development" 24 helpSummary = "Checkout a branch for development"
25 helpUsage = """ 25 helpUsage = """
26%prog <branchname> [<project>...] 26%prog <branchname> [<project>...]
diff --git a/subcmds/cherry_pick.py b/subcmds/cherry_pick.py
index fc4998c3..7bd858bf 100644
--- a/subcmds/cherry_pick.py
+++ b/subcmds/cherry_pick.py
@@ -21,7 +21,7 @@ CHANGE_ID_RE = re.compile(r'^\s*Change-Id: I([0-9a-f]{40})\s*$')
21 21
22 22
23class CherryPick(Command): 23class CherryPick(Command):
24 common = True 24 COMMON = True
25 helpSummary = "Cherry-pick a change." 25 helpSummary = "Cherry-pick a change."
26 helpUsage = """ 26 helpUsage = """
27%prog <sha1> 27%prog <sha1>
diff --git a/subcmds/diff.py b/subcmds/diff.py
index 4966bb1a..00a7ec29 100644
--- a/subcmds/diff.py
+++ b/subcmds/diff.py
@@ -19,7 +19,7 @@ from command import DEFAULT_LOCAL_JOBS, PagedCommand
19 19
20 20
21class Diff(PagedCommand): 21class Diff(PagedCommand):
22 common = True 22 COMMON = True
23 helpSummary = "Show changes between commit and working tree" 23 helpSummary = "Show changes between commit and working tree"
24 helpUsage = """ 24 helpUsage = """
25%prog [<project>...] 25%prog [<project>...]
@@ -33,7 +33,7 @@ to the Unix 'patch' command.
33 def _Options(self, p): 33 def _Options(self, p):
34 p.add_option('-u', '--absolute', 34 p.add_option('-u', '--absolute',
35 dest='absolute', action='store_true', 35 dest='absolute', action='store_true',
36 help='Paths are relative to the repository root') 36 help='paths are relative to the repository root')
37 37
38 def _ExecuteOne(self, absolute, project): 38 def _ExecuteOne(self, absolute, project):
39 """Obtains the diff for a specific project. 39 """Obtains the diff for a specific project.
diff --git a/subcmds/diffmanifests.py b/subcmds/diffmanifests.py
index 392e5972..f6cc30a2 100644
--- a/subcmds/diffmanifests.py
+++ b/subcmds/diffmanifests.py
@@ -31,7 +31,7 @@ class Diffmanifests(PagedCommand):
31 deeper level. 31 deeper level.
32 """ 32 """
33 33
34 common = True 34 COMMON = True
35 helpSummary = "Manifest diff utility" 35 helpSummary = "Manifest diff utility"
36 helpUsage = """%prog manifest1.xml [manifest2.xml] [options]""" 36 helpUsage = """%prog manifest1.xml [manifest2.xml] [options]"""
37 37
@@ -68,10 +68,10 @@ synced and their revisions won't be found.
68 def _Options(self, p): 68 def _Options(self, p):
69 p.add_option('--raw', 69 p.add_option('--raw',
70 dest='raw', action='store_true', 70 dest='raw', action='store_true',
71 help='Display raw diff.') 71 help='display raw diff')
72 p.add_option('--no-color', 72 p.add_option('--no-color',
73 dest='color', action='store_false', default=True, 73 dest='color', action='store_false', default=True,
74 help='does not display the diff in color.') 74 help='does not display the diff in color')
75 p.add_option('--pretty-format', 75 p.add_option('--pretty-format',
76 dest='pretty_format', action='store', 76 dest='pretty_format', action='store',
77 metavar='<FORMAT>', 77 metavar='<FORMAT>',
diff --git a/subcmds/download.py b/subcmds/download.py
index 81d997e0..523f25e0 100644
--- a/subcmds/download.py
+++ b/subcmds/download.py
@@ -22,7 +22,7 @@ CHANGE_RE = re.compile(r'^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$')
22 22
23 23
24class Download(Command): 24class Download(Command):
25 common = True 25 COMMON = True
26 helpSummary = "Download and checkout a change" 26 helpSummary = "Download and checkout a change"
27 helpUsage = """ 27 helpUsage = """
28%prog {[project] change[/patchset]}... 28%prog {[project] change[/patchset]}...
diff --git a/subcmds/forall.py b/subcmds/forall.py
index 4a631fb7..7c1dea9e 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -41,7 +41,7 @@ class ForallColoring(Coloring):
41 41
42 42
43class Forall(Command, MirrorSafeCommand): 43class Forall(Command, MirrorSafeCommand):
44 common = False 44 COMMON = False
45 helpSummary = "Run a shell command in each project" 45 helpSummary = "Run a shell command in each project"
46 helpUsage = """ 46 helpUsage = """
47%prog [<project>...] -c <command> [<arg>...] 47%prog [<project>...] -c <command> [<arg>...]
@@ -131,30 +131,30 @@ without iterating through the remaining projects.
131 def _Options(self, p): 131 def _Options(self, p):
132 p.add_option('-r', '--regex', 132 p.add_option('-r', '--regex',
133 dest='regex', action='store_true', 133 dest='regex', action='store_true',
134 help="Execute the command only on projects matching regex or wildcard expression") 134 help='execute the command only on projects matching regex or wildcard expression')
135 p.add_option('-i', '--inverse-regex', 135 p.add_option('-i', '--inverse-regex',
136 dest='inverse_regex', action='store_true', 136 dest='inverse_regex', action='store_true',
137 help="Execute the command only on projects not matching regex or " 137 help='execute the command only on projects not matching regex or '
138 "wildcard expression") 138 'wildcard expression')
139 p.add_option('-g', '--groups', 139 p.add_option('-g', '--groups',
140 dest='groups', 140 dest='groups',
141 help="Execute the command only on projects matching the specified groups") 141 help='execute the command only on projects matching the specified groups')
142 p.add_option('-c', '--command', 142 p.add_option('-c', '--command',
143 help='Command (and arguments) to execute', 143 help='command (and arguments) to execute',
144 dest='command', 144 dest='command',
145 action='callback', 145 action='callback',
146 callback=self._cmd_option) 146 callback=self._cmd_option)
147 p.add_option('-e', '--abort-on-errors', 147 p.add_option('-e', '--abort-on-errors',
148 dest='abort_on_errors', action='store_true', 148 dest='abort_on_errors', action='store_true',
149 help='Abort if a command exits unsuccessfully') 149 help='abort if a command exits unsuccessfully')
150 p.add_option('--ignore-missing', action='store_true', 150 p.add_option('--ignore-missing', action='store_true',
151 help='Silently skip & do not exit non-zero due missing ' 151 help='silently skip & do not exit non-zero due missing '
152 'checkouts') 152 'checkouts')
153 153
154 g = p.get_option_group('--quiet') 154 g = p.get_option_group('--quiet')
155 g.add_option('-p', 155 g.add_option('-p',
156 dest='project_header', action='store_true', 156 dest='project_header', action='store_true',
157 help='Show project headers before output') 157 help='show project headers before output')
158 p.add_option('--interactive', 158 p.add_option('--interactive',
159 action='store_true', 159 action='store_true',
160 help='force interactive usage') 160 help='force interactive usage')
diff --git a/subcmds/gitc_delete.py b/subcmds/gitc_delete.py
index 56e0eaba..df749469 100644
--- a/subcmds/gitc_delete.py
+++ b/subcmds/gitc_delete.py
@@ -19,7 +19,7 @@ import platform_utils
19 19
20 20
21class GitcDelete(Command, GitcClientCommand): 21class GitcDelete(Command, GitcClientCommand):
22 common = True 22 COMMON = True
23 visible_everywhere = False 23 visible_everywhere = False
24 helpSummary = "Delete a GITC Client." 24 helpSummary = "Delete a GITC Client."
25 helpUsage = """ 25 helpUsage = """
@@ -33,7 +33,7 @@ and all locally downloaded sources.
33 def _Options(self, p): 33 def _Options(self, p):
34 p.add_option('-f', '--force', 34 p.add_option('-f', '--force',
35 dest='force', action='store_true', 35 dest='force', action='store_true',
36 help='Force the deletion (no prompt).') 36 help='force the deletion (no prompt)')
37 37
38 def Execute(self, opt, args): 38 def Execute(self, opt, args):
39 if not opt.force: 39 if not opt.force:
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py
index 23a4ebb6..e705b613 100644
--- a/subcmds/gitc_init.py
+++ b/subcmds/gitc_init.py
@@ -23,7 +23,7 @@ import wrapper
23 23
24 24
25class GitcInit(init.Init, GitcAvailableCommand): 25class GitcInit(init.Init, GitcAvailableCommand):
26 common = True 26 COMMON = True
27 helpSummary = "Initialize a GITC Client." 27 helpSummary = "Initialize a GITC Client."
28 helpUsage = """ 28 helpUsage = """
29%prog [options] [client name] 29%prog [options] [client name]
diff --git a/subcmds/grep.py b/subcmds/grep.py
index 6cb1445a..8ac4ba14 100644
--- a/subcmds/grep.py
+++ b/subcmds/grep.py
@@ -29,7 +29,7 @@ class GrepColoring(Coloring):
29 29
30 30
31class Grep(PagedCommand): 31class Grep(PagedCommand):
32 common = True 32 COMMON = True
33 helpSummary = "Print lines matching a pattern" 33 helpSummary = "Print lines matching a pattern"
34 helpUsage = """ 34 helpUsage = """
35%prog {pattern | -e pattern} [<project>...] 35%prog {pattern | -e pattern} [<project>...]
diff --git a/subcmds/help.py b/subcmds/help.py
index 6a767e6f..1a60ef45 100644
--- a/subcmds/help.py
+++ b/subcmds/help.py
@@ -20,10 +20,11 @@ from subcmds import all_commands
20from color import Coloring 20from color import Coloring
21from command import PagedCommand, MirrorSafeCommand, GitcAvailableCommand, GitcClientCommand 21from command import PagedCommand, MirrorSafeCommand, GitcAvailableCommand, GitcClientCommand
22import gitc_utils 22import gitc_utils
23from wrapper import Wrapper
23 24
24 25
25class Help(PagedCommand, MirrorSafeCommand): 26class Help(PagedCommand, MirrorSafeCommand):
26 common = False 27 COMMON = False
27 helpSummary = "Display detailed help on a command" 28 helpSummary = "Display detailed help on a command"
28 helpUsage = """ 29 helpUsage = """
29%prog [--all|command] 30%prog [--all|command]
@@ -49,14 +50,21 @@ Displays detailed usage information about a command.
49 50
50 def _PrintAllCommands(self): 51 def _PrintAllCommands(self):
51 print('usage: repo COMMAND [ARGS]') 52 print('usage: repo COMMAND [ARGS]')
53 self.PrintAllCommandsBody()
54
55 def PrintAllCommandsBody(self):
52 print('The complete list of recognized repo commands are:') 56 print('The complete list of recognized repo commands are:')
53 commandNames = list(sorted(all_commands)) 57 commandNames = list(sorted(all_commands))
54 self._PrintCommands(commandNames) 58 self._PrintCommands(commandNames)
55 print("See 'repo help <command>' for more information on a " 59 print("See 'repo help <command>' for more information on a "
56 'specific command.') 60 'specific command.')
61 print('Bug reports:', Wrapper().BUG_URL)
57 62
58 def _PrintCommonCommands(self): 63 def _PrintCommonCommands(self):
59 print('usage: repo COMMAND [ARGS]') 64 print('usage: repo COMMAND [ARGS]')
65 self.PrintCommonCommandsBody()
66
67 def PrintCommonCommandsBody(self):
60 print('The most commonly used repo commands are:') 68 print('The most commonly used repo commands are:')
61 69
62 def gitc_supported(cmd): 70 def gitc_supported(cmd):
@@ -72,12 +80,13 @@ Displays detailed usage information about a command.
72 80
73 commandNames = list(sorted([name 81 commandNames = list(sorted([name
74 for name, command in all_commands.items() 82 for name, command in all_commands.items()
75 if command.common and gitc_supported(command)])) 83 if command.COMMON and gitc_supported(command)]))
76 self._PrintCommands(commandNames) 84 self._PrintCommands(commandNames)
77 85
78 print( 86 print(
79 "See 'repo help <command>' for more information on a specific command.\n" 87 "See 'repo help <command>' for more information on a specific command.\n"
80 "See 'repo help --all' for a complete list of recognized commands.") 88 "See 'repo help --all' for a complete list of recognized commands.")
89 print('Bug reports:', Wrapper().BUG_URL)
81 90
82 def _PrintCommandHelp(self, cmd, header_prefix=''): 91 def _PrintCommandHelp(self, cmd, header_prefix=''):
83 class _Out(Coloring): 92 class _Out(Coloring):
@@ -136,8 +145,7 @@ Displays detailed usage information about a command.
136 145
137 def _PrintAllCommandHelp(self): 146 def _PrintAllCommandHelp(self):
138 for name in sorted(all_commands): 147 for name in sorted(all_commands):
139 cmd = all_commands[name]() 148 cmd = all_commands[name](manifest=self.manifest)
140 cmd.manifest = self.manifest
141 self._PrintCommandHelp(cmd, header_prefix='[%s] ' % (name,)) 149 self._PrintCommandHelp(cmd, header_prefix='[%s] ' % (name,))
142 150
143 def _Options(self, p): 151 def _Options(self, p):
@@ -161,12 +169,11 @@ Displays detailed usage information about a command.
161 name = args[0] 169 name = args[0]
162 170
163 try: 171 try:
164 cmd = all_commands[name]() 172 cmd = all_commands[name](manifest=self.manifest)
165 except KeyError: 173 except KeyError:
166 print("repo: '%s' is not a repo command." % name, file=sys.stderr) 174 print("repo: '%s' is not a repo command." % name, file=sys.stderr)
167 sys.exit(1) 175 sys.exit(1)
168 176
169 cmd.manifest = self.manifest
170 self._PrintCommandHelp(cmd) 177 self._PrintCommandHelp(cmd)
171 178
172 else: 179 else:
diff --git a/subcmds/info.py b/subcmds/info.py
index 6381fa8e..6c1246ef 100644
--- a/subcmds/info.py
+++ b/subcmds/info.py
@@ -12,6 +12,8 @@
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14 14
15import optparse
16
15from command import PagedCommand 17from command import PagedCommand
16from color import Coloring 18from color import Coloring
17from git_refs import R_M, R_HEADS 19from git_refs import R_M, R_HEADS
@@ -23,9 +25,9 @@ class _Coloring(Coloring):
23 25
24 26
25class Info(PagedCommand): 27class Info(PagedCommand):
26 common = True 28 COMMON = True
27 helpSummary = "Get info on the manifest branch, current branch or unmerged branches" 29 helpSummary = "Get info on the manifest branch, current branch or unmerged branches"
28 helpUsage = "%prog [-dl] [-o [-b]] [<project>...]" 30 helpUsage = "%prog [-dl] [-o [-c]] [<project>...]"
29 31
30 def _Options(self, p): 32 def _Options(self, p):
31 p.add_option('-d', '--diff', 33 p.add_option('-d', '--diff',
@@ -34,12 +36,19 @@ class Info(PagedCommand):
34 p.add_option('-o', '--overview', 36 p.add_option('-o', '--overview',
35 dest='overview', action='store_true', 37 dest='overview', action='store_true',
36 help='show overview of all local commits') 38 help='show overview of all local commits')
37 p.add_option('-b', '--current-branch', 39 p.add_option('-c', '--current-branch',
38 dest="current_branch", action="store_true", 40 dest="current_branch", action="store_true",
39 help="consider only checked out branches") 41 help="consider only checked out branches")
42 p.add_option('--no-current-branch',
43 dest='current_branch', action='store_false',
44 help='consider all local branches')
45 # Turn this into a warning & remove this someday.
46 p.add_option('-b',
47 dest='current_branch', action='store_true',
48 help=optparse.SUPPRESS_HELP)
40 p.add_option('-l', '--local-only', 49 p.add_option('-l', '--local-only',
41 dest="local", action="store_true", 50 dest="local", action="store_true",
42 help="Disable all remote operations") 51 help="disable all remote operations")
43 52
44 def Execute(self, opt, args): 53 def Execute(self, opt, args):
45 self.out = _Coloring(self.client.globalConfig) 54 self.out = _Coloring(self.client.globalConfig)
diff --git a/subcmds/init.py b/subcmds/init.py
index 4182262e..9c6b2ad9 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -12,10 +12,10 @@
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14 14
15import optparse
16import os 15import os
17import platform 16import platform
18import re 17import re
18import subprocess
19import sys 19import sys
20import urllib.parse 20import urllib.parse
21 21
@@ -25,13 +25,14 @@ from error import ManifestParseError
25from project import SyncBuffer 25from project import SyncBuffer
26from git_config import GitConfig 26from git_config import GitConfig
27from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD 27from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
28import fetch
28import git_superproject 29import git_superproject
29import platform_utils 30import platform_utils
30from wrapper import Wrapper 31from wrapper import Wrapper
31 32
32 33
33class Init(InteractiveCommand, MirrorSafeCommand): 34class Init(InteractiveCommand, MirrorSafeCommand):
34 common = True 35 COMMON = True
35 helpSummary = "Initialize a repo client checkout in the current directory" 36 helpSummary = "Initialize a repo client checkout in the current directory"
36 helpUsage = """ 37 helpUsage = """
37%prog [options] [manifest url] 38%prog [options] [manifest url]
@@ -54,6 +55,12 @@ The optional -m argument can be used to specify an alternate manifest
54to be used. If no manifest is specified, the manifest default.xml 55to be used. If no manifest is specified, the manifest default.xml
55will be used. 56will be used.
56 57
58If the --standalone-manifest argument is set, the manifest will be downloaded
59directly from the specified --manifest-url as a static file (rather than
60setting up a manifest git checkout). With --standalone-manifest, the manifest
61will be fully static and will not be re-downloaded during subsesquent
62`repo init` and `repo sync` calls.
63
57The --reference option can be used to point to a directory that 64The --reference option can be used to point to a directory that
58has the content of a --mirror sync. This will make the working 65has the content of a --mirror sync. This will make the working
59directory use as much data as possible from the local reference 66directory use as much data as possible from the local reference
@@ -97,15 +104,38 @@ to update the working directory files.
97 """ 104 """
98 superproject = git_superproject.Superproject(self.manifest, 105 superproject = git_superproject.Superproject(self.manifest,
99 self.repodir, 106 self.repodir,
107 self.git_event_log,
100 quiet=opt.quiet) 108 quiet=opt.quiet)
101 if not superproject.Sync(): 109 sync_result = superproject.Sync()
102 print('error: git update of superproject failed', file=sys.stderr) 110 if not sync_result.success:
103 sys.exit(1) 111 print('warning: git update of superproject failed, repo sync will not '
112 'use superproject to fetch source; while this error is not fatal, '
113 'and you can continue to run repo sync, please run repo init with '
114 'the --no-use-superproject option to stop seeing this warning',
115 file=sys.stderr)
116 if sync_result.fatal and opt.use_superproject is not None:
117 sys.exit(1)
104 118
105 def _SyncManifest(self, opt): 119 def _SyncManifest(self, opt):
106 m = self.manifest.manifestProject 120 m = self.manifest.manifestProject
107 is_new = not m.Exists 121 is_new = not m.Exists
108 122
123 # If repo has already been initialized, we take -u with the absence of
124 # --standalone-manifest to mean "transition to a standard repo set up",
125 # which necessitates starting fresh.
126 # If --standalone-manifest is set, we always tear everything down and start
127 # anew.
128 if not is_new:
129 was_standalone_manifest = m.config.GetString('manifest.standalone')
130 if opt.standalone_manifest or (
131 was_standalone_manifest and opt.manifest_url):
132 m.config.ClearCache()
133 if m.gitdir and os.path.exists(m.gitdir):
134 platform_utils.rmtree(m.gitdir)
135 if m.worktree and os.path.exists(m.worktree):
136 platform_utils.rmtree(m.worktree)
137
138 is_new = not m.Exists
109 if is_new: 139 if is_new:
110 if not opt.manifest_url: 140 if not opt.manifest_url:
111 print('fatal: manifest url is required.', file=sys.stderr) 141 print('fatal: manifest url is required.', file=sys.stderr)
@@ -130,6 +160,19 @@ to update the working directory files.
130 160
131 m._InitGitDir(mirror_git=mirrored_manifest_git) 161 m._InitGitDir(mirror_git=mirrored_manifest_git)
132 162
163 # If standalone_manifest is set, mark the project as "standalone" -- we'll
164 # still do much of the manifests.git set up, but will avoid actual syncs to
165 # a remote.
166 standalone_manifest = False
167 if opt.standalone_manifest:
168 standalone_manifest = True
169 elif not opt.manifest_url:
170 # If -u is set and --standalone-manifest is not, then we're not in
171 # standalone mode. Otherwise, use config to infer what we were in the last
172 # init.
173 standalone_manifest = bool(m.config.GetString('manifest.standalone'))
174 m.config.SetString('manifest.standalone', opt.manifest_url)
175
133 self._ConfigureDepth(opt) 176 self._ConfigureDepth(opt)
134 177
135 # Set the remote URL before the remote branch as we might need it below. 178 # Set the remote URL before the remote branch as we might need it below.
@@ -139,22 +182,23 @@ to update the working directory files.
139 r.ResetFetch() 182 r.ResetFetch()
140 r.Save() 183 r.Save()
141 184
142 if opt.manifest_branch: 185 if not standalone_manifest:
143 if opt.manifest_branch == 'HEAD': 186 if opt.manifest_branch:
144 opt.manifest_branch = m.ResolveRemoteHead() 187 if opt.manifest_branch == 'HEAD':
145 if opt.manifest_branch is None: 188 opt.manifest_branch = m.ResolveRemoteHead()
146 print('fatal: unable to resolve HEAD', file=sys.stderr) 189 if opt.manifest_branch is None:
147 sys.exit(1) 190 print('fatal: unable to resolve HEAD', file=sys.stderr)
148 m.revisionExpr = opt.manifest_branch 191 sys.exit(1)
149 else: 192 m.revisionExpr = opt.manifest_branch
150 if is_new:
151 default_branch = m.ResolveRemoteHead()
152 if default_branch is None:
153 # If the remote doesn't have HEAD configured, default to master.
154 default_branch = 'refs/heads/master'
155 m.revisionExpr = default_branch
156 else: 193 else:
157 m.PreSync() 194 if is_new:
195 default_branch = m.ResolveRemoteHead()
196 if default_branch is None:
197 # If the remote doesn't have HEAD configured, default to master.
198 default_branch = 'refs/heads/master'
199 m.revisionExpr = default_branch
200 else:
201 m.PreSync()
158 202
159 groups = re.split(r'[,\s]+', opt.groups) 203 groups = re.split(r'[,\s]+', opt.groups)
160 all_platforms = ['linux', 'darwin', 'windows'] 204 all_platforms = ['linux', 'darwin', 'windows']
@@ -244,6 +288,16 @@ to update the working directory files.
244 if opt.use_superproject is not None: 288 if opt.use_superproject is not None:
245 m.config.SetBoolean('repo.superproject', opt.use_superproject) 289 m.config.SetBoolean('repo.superproject', opt.use_superproject)
246 290
291 if standalone_manifest:
292 if is_new:
293 manifest_name = 'default.xml'
294 manifest_data = fetch.fetch_file(opt.manifest_url)
295 dest = os.path.join(m.worktree, manifest_name)
296 os.makedirs(os.path.dirname(dest), exist_ok=True)
297 with open(dest, 'wb') as f:
298 f.write(manifest_data)
299 return
300
247 if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose, 301 if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose,
248 clone_bundle=opt.clone_bundle, 302 clone_bundle=opt.clone_bundle,
249 current_branch_only=opt.current_branch_only, 303 current_branch_only=opt.current_branch_only,
@@ -420,6 +474,11 @@ to update the working directory files.
420 if opt.archive and opt.mirror: 474 if opt.archive and opt.mirror:
421 self.OptionParser.error('--mirror and --archive cannot be used together.') 475 self.OptionParser.error('--mirror and --archive cannot be used together.')
422 476
477 if opt.standalone_manifest and (
478 opt.manifest_branch or opt.manifest_name != 'default.xml'):
479 self.OptionParser.error('--manifest-branch and --manifest-name cannot'
480 ' be used with --standalone-manifest.')
481
423 if args: 482 if args:
424 if opt.manifest_url: 483 if opt.manifest_url:
425 self.OptionParser.error( 484 self.OptionParser.error(
diff --git a/subcmds/list.py b/subcmds/list.py
index 5cbc0c22..6adf85b7 100644
--- a/subcmds/list.py
+++ b/subcmds/list.py
@@ -12,11 +12,13 @@
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14 14
15import os
16
15from command import Command, MirrorSafeCommand 17from command import Command, MirrorSafeCommand
16 18
17 19
18class List(Command, MirrorSafeCommand): 20class List(Command, MirrorSafeCommand):
19 common = True 21 COMMON = True
20 helpSummary = "List projects and their associated directories" 22 helpSummary = "List projects and their associated directories"
21 helpUsage = """ 23 helpUsage = """
22%prog [-f] [<project>...] 24%prog [-f] [<project>...]
@@ -36,27 +38,33 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
36 def _Options(self, p): 38 def _Options(self, p):
37 p.add_option('-r', '--regex', 39 p.add_option('-r', '--regex',
38 dest='regex', action='store_true', 40 dest='regex', action='store_true',
39 help="Filter the project list based on regex or wildcard matching of strings") 41 help='filter the project list based on regex or wildcard matching of strings')
40 p.add_option('-g', '--groups', 42 p.add_option('-g', '--groups',
41 dest='groups', 43 dest='groups',
42 help="Filter the project list based on the groups the project is in") 44 help='filter the project list based on the groups the project is in')
43 p.add_option('-a', '--all', 45 p.add_option('-a', '--all',
44 action='store_true', 46 action='store_true',
45 help='Show projects regardless of checkout state') 47 help='show projects regardless of checkout state')
46 p.add_option('-f', '--fullpath',
47 dest='fullpath', action='store_true',
48 help="Display the full work tree path instead of the relative path")
49 p.add_option('-n', '--name-only', 48 p.add_option('-n', '--name-only',
50 dest='name_only', action='store_true', 49 dest='name_only', action='store_true',
51 help="Display only the name of the repository") 50 help='display only the name of the repository')
52 p.add_option('-p', '--path-only', 51 p.add_option('-p', '--path-only',
53 dest='path_only', action='store_true', 52 dest='path_only', action='store_true',
54 help="Display only the path of the repository") 53 help='display only the path of the repository')
54 p.add_option('-f', '--fullpath',
55 dest='fullpath', action='store_true',
56 help='display the full work tree path instead of the relative path')
57 p.add_option('--relative-to', metavar='PATH',
58 help='display paths relative to this one (default: top of repo client checkout)')
55 59
56 def ValidateOptions(self, opt, args): 60 def ValidateOptions(self, opt, args):
57 if opt.fullpath and opt.name_only: 61 if opt.fullpath and opt.name_only:
58 self.OptionParser.error('cannot combine -f and -n') 62 self.OptionParser.error('cannot combine -f and -n')
59 63
64 # Resolve any symlinks so the output is stable.
65 if opt.relative_to:
66 opt.relative_to = os.path.realpath(opt.relative_to)
67
60 def Execute(self, opt, args): 68 def Execute(self, opt, args):
61 """List all projects and the associated directories. 69 """List all projects and the associated directories.
62 70
@@ -76,6 +84,8 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
76 def _getpath(x): 84 def _getpath(x):
77 if opt.fullpath: 85 if opt.fullpath:
78 return x.worktree 86 return x.worktree
87 if opt.relative_to:
88 return os.path.relpath(x.worktree, opt.relative_to)
79 return x.relpath 89 return x.relpath
80 90
81 lines = [] 91 lines = []
diff --git a/subcmds/manifest.py b/subcmds/manifest.py
index e33e683c..0fbdeac0 100644
--- a/subcmds/manifest.py
+++ b/subcmds/manifest.py
@@ -20,7 +20,7 @@ from command import PagedCommand
20 20
21 21
22class Manifest(PagedCommand): 22class Manifest(PagedCommand):
23 common = False 23 COMMON = False
24 helpSummary = "Manifest inspection utility" 24 helpSummary = "Manifest inspection utility"
25 helpUsage = """ 25 helpUsage = """
26%prog [-o {-|NAME.xml}] [-m MANIFEST.xml] [-r] 26%prog [-o {-|NAME.xml}] [-m MANIFEST.xml] [-r]
@@ -53,27 +53,29 @@ to indicate the remote ref to push changes to via 'repo upload'.
53 def _Options(self, p): 53 def _Options(self, p):
54 p.add_option('-r', '--revision-as-HEAD', 54 p.add_option('-r', '--revision-as-HEAD',
55 dest='peg_rev', action='store_true', 55 dest='peg_rev', action='store_true',
56 help='Save revisions as current HEAD') 56 help='save revisions as current HEAD')
57 p.add_option('-m', '--manifest-name', 57 p.add_option('-m', '--manifest-name',
58 help='temporary manifest to use for this sync', metavar='NAME.xml') 58 help='temporary manifest to use for this sync', metavar='NAME.xml')
59 p.add_option('--suppress-upstream-revision', dest='peg_rev_upstream', 59 p.add_option('--suppress-upstream-revision', dest='peg_rev_upstream',
60 default=True, action='store_false', 60 default=True, action='store_false',
61 help='If in -r mode, do not write the upstream field. ' 61 help='if in -r mode, do not write the upstream field '
62 'Only of use if the branch names for a sha1 manifest are ' 62 '(only of use if the branch names for a sha1 manifest are '
63 'sensitive.') 63 'sensitive)')
64 p.add_option('--suppress-dest-branch', dest='peg_rev_dest_branch', 64 p.add_option('--suppress-dest-branch', dest='peg_rev_dest_branch',
65 default=True, action='store_false', 65 default=True, action='store_false',
66 help='If in -r mode, do not write the dest-branch field. ' 66 help='if in -r mode, do not write the dest-branch field '
67 'Only of use if the branch names for a sha1 manifest are ' 67 '(only of use if the branch names for a sha1 manifest are '
68 'sensitive.') 68 'sensitive)')
69 p.add_option('--json', default=False, action='store_true', 69 p.add_option('--json', default=False, action='store_true',
70 help='Output manifest in JSON format (experimental).') 70 help='output manifest in JSON format (experimental)')
71 p.add_option('--pretty', default=False, action='store_true', 71 p.add_option('--pretty', default=False, action='store_true',
72 help='Format output for humans to read.') 72 help='format output for humans to read')
73 p.add_option('--no-local-manifests', default=False, action='store_true',
74 dest='ignore_local_manifests', help='ignore local manifests')
73 p.add_option('-o', '--output-file', 75 p.add_option('-o', '--output-file',
74 dest='output_file', 76 dest='output_file',
75 default='-', 77 default='-',
76 help='File to save the manifest to', 78 help='file to save the manifest to',
77 metavar='-|NAME.xml') 79 metavar='-|NAME.xml')
78 80
79 def _Output(self, opt): 81 def _Output(self, opt):
@@ -85,6 +87,9 @@ to indicate the remote ref to push changes to via 'repo upload'.
85 fd = sys.stdout 87 fd = sys.stdout
86 else: 88 else:
87 fd = open(opt.output_file, 'w') 89 fd = open(opt.output_file, 'w')
90
91 self.manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
92
88 if opt.json: 93 if opt.json:
89 print('warning: --json is experimental!', file=sys.stderr) 94 print('warning: --json is experimental!', file=sys.stderr)
90 doc = self.manifest.ToDict(peg_rev=opt.peg_rev, 95 doc = self.manifest.ToDict(peg_rev=opt.peg_rev,
diff --git a/subcmds/overview.py b/subcmds/overview.py
index 004a847c..63f5a79e 100644
--- a/subcmds/overview.py
+++ b/subcmds/overview.py
@@ -12,12 +12,14 @@
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14 14
15import optparse
16
15from color import Coloring 17from color import Coloring
16from command import PagedCommand 18from command import PagedCommand
17 19
18 20
19class Overview(PagedCommand): 21class Overview(PagedCommand):
20 common = True 22 COMMON = True
21 helpSummary = "Display overview of unmerged project branches" 23 helpSummary = "Display overview of unmerged project branches"
22 helpUsage = """ 24 helpUsage = """
23%prog [--current-branch] [<project>...] 25%prog [--current-branch] [<project>...]
@@ -26,15 +28,22 @@ class Overview(PagedCommand):
26The '%prog' command is used to display an overview of the projects branches, 28The '%prog' command is used to display an overview of the projects branches,
27and list any local commits that have not yet been merged into the project. 29and list any local commits that have not yet been merged into the project.
28 30
29The -b/--current-branch option can be used to restrict the output to only 31The -c/--current-branch option can be used to restrict the output to only
30branches currently checked out in each project. By default, all branches 32branches currently checked out in each project. By default, all branches
31are displayed. 33are displayed.
32""" 34"""
33 35
34 def _Options(self, p): 36 def _Options(self, p):
35 p.add_option('-b', '--current-branch', 37 p.add_option('-c', '--current-branch',
36 dest="current_branch", action="store_true", 38 dest="current_branch", action="store_true",
37 help="Consider only checked out branches") 39 help="consider only checked out branches")
40 p.add_option('--no-current-branch',
41 dest='current_branch', action='store_false',
42 help='consider all local branches')
43 # Turn this into a warning & remove this someday.
44 p.add_option('-b',
45 dest='current_branch', action='store_true',
46 help=optparse.SUPPRESS_HELP)
38 47
39 def Execute(self, opt, args): 48 def Execute(self, opt, args):
40 all_branches = [] 49 all_branches = []
diff --git a/subcmds/prune.py b/subcmds/prune.py
index 236b647f..584ee7ed 100644
--- a/subcmds/prune.py
+++ b/subcmds/prune.py
@@ -19,7 +19,7 @@ from command import DEFAULT_LOCAL_JOBS, PagedCommand
19 19
20 20
21class Prune(PagedCommand): 21class Prune(PagedCommand):
22 common = True 22 COMMON = True
23 helpSummary = "Prune (delete) already merged topics" 23 helpSummary = "Prune (delete) already merged topics"
24 helpUsage = """ 24 helpUsage = """
25%prog [<project>...] 25%prog [<project>...]
diff --git a/subcmds/rebase.py b/subcmds/rebase.py
index e0186d4d..7c53eb7a 100644
--- a/subcmds/rebase.py
+++ b/subcmds/rebase.py
@@ -27,7 +27,7 @@ class RebaseColoring(Coloring):
27 27
28 28
29class Rebase(Command): 29class Rebase(Command):
30 common = True 30 COMMON = True
31 helpSummary = "Rebase local branches on upstream branch" 31 helpSummary = "Rebase local branches on upstream branch"
32 helpUsage = """ 32 helpUsage = """
33%prog {[<project>...] | -i <project>...} 33%prog {[<project>...] | -i <project>...}
@@ -46,27 +46,27 @@ branch but need to incorporate new upstream changes "underneath" them.
46 46
47 p.add_option('--fail-fast', 47 p.add_option('--fail-fast',
48 dest='fail_fast', action='store_true', 48 dest='fail_fast', action='store_true',
49 help='Stop rebasing after first error is hit') 49 help='stop rebasing after first error is hit')
50 p.add_option('-f', '--force-rebase', 50 p.add_option('-f', '--force-rebase',
51 dest='force_rebase', action='store_true', 51 dest='force_rebase', action='store_true',
52 help='Pass --force-rebase to git rebase') 52 help='pass --force-rebase to git rebase')
53 p.add_option('--no-ff', 53 p.add_option('--no-ff',
54 dest='ff', default=True, action='store_false', 54 dest='ff', default=True, action='store_false',
55 help='Pass --no-ff to git rebase') 55 help='pass --no-ff to git rebase')
56 p.add_option('--autosquash', 56 p.add_option('--autosquash',
57 dest='autosquash', action='store_true', 57 dest='autosquash', action='store_true',
58 help='Pass --autosquash to git rebase') 58 help='pass --autosquash to git rebase')
59 p.add_option('--whitespace', 59 p.add_option('--whitespace',
60 dest='whitespace', action='store', metavar='WS', 60 dest='whitespace', action='store', metavar='WS',
61 help='Pass --whitespace to git rebase') 61 help='pass --whitespace to git rebase')
62 p.add_option('--auto-stash', 62 p.add_option('--auto-stash',
63 dest='auto_stash', action='store_true', 63 dest='auto_stash', action='store_true',
64 help='Stash local modifications before starting') 64 help='stash local modifications before starting')
65 p.add_option('-m', '--onto-manifest', 65 p.add_option('-m', '--onto-manifest',
66 dest='onto_manifest', action='store_true', 66 dest='onto_manifest', action='store_true',
67 help='Rebase onto the manifest version instead of upstream ' 67 help='rebase onto the manifest version instead of upstream '
68 'HEAD. This helps to make sure the local tree stays ' 68 'HEAD (this helps to make sure the local tree stays '
69 'consistent if you previously synced to a manifest.') 69 'consistent if you previously synced to a manifest)')
70 70
71 def Execute(self, opt, args): 71 def Execute(self, opt, args):
72 all_projects = self.GetProjects(args) 72 all_projects = self.GetProjects(args)
diff --git a/subcmds/selfupdate.py b/subcmds/selfupdate.py
index 388881d9..282f518e 100644
--- a/subcmds/selfupdate.py
+++ b/subcmds/selfupdate.py
@@ -21,7 +21,7 @@ from subcmds.sync import _PostRepoFetch
21 21
22 22
23class Selfupdate(Command, MirrorSafeCommand): 23class Selfupdate(Command, MirrorSafeCommand):
24 common = False 24 COMMON = False
25 helpSummary = "Update repo to the latest version" 25 helpSummary = "Update repo to the latest version"
26 helpUsage = """ 26 helpUsage = """
27%prog 27%prog
diff --git a/subcmds/smartsync.py b/subcmds/smartsync.py
index c7d1d4d4..d91d59c6 100644
--- a/subcmds/smartsync.py
+++ b/subcmds/smartsync.py
@@ -16,7 +16,7 @@ from subcmds.sync import Sync
16 16
17 17
18class Smartsync(Sync): 18class Smartsync(Sync):
19 common = True 19 COMMON = True
20 helpSummary = "Update working tree to the latest known good revision" 20 helpSummary = "Update working tree to the latest known good revision"
21 helpUsage = """ 21 helpUsage = """
22%prog [<project>...] 22%prog [<project>...]
diff --git a/subcmds/stage.py b/subcmds/stage.py
index ff0f1738..0389a4ff 100644
--- a/subcmds/stage.py
+++ b/subcmds/stage.py
@@ -28,7 +28,7 @@ class _ProjectList(Coloring):
28 28
29 29
30class Stage(InteractiveCommand): 30class Stage(InteractiveCommand):
31 common = True 31 COMMON = True
32 helpSummary = "Stage file(s) for commit" 32 helpSummary = "Stage file(s) for commit"
33 helpUsage = """ 33 helpUsage = """
34%prog -i [<project>...] 34%prog -i [<project>...]
diff --git a/subcmds/start.py b/subcmds/start.py
index ff2bae56..2addaf2e 100644
--- a/subcmds/start.py
+++ b/subcmds/start.py
@@ -25,7 +25,7 @@ from project import SyncBuffer
25 25
26 26
27class Start(Command): 27class Start(Command):
28 common = True 28 COMMON = True
29 helpSummary = "Start a new branch for development" 29 helpSummary = "Start a new branch for development"
30 helpUsage = """ 30 helpUsage = """
31%prog <newbranchname> [--all | <project>...] 31%prog <newbranchname> [--all | <project>...]
diff --git a/subcmds/status.py b/subcmds/status.py
index 1b48dcea..5b669547 100644
--- a/subcmds/status.py
+++ b/subcmds/status.py
@@ -24,7 +24,7 @@ import platform_utils
24 24
25 25
26class Status(PagedCommand): 26class Status(PagedCommand):
27 common = True 27 COMMON = True
28 helpSummary = "Show the working tree status" 28 helpSummary = "Show the working tree status"
29 helpUsage = """ 29 helpUsage = """
30%prog [<project>...] 30%prog [<project>...]
diff --git a/subcmds/sync.py b/subcmds/sync.py
index d41052d7..3211cbb1 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -12,6 +12,7 @@
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14 14
15import errno
15import functools 16import functools
16import http.cookiejar as cookielib 17import http.cookiejar as cookielib
17import io 18import io
@@ -56,6 +57,7 @@ from error import RepoChangedException, GitError, ManifestParseError
56import platform_utils 57import platform_utils
57from project import SyncBuffer 58from project import SyncBuffer
58from progress import Progress 59from progress import Progress
60import ssh
59from wrapper import Wrapper 61from wrapper import Wrapper
60from manifest_xml import GitcManifest 62from manifest_xml import GitcManifest
61 63
@@ -64,7 +66,7 @@ _ONE_DAY_S = 24 * 60 * 60
64 66
65class Sync(Command, MirrorSafeCommand): 67class Sync(Command, MirrorSafeCommand):
66 jobs = 1 68 jobs = 1
67 common = True 69 COMMON = True
68 helpSummary = "Update working tree to the latest revision" 70 helpSummary = "Update working tree to the latest revision"
69 helpUsage = """ 71 helpUsage = """
70%prog [<project>...] 72%prog [<project>...]
@@ -168,10 +170,11 @@ later is required to fix a server side protocol bug.
168 PARALLEL_JOBS = 1 170 PARALLEL_JOBS = 1
169 171
170 def _CommonOptions(self, p): 172 def _CommonOptions(self, p):
171 try: 173 if self.manifest:
172 self.PARALLEL_JOBS = self.manifest.default.sync_j 174 try:
173 except ManifestParseError: 175 self.PARALLEL_JOBS = self.manifest.default.sync_j
174 pass 176 except ManifestParseError:
177 pass
175 super()._CommonOptions(p) 178 super()._CommonOptions(p)
176 179
177 def _Options(self, p, show_smart=True): 180 def _Options(self, p, show_smart=True):
@@ -212,6 +215,9 @@ later is required to fix a server side protocol bug.
212 p.add_option('-c', '--current-branch', 215 p.add_option('-c', '--current-branch',
213 dest='current_branch_only', action='store_true', 216 dest='current_branch_only', action='store_true',
214 help='fetch only current branch from server') 217 help='fetch only current branch from server')
218 p.add_option('--no-current-branch',
219 dest='current_branch_only', action='store_false',
220 help='fetch all branches from server')
215 p.add_option('-m', '--manifest-name', 221 p.add_option('-m', '--manifest-name',
216 dest='manifest_name', 222 dest='manifest_name',
217 help='temporary manifest to use for this sync', metavar='NAME.xml') 223 help='temporary manifest to use for this sync', metavar='NAME.xml')
@@ -230,8 +236,14 @@ later is required to fix a server side protocol bug.
230 help='fetch submodules from server') 236 help='fetch submodules from server')
231 p.add_option('--use-superproject', action='store_true', 237 p.add_option('--use-superproject', action='store_true',
232 help='use the manifest superproject to sync projects') 238 help='use the manifest superproject to sync projects')
239 p.add_option('--no-use-superproject', action='store_false',
240 dest='use_superproject',
241 help='disable use of manifest superprojects')
242 p.add_option('--tags',
243 action='store_false',
244 help='fetch tags')
233 p.add_option('--no-tags', 245 p.add_option('--no-tags',
234 dest='tags', default=True, action='store_false', 246 dest='tags', action='store_false',
235 help="don't fetch tags") 247 help="don't fetch tags")
236 p.add_option('--optimized-fetch', 248 p.add_option('--optimized-fetch',
237 dest='optimized_fetch', action='store_true', 249 dest='optimized_fetch', action='store_true',
@@ -266,17 +278,11 @@ later is required to fix a server side protocol bug.
266 branch = branch[len(R_HEADS):] 278 branch = branch[len(R_HEADS):]
267 return branch 279 return branch
268 280
269 def _UseSuperproject(self, opt):
270 """Returns True if use-superproject option is enabled"""
271 return (opt.use_superproject or
272 self.manifest.manifestProject.config.GetBoolean(
273 'repo.superproject'))
274
275 def _GetCurrentBranchOnly(self, opt): 281 def _GetCurrentBranchOnly(self, opt):
276 """Returns True if current-branch or use-superproject options are enabled.""" 282 """Returns True if current-branch or use-superproject options are enabled."""
277 return opt.current_branch_only or self._UseSuperproject(opt) 283 return opt.current_branch_only or git_superproject.UseSuperproject(opt, self.manifest)
278 284
279 def _UpdateProjectsRevisionId(self, opt, args): 285 def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data):
280 """Update revisionId of every project with the SHA from superproject. 286 """Update revisionId of every project with the SHA from superproject.
281 287
282 This function updates each project's revisionId with SHA from superproject. 288 This function updates each project's revisionId with SHA from superproject.
@@ -286,22 +292,40 @@ later is required to fix a server side protocol bug.
286 opt: Program options returned from optparse. See _Options(). 292 opt: Program options returned from optparse. See _Options().
287 args: Arguments to pass to GetProjects. See the GetProjects 293 args: Arguments to pass to GetProjects. See the GetProjects
288 docstring for details. 294 docstring for details.
295 load_local_manifests: Whether to load local manifests.
296 superproject_logging_data: A dictionary of superproject data that is to be logged.
289 297
290 Returns: 298 Returns:
291 Returns path to the overriding manifest file. 299 Returns path to the overriding manifest file instead of None.
292 """ 300 """
301 print_messages = git_superproject.PrintMessages(opt, self.manifest)
293 superproject = git_superproject.Superproject(self.manifest, 302 superproject = git_superproject.Superproject(self.manifest,
294 self.repodir, 303 self.repodir,
295 quiet=opt.quiet) 304 self.git_event_log,
305 quiet=opt.quiet,
306 print_messages=print_messages)
307 if opt.local_only:
308 manifest_path = superproject.manifest_path
309 if manifest_path:
310 self._ReloadManifest(manifest_path, load_local_manifests)
311 return manifest_path
312
296 all_projects = self.GetProjects(args, 313 all_projects = self.GetProjects(args,
297 missing_ok=True, 314 missing_ok=True,
298 submodules_ok=opt.fetch_submodules) 315 submodules_ok=opt.fetch_submodules)
299 manifest_path = superproject.UpdateProjectsRevisionId(all_projects) 316 update_result = superproject.UpdateProjectsRevisionId(all_projects)
300 if not manifest_path: 317 manifest_path = update_result.manifest_path
301 print('error: Update of revsionId from superproject has failed', 318 superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
302 file=sys.stderr) 319 if manifest_path:
303 sys.exit(1) 320 self._ReloadManifest(manifest_path, load_local_manifests)
304 self._ReloadManifest(manifest_path) 321 else:
322 if print_messages:
323 print('warning: Update of revisionId from superproject has failed, '
324 'repo sync will not use superproject to fetch the source. ',
325 'Please resync with the --no-use-superproject option to avoid this repo warning.',
326 file=sys.stderr)
327 if update_result.fatal and opt.use_superproject is not None:
328 sys.exit(1)
305 return manifest_path 329 return manifest_path
306 330
307 def _FetchProjectList(self, opt, projects): 331 def _FetchProjectList(self, opt, projects):
@@ -343,11 +367,12 @@ later is required to fix a server side protocol bug.
343 optimized_fetch=opt.optimized_fetch, 367 optimized_fetch=opt.optimized_fetch,
344 retry_fetches=opt.retry_fetches, 368 retry_fetches=opt.retry_fetches,
345 prune=opt.prune, 369 prune=opt.prune,
370 ssh_proxy=self.ssh_proxy,
346 clone_filter=self.manifest.CloneFilter, 371 clone_filter=self.manifest.CloneFilter,
347 partial_clone_exclude=self.manifest.PartialCloneExclude) 372 partial_clone_exclude=self.manifest.PartialCloneExclude)
348 373
349 output = buf.getvalue() 374 output = buf.getvalue()
350 if opt.verbose and output: 375 if (opt.verbose or not success) and output:
351 print('\n' + output.rstrip()) 376 print('\n' + output.rstrip())
352 377
353 if not success: 378 if not success:
@@ -364,7 +389,11 @@ later is required to fix a server side protocol bug.
364 finish = time.time() 389 finish = time.time()
365 return (success, project, start, finish) 390 return (success, project, start, finish)
366 391
367 def _Fetch(self, projects, opt, err_event): 392 @classmethod
393 def _FetchInitChild(cls, ssh_proxy):
394 cls.ssh_proxy = ssh_proxy
395
396 def _Fetch(self, projects, opt, err_event, ssh_proxy):
368 ret = True 397 ret = True
369 398
370 jobs = opt.jobs_network if opt.jobs_network else self.jobs 399 jobs = opt.jobs_network if opt.jobs_network else self.jobs
@@ -394,8 +423,14 @@ later is required to fix a server side protocol bug.
394 break 423 break
395 return ret 424 return ret
396 425
426 # We pass the ssh proxy settings via the class. This allows multiprocessing
427 # to pickle it up when spawning children. We can't pass it as an argument
428 # to _FetchProjectList below as multiprocessing is unable to pickle those.
429 Sync.ssh_proxy = None
430
397 # NB: Multiprocessing is heavy, so don't spin it up for one job. 431 # NB: Multiprocessing is heavy, so don't spin it up for one job.
398 if len(projects_list) == 1 or jobs == 1: 432 if len(projects_list) == 1 or jobs == 1:
433 self._FetchInitChild(ssh_proxy)
399 if not _ProcessResults(self._FetchProjectList(opt, x) for x in projects_list): 434 if not _ProcessResults(self._FetchProjectList(opt, x) for x in projects_list):
400 ret = False 435 ret = False
401 else: 436 else:
@@ -413,7 +448,8 @@ later is required to fix a server side protocol bug.
413 else: 448 else:
414 pm.update(inc=0, msg='warming up') 449 pm.update(inc=0, msg='warming up')
415 chunksize = 4 450 chunksize = 4
416 with multiprocessing.Pool(jobs) as pool: 451 with multiprocessing.Pool(
452 jobs, initializer=self._FetchInitChild, initargs=(ssh_proxy,)) as pool:
417 results = pool.imap_unordered( 453 results = pool.imap_unordered(
418 functools.partial(self._FetchProjectList, opt), 454 functools.partial(self._FetchProjectList, opt),
419 projects_list, 455 projects_list,
@@ -422,6 +458,11 @@ later is required to fix a server side protocol bug.
422 ret = False 458 ret = False
423 pool.close() 459 pool.close()
424 460
461 # Cleanup the reference now that we're done with it, and we're going to
462 # release any resources it points to. If we don't, later multiprocessing
463 # usage (e.g. checkouts) will try to pickle and then crash.
464 del Sync.ssh_proxy
465
425 pm.end() 466 pm.end()
426 self._fetch_times.Save() 467 self._fetch_times.Save()
427 468
@@ -430,6 +471,69 @@ later is required to fix a server side protocol bug.
430 471
431 return (ret, fetched) 472 return (ret, fetched)
432 473
474 def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
475 load_local_manifests, ssh_proxy):
476 """The main network fetch loop.
477
478 Args:
479 opt: Program options returned from optparse. See _Options().
480 args: Command line args used to filter out projects.
481 all_projects: List of all projects that should be fetched.
482 err_event: Whether an error was hit while processing.
483 manifest_name: Manifest file to be reloaded.
484 load_local_manifests: Whether to load local manifests.
485 ssh_proxy: SSH manager for clients & masters.
486
487 Returns:
488 List of all projects that should be checked out.
489 """
490 rp = self.manifest.repoProject
491
492 to_fetch = []
493 now = time.time()
494 if _ONE_DAY_S <= (now - rp.LastFetch):
495 to_fetch.append(rp)
496 to_fetch.extend(all_projects)
497 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
498
499 success, fetched = self._Fetch(to_fetch, opt, err_event, ssh_proxy)
500 if not success:
501 err_event.set()
502
503 _PostRepoFetch(rp, opt.repo_verify)
504 if opt.network_only:
505 # bail out now; the rest touches the working tree
506 if err_event.is_set():
507 print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
508 sys.exit(1)
509 return
510
511 # Iteratively fetch missing and/or nested unregistered submodules
512 previously_missing_set = set()
513 while True:
514 self._ReloadManifest(manifest_name, load_local_manifests)
515 all_projects = self.GetProjects(args,
516 missing_ok=True,
517 submodules_ok=opt.fetch_submodules)
518 missing = []
519 for project in all_projects:
520 if project.gitdir not in fetched:
521 missing.append(project)
522 if not missing:
523 break
524 # Stop us from non-stopped fetching actually-missing repos: If set of
525 # missing repos has not been changed from last fetch, we break.
526 missing_set = set(p.name for p in missing)
527 if previously_missing_set == missing_set:
528 break
529 previously_missing_set = missing_set
530 success, new_fetched = self._Fetch(missing, opt, err_event, ssh_proxy)
531 if not success:
532 err_event.set()
533 fetched.update(new_fetched)
534
535 return all_projects
536
433 def _CheckoutOne(self, detach_head, force_sync, project): 537 def _CheckoutOne(self, detach_head, force_sync, project):
434 """Checkout work tree for one project 538 """Checkout work tree for one project
435 539
@@ -564,10 +668,18 @@ later is required to fix a server side protocol bug.
564 t.join() 668 t.join()
565 pm.end() 669 pm.end()
566 670
567 def _ReloadManifest(self, manifest_name=None): 671 def _ReloadManifest(self, manifest_name=None, load_local_manifests=True):
672 """Reload the manfiest from the file specified by the |manifest_name|.
673
674 It unloads the manifest if |manifest_name| is None.
675
676 Args:
677 manifest_name: Manifest file to be reloaded.
678 load_local_manifests: Whether to load local manifests.
679 """
568 if manifest_name: 680 if manifest_name:
569 # Override calls _Unload already 681 # Override calls _Unload already
570 self.manifest.Override(manifest_name) 682 self.manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
571 else: 683 else:
572 self.manifest._Unload() 684 self.manifest._Unload()
573 685
@@ -614,6 +726,56 @@ later is required to fix a server side protocol bug.
614 fd.write('\n') 726 fd.write('\n')
615 return 0 727 return 0
616 728
729 def UpdateCopyLinkfileList(self):
730 """Save all dests of copyfile and linkfile, and update them if needed.
731
732 Returns:
733 Whether update was successful.
734 """
735 new_paths = {}
736 new_linkfile_paths = []
737 new_copyfile_paths = []
738 for project in self.GetProjects(None, missing_ok=True):
739 new_linkfile_paths.extend(x.dest for x in project.linkfiles)
740 new_copyfile_paths.extend(x.dest for x in project.copyfiles)
741
742 new_paths = {
743 'linkfile': new_linkfile_paths,
744 'copyfile': new_copyfile_paths,
745 }
746
747 copylinkfile_name = 'copy-link-files.json'
748 copylinkfile_path = os.path.join(self.manifest.repodir, copylinkfile_name)
749 old_copylinkfile_paths = {}
750
751 if os.path.exists(copylinkfile_path):
752 with open(copylinkfile_path, 'rb') as fp:
753 try:
754 old_copylinkfile_paths = json.load(fp)
755 except:
756 print('error: %s is not a json formatted file.' %
757 copylinkfile_path, file=sys.stderr)
758 platform_utils.remove(copylinkfile_path)
759 return False
760
761 need_remove_files = []
762 need_remove_files.extend(
763 set(old_copylinkfile_paths.get('linkfile', [])) -
764 set(new_linkfile_paths))
765 need_remove_files.extend(
766 set(old_copylinkfile_paths.get('copyfile', [])) -
767 set(new_copyfile_paths))
768
769 for need_remove_file in need_remove_files:
770 # Try to remove the updated copyfile or linkfile.
771 # So, if the file is not exist, nothing need to do.
772 platform_utils.remove(need_remove_file, missing_ok=True)
773
774 # Create copy-link-files.json, save dest path of "copyfile" and "linkfile".
775 with open(copylinkfile_path, 'w', encoding='utf-8') as fp:
776 json.dump(new_paths, fp)
777 return True
778
617 def _SmartSyncSetup(self, opt, smart_sync_manifest_path): 779 def _SmartSyncSetup(self, opt, smart_sync_manifest_path):
618 if not self.manifest.manifest_server: 780 if not self.manifest.manifest_server:
619 print('error: cannot smart sync: no manifest server defined in ' 781 print('error: cannot smart sync: no manifest server defined in '
@@ -730,7 +892,7 @@ later is required to fix a server side protocol bug.
730 start, time.time(), clean) 892 start, time.time(), clean)
731 if not clean: 893 if not clean:
732 sys.exit(1) 894 sys.exit(1)
733 self._ReloadManifest(opt.manifest_name) 895 self._ReloadManifest(manifest_name)
734 if opt.jobs is None: 896 if opt.jobs is None:
735 self.jobs = self.manifest.default.sync_j 897 self.jobs = self.manifest.default.sync_j
736 898
@@ -779,7 +941,7 @@ later is required to fix a server side protocol bug.
779 print('error: failed to remove existing smart sync override manifest: %s' % 941 print('error: failed to remove existing smart sync override manifest: %s' %
780 e, file=sys.stderr) 942 e, file=sys.stderr)
781 943
782 err_event = _threading.Event() 944 err_event = multiprocessing.Event()
783 945
784 rp = self.manifest.repoProject 946 rp = self.manifest.repoProject
785 rp.PreSync() 947 rp.PreSync()
@@ -802,8 +964,16 @@ later is required to fix a server side protocol bug.
802 else: 964 else:
803 self._UpdateManifestProject(opt, mp, manifest_name) 965 self._UpdateManifestProject(opt, mp, manifest_name)
804 966
805 if self._UseSuperproject(opt): 967 load_local_manifests = not self.manifest.HasLocalManifests
806 manifest_name = self._UpdateProjectsRevisionId(opt, args) 968 use_superproject = git_superproject.UseSuperproject(opt, self.manifest)
969 superproject_logging_data = {
970 'superproject': use_superproject,
971 'haslocalmanifests': bool(self.manifest.HasLocalManifests),
972 'hassuperprojecttag': bool(self.manifest.superproject),
973 }
974 if use_superproject:
975 manifest_name = self._UpdateProjectsRevisionId(
976 opt, args, load_local_manifests, superproject_logging_data) or opt.manifest_name
807 977
808 if self.gitc_manifest: 978 if self.gitc_manifest:
809 gitc_manifest_projects = self.GetProjects(args, 979 gitc_manifest_projects = self.GetProjects(args,
@@ -849,49 +1019,17 @@ later is required to fix a server side protocol bug.
849 1019
850 self._fetch_times = _FetchTimes(self.manifest) 1020 self._fetch_times = _FetchTimes(self.manifest)
851 if not opt.local_only: 1021 if not opt.local_only:
852 to_fetch = [] 1022 with multiprocessing.Manager() as manager:
853 now = time.time() 1023 with ssh.ProxyManager(manager) as ssh_proxy:
854 if _ONE_DAY_S <= (now - rp.LastFetch): 1024 # Initialize the socket dir once in the parent.
855 to_fetch.append(rp) 1025 ssh_proxy.sock()
856 to_fetch.extend(all_projects) 1026 all_projects = self._FetchMain(opt, args, all_projects, err_event,
857 to_fetch.sort(key=self._fetch_times.Get, reverse=True) 1027 manifest_name, load_local_manifests,
858 1028 ssh_proxy)
859 success, fetched = self._Fetch(to_fetch, opt, err_event)
860 if not success:
861 err_event.set()
862 1029
863 _PostRepoFetch(rp, opt.repo_verify)
864 if opt.network_only: 1030 if opt.network_only:
865 # bail out now; the rest touches the working tree
866 if err_event.is_set():
867 print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
868 sys.exit(1)
869 return 1031 return
870 1032
871 # Iteratively fetch missing and/or nested unregistered submodules
872 previously_missing_set = set()
873 while True:
874 self._ReloadManifest(manifest_name)
875 all_projects = self.GetProjects(args,
876 missing_ok=True,
877 submodules_ok=opt.fetch_submodules)
878 missing = []
879 for project in all_projects:
880 if project.gitdir not in fetched:
881 missing.append(project)
882 if not missing:
883 break
884 # Stop us from non-stopped fetching actually-missing repos: If set of
885 # missing repos has not been changed from last fetch, we break.
886 missing_set = set(p.name for p in missing)
887 if previously_missing_set == missing_set:
888 break
889 previously_missing_set = missing_set
890 success, new_fetched = self._Fetch(missing, opt, err_event)
891 if not success:
892 err_event.set()
893 fetched.update(new_fetched)
894
895 # If we saw an error, exit with code 1 so that other scripts can check. 1033 # If we saw an error, exit with code 1 so that other scripts can check.
896 if err_event.is_set(): 1034 if err_event.is_set():
897 err_network_sync = True 1035 err_network_sync = True
@@ -914,6 +1052,13 @@ later is required to fix a server side protocol bug.
914 print('\nerror: Local checkouts *not* updated.', file=sys.stderr) 1052 print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
915 sys.exit(1) 1053 sys.exit(1)
916 1054
1055 err_update_linkfiles = not self.UpdateCopyLinkfileList()
1056 if err_update_linkfiles:
1057 err_event.set()
1058 if opt.fail_fast:
1059 print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr)
1060 sys.exit(1)
1061
917 err_results = [] 1062 err_results = []
918 # NB: We don't exit here because this is the last step. 1063 # NB: We don't exit here because this is the last step.
919 err_checkout = not self._Checkout(all_projects, opt, err_results) 1064 err_checkout = not self._Checkout(all_projects, opt, err_results)
@@ -932,6 +1077,8 @@ later is required to fix a server side protocol bug.
932 print('error: Downloading network changes failed.', file=sys.stderr) 1077 print('error: Downloading network changes failed.', file=sys.stderr)
933 if err_update_projects: 1078 if err_update_projects:
934 print('error: Updating local project lists failed.', file=sys.stderr) 1079 print('error: Updating local project lists failed.', file=sys.stderr)
1080 if err_update_linkfiles:
1081 print('error: Updating copyfiles or linkfiles failed.', file=sys.stderr)
935 if err_checkout: 1082 if err_checkout:
936 print('error: Checking out local projects failed.', file=sys.stderr) 1083 print('error: Checking out local projects failed.', file=sys.stderr)
937 if err_results: 1084 if err_results:
@@ -940,6 +1087,15 @@ later is required to fix a server side protocol bug.
940 file=sys.stderr) 1087 file=sys.stderr)
941 sys.exit(1) 1088 sys.exit(1)
942 1089
1090 # Log the previous sync analysis state from the config.
1091 self.git_event_log.LogDataConfigEvents(mp.config.GetSyncAnalysisStateData(),
1092 'previous_sync_state')
1093
1094 # Update and log with the new sync analysis state.
1095 mp.config.UpdateSyncAnalysisState(opt, superproject_logging_data)
1096 self.git_event_log.LogDataConfigEvents(mp.config.GetSyncAnalysisStateData(),
1097 'current_sync_state')
1098
943 if not opt.quiet: 1099 if not opt.quiet:
944 print('repo sync has finished successfully.') 1100 print('repo sync has finished successfully.')
945 1101
@@ -1011,10 +1167,7 @@ class _FetchTimes(object):
1011 with open(self._path) as f: 1167 with open(self._path) as f:
1012 self._times = json.load(f) 1168 self._times = json.load(f)
1013 except (IOError, ValueError): 1169 except (IOError, ValueError):
1014 try: 1170 platform_utils.remove(self._path, missing_ok=True)
1015 platform_utils.remove(self._path)
1016 except OSError:
1017 pass
1018 self._times = {} 1171 self._times = {}
1019 1172
1020 def Save(self): 1173 def Save(self):
@@ -1032,10 +1185,7 @@ class _FetchTimes(object):
1032 with open(self._path, 'w') as f: 1185 with open(self._path, 'w') as f:
1033 json.dump(self._times, f, indent=2) 1186 json.dump(self._times, f, indent=2)
1034 except (IOError, TypeError): 1187 except (IOError, TypeError):
1035 try: 1188 platform_utils.remove(self._path, missing_ok=True)
1036 platform_utils.remove(self._path)
1037 except OSError:
1038 pass
1039 1189
1040# This is a replacement for xmlrpc.client.Transport using urllib2 1190# This is a replacement for xmlrpc.client.Transport using urllib2
1041# and supporting persistent-http[s]. It cannot change hosts from 1191# and supporting persistent-http[s]. It cannot change hosts from
diff --git a/subcmds/upload.py b/subcmds/upload.py
index 50dccc52..c48deab6 100644
--- a/subcmds/upload.py
+++ b/subcmds/upload.py
@@ -13,10 +13,12 @@
13# limitations under the License. 13# limitations under the License.
14 14
15import copy 15import copy
16import functools
17import optparse
16import re 18import re
17import sys 19import sys
18 20
19from command import InteractiveCommand 21from command import DEFAULT_LOCAL_JOBS, InteractiveCommand
20from editor import Editor 22from editor import Editor
21from error import UploadError 23from error import UploadError
22from git_command import GitCommand 24from git_command import GitCommand
@@ -53,7 +55,7 @@ def _SplitEmails(values):
53 55
54 56
55class Upload(InteractiveCommand): 57class Upload(InteractiveCommand):
56 common = True 58 COMMON = True
57 helpSummary = "Upload changes for code review" 59 helpSummary = "Upload changes for code review"
58 helpUsage = """ 60 helpUsage = """
59%prog [--re --cc] [<project>]... 61%prog [--re --cc] [<project>]...
@@ -145,58 +147,66 @@ https://gerrit-review.googlesource.com/Documentation/user-upload.html#notify
145Gerrit Code Review: https://www.gerritcodereview.com/ 147Gerrit Code Review: https://www.gerritcodereview.com/
146 148
147""" 149"""
150 PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
148 151
149 def _Options(self, p): 152 def _Options(self, p):
150 p.add_option('-t', 153 p.add_option('-t',
151 dest='auto_topic', action='store_true', 154 dest='auto_topic', action='store_true',
152 help='Send local branch name to Gerrit Code Review') 155 help='send local branch name to Gerrit Code Review')
153 p.add_option('--hashtag', '--ht', 156 p.add_option('--hashtag', '--ht',
154 dest='hashtags', action='append', default=[], 157 dest='hashtags', action='append', default=[],
155 help='Add hashtags (comma delimited) to the review.') 158 help='add hashtags (comma delimited) to the review')
156 p.add_option('--hashtag-branch', '--htb', 159 p.add_option('--hashtag-branch', '--htb',
157 action='store_true', 160 action='store_true',
158 help='Add local branch name as a hashtag.') 161 help='add local branch name as a hashtag')
159 p.add_option('-l', '--label', 162 p.add_option('-l', '--label',
160 dest='labels', action='append', default=[], 163 dest='labels', action='append', default=[],
161 help='Add a label when uploading.') 164 help='add a label when uploading')
162 p.add_option('--re', '--reviewers', 165 p.add_option('--re', '--reviewers',
163 type='string', action='append', dest='reviewers', 166 type='string', action='append', dest='reviewers',
164 help='Request reviews from these people.') 167 help='request reviews from these people')
165 p.add_option('--cc', 168 p.add_option('--cc',
166 type='string', action='append', dest='cc', 169 type='string', action='append', dest='cc',
167 help='Also send email to these email addresses.') 170 help='also send email to these email addresses')
168 p.add_option('--br', 171 p.add_option('--br', '--branch',
169 type='string', action='store', dest='branch', 172 type='string', action='store', dest='branch',
170 help='Branch to upload.') 173 help='(local) branch to upload')
171 p.add_option('--cbr', '--current-branch', 174 p.add_option('-c', '--current-branch',
172 dest='current_branch', action='store_true', 175 dest='current_branch', action='store_true',
173 help='Upload current git branch.') 176 help='upload current git branch')
177 p.add_option('--no-current-branch',
178 dest='current_branch', action='store_false',
179 help='upload all git branches')
180 # Turn this into a warning & remove this someday.
181 p.add_option('--cbr',
182 dest='current_branch', action='store_true',
183 help=optparse.SUPPRESS_HELP)
174 p.add_option('--ne', '--no-emails', 184 p.add_option('--ne', '--no-emails',
175 action='store_false', dest='notify', default=True, 185 action='store_false', dest='notify', default=True,
176 help='If specified, do not send emails on upload.') 186 help='do not send e-mails on upload')
177 p.add_option('-p', '--private', 187 p.add_option('-p', '--private',
178 action='store_true', dest='private', default=False, 188 action='store_true', dest='private', default=False,
179 help='If specified, upload as a private change.') 189 help='upload as a private change (deprecated; use --wip)')
180 p.add_option('-w', '--wip', 190 p.add_option('-w', '--wip',
181 action='store_true', dest='wip', default=False, 191 action='store_true', dest='wip', default=False,
182 help='If specified, upload as a work-in-progress change.') 192 help='upload as a work-in-progress change')
183 p.add_option('-o', '--push-option', 193 p.add_option('-o', '--push-option',
184 type='string', action='append', dest='push_options', 194 type='string', action='append', dest='push_options',
185 default=[], 195 default=[],
186 help='Additional push options to transmit') 196 help='additional push options to transmit')
187 p.add_option('-D', '--destination', '--dest', 197 p.add_option('-D', '--destination', '--dest',
188 type='string', action='store', dest='dest_branch', 198 type='string', action='store', dest='dest_branch',
189 metavar='BRANCH', 199 metavar='BRANCH',
190 help='Submit for review on this target branch.') 200 help='submit for review on this target branch')
191 p.add_option('-n', '--dry-run', 201 p.add_option('-n', '--dry-run',
192 dest='dryrun', default=False, action='store_true', 202 dest='dryrun', default=False, action='store_true',
193 help='Do everything except actually upload the CL.') 203 help='do everything except actually upload the CL')
194 p.add_option('-y', '--yes', 204 p.add_option('-y', '--yes',
195 default=False, action='store_true', 205 default=False, action='store_true',
196 help='Answer yes to all safe prompts.') 206 help='answer yes to all safe prompts')
197 p.add_option('--no-cert-checks', 207 p.add_option('--no-cert-checks',
198 dest='validate_certs', action='store_false', default=True, 208 dest='validate_certs', action='store_false', default=True,
199 help='Disable verifying ssl certs (unsafe).') 209 help='disable verifying ssl certs (unsafe)')
200 RepoHook.AddOptionGroup(p, 'pre-upload') 210 RepoHook.AddOptionGroup(p, 'pre-upload')
201 211
202 def _SingleBranch(self, opt, branch, people): 212 def _SingleBranch(self, opt, branch, people):
@@ -502,40 +512,46 @@ Gerrit Code Review: https://www.gerritcodereview.com/
502 merge_branch = p.stdout.strip() 512 merge_branch = p.stdout.strip()
503 return merge_branch 513 return merge_branch
504 514
515 @staticmethod
516 def _GatherOne(opt, project):
517 """Figure out the upload status for |project|."""
518 if opt.current_branch:
519 cbr = project.CurrentBranch
520 up_branch = project.GetUploadableBranch(cbr)
521 avail = [up_branch] if up_branch else None
522 else:
523 avail = project.GetUploadableBranches(opt.branch)
524 return (project, avail)
525
505 def Execute(self, opt, args): 526 def Execute(self, opt, args):
506 project_list = self.GetProjects(args) 527 projects = self.GetProjects(args)
507 pending = [] 528
508 reviewers = [] 529 def _ProcessResults(_pool, _out, results):
509 cc = [] 530 pending = []
510 branch = None 531 for result in results:
511 532 project, avail = result
512 if opt.branch: 533 if avail is None:
513 branch = opt.branch 534 print('repo: error: %s: Unable to upload branch "%s". '
514
515 for project in project_list:
516 if opt.current_branch:
517 cbr = project.CurrentBranch
518 up_branch = project.GetUploadableBranch(cbr)
519 if up_branch:
520 avail = [up_branch]
521 else:
522 avail = None
523 print('repo: error: Unable to upload branch "%s". '
524 'You might be able to fix the branch by running:\n' 535 'You might be able to fix the branch by running:\n'
525 ' git branch --set-upstream-to m/%s' % 536 ' git branch --set-upstream-to m/%s' %
526 (str(cbr), self.manifest.branch), 537 (project.relpath, project.CurrentBranch, self.manifest.branch),
527 file=sys.stderr) 538 file=sys.stderr)
528 else: 539 elif avail:
529 avail = project.GetUploadableBranches(branch) 540 pending.append(result)
530 if avail: 541 return pending
531 pending.append((project, avail)) 542
543 pending = self.ExecuteInParallel(
544 opt.jobs,
545 functools.partial(self._GatherOne, opt),
546 projects,
547 callback=_ProcessResults)
532 548
533 if not pending: 549 if not pending:
534 if branch is None: 550 if opt.branch is None:
535 print('repo: error: no branches ready for upload', file=sys.stderr) 551 print('repo: error: no branches ready for upload', file=sys.stderr)
536 else: 552 else:
537 print('repo: error: no branches named "%s" ready for upload' % 553 print('repo: error: no branches named "%s" ready for upload' %
538 (branch,), file=sys.stderr) 554 (opt.branch,), file=sys.stderr)
539 return 1 555 return 1
540 556
541 pending_proj_names = [project.name for (project, available) in pending] 557 pending_proj_names = [project.name for (project, available) in pending]
@@ -548,10 +564,8 @@ Gerrit Code Review: https://www.gerritcodereview.com/
548 worktree_list=pending_worktrees): 564 worktree_list=pending_worktrees):
549 return 1 565 return 1
550 566
551 if opt.reviewers: 567 reviewers = _SplitEmails(opt.reviewers) if opt.reviewers else []
552 reviewers = _SplitEmails(opt.reviewers) 568 cc = _SplitEmails(opt.cc) if opt.cc else []
553 if opt.cc:
554 cc = _SplitEmails(opt.cc)
555 people = (reviewers, cc) 569 people = (reviewers, cc)
556 570
557 if len(pending) == 1 and len(pending[0][1]) == 1: 571 if len(pending) == 1 and len(pending[0][1]) == 1:
diff --git a/subcmds/version.py b/subcmds/version.py
index e95a86dc..09b053ea 100644
--- a/subcmds/version.py
+++ b/subcmds/version.py
@@ -18,13 +18,14 @@ import sys
18from command import Command, MirrorSafeCommand 18from command import Command, MirrorSafeCommand
19from git_command import git, RepoSourceVersion, user_agent 19from git_command import git, RepoSourceVersion, user_agent
20from git_refs import HEAD 20from git_refs import HEAD
21from wrapper import Wrapper
21 22
22 23
23class Version(Command, MirrorSafeCommand): 24class Version(Command, MirrorSafeCommand):
24 wrapper_version = None 25 wrapper_version = None
25 wrapper_path = None 26 wrapper_path = None
26 27
27 common = False 28 COMMON = False
28 helpSummary = "Display the version of repo" 29 helpSummary = "Display the version of repo"
29 helpUsage = """ 30 helpUsage = """
30%prog 31%prog
@@ -62,3 +63,4 @@ class Version(Command, MirrorSafeCommand):
62 print('OS %s %s (%s)' % (uname.system, uname.release, uname.version)) 63 print('OS %s %s (%s)' % (uname.system, uname.release, uname.version))
63 print('CPU %s (%s)' % 64 print('CPU %s (%s)' %
64 (uname.machine, uname.processor if uname.processor else 'unknown')) 65 (uname.machine, uname.processor if uname.processor else 'unknown'))
66 print('Bug reports:', Wrapper().BUG_URL)
diff --git a/tests/fixtures/test.gitconfig b/tests/fixtures/test.gitconfig
index 9b3f2574..b178cf60 100644
--- a/tests/fixtures/test.gitconfig
+++ b/tests/fixtures/test.gitconfig
@@ -11,3 +11,13 @@
11 intk = 10k 11 intk = 10k
12 intm = 10m 12 intm = 10m
13 intg = 10g 13 intg = 10g
14[repo "syncstate.main"]
15 synctime = 2021-09-14T17:23:43.537338Z
16 version = 1
17[repo "syncstate.sys"]
18 argv = ['/usr/bin/pytest-3']
19[repo "syncstate.superproject"]
20 test = false
21[repo "syncstate.options"]
22 verbose = true
23 mpupdate = false
diff --git a/tests/test_git_command.py b/tests/test_git_command.py
index 912a9dbe..93300a6f 100644
--- a/tests/test_git_command.py
+++ b/tests/test_git_command.py
@@ -26,33 +26,6 @@ import git_command
26import wrapper 26import wrapper
27 27
28 28
29class SSHUnitTest(unittest.TestCase):
30 """Tests the ssh functions."""
31
32 def test_ssh_version(self):
33 """Check ssh_version() handling."""
34 ver = git_command._parse_ssh_version('Unknown\n')
35 self.assertEqual(ver, ())
36 ver = git_command._parse_ssh_version('OpenSSH_1.0\n')
37 self.assertEqual(ver, (1, 0))
38 ver = git_command._parse_ssh_version('OpenSSH_6.6.1p1 Ubuntu-2ubuntu2.13, OpenSSL 1.0.1f 6 Jan 2014\n')
39 self.assertEqual(ver, (6, 6, 1))
40 ver = git_command._parse_ssh_version('OpenSSH_7.6p1 Ubuntu-4ubuntu0.3, OpenSSL 1.0.2n 7 Dec 2017\n')
41 self.assertEqual(ver, (7, 6))
42
43 def test_ssh_sock(self):
44 """Check ssh_sock() function."""
45 with mock.patch('tempfile.mkdtemp', return_value='/tmp/foo'):
46 # old ssh version uses port
47 with mock.patch('git_command.ssh_version', return_value=(6, 6)):
48 self.assertTrue(git_command.ssh_sock().endswith('%p'))
49 git_command._ssh_sock_path = None
50 # new ssh version uses hash
51 with mock.patch('git_command.ssh_version', return_value=(6, 7)):
52 self.assertTrue(git_command.ssh_sock().endswith('%C'))
53 git_command._ssh_sock_path = None
54
55
56class GitCallUnitTest(unittest.TestCase): 29class GitCallUnitTest(unittest.TestCase):
57 """Tests the _GitCall class (via git_command.git).""" 30 """Tests the _GitCall class (via git_command.git)."""
58 31
diff --git a/tests/test_git_config.py b/tests/test_git_config.py
index 3300c12f..faf12a2e 100644
--- a/tests/test_git_config.py
+++ b/tests/test_git_config.py
@@ -104,6 +104,25 @@ class GitConfigReadOnlyTests(unittest.TestCase):
104 for key, value in TESTS: 104 for key, value in TESTS:
105 self.assertEqual(value, self.config.GetInt('section.%s' % (key,))) 105 self.assertEqual(value, self.config.GetInt('section.%s' % (key,)))
106 106
107 def test_GetSyncAnalysisStateData(self):
108 """Test config entries with a sync state analysis data."""
109 superproject_logging_data = {}
110 superproject_logging_data['test'] = False
111 options = type('options', (object,), {})()
112 options.verbose = 'true'
113 options.mp_update = 'false'
114 TESTS = (
115 ('superproject.test', 'false'),
116 ('options.verbose', 'true'),
117 ('options.mpupdate', 'false'),
118 ('main.version', '1'),
119 )
120 self.config.UpdateSyncAnalysisState(options, superproject_logging_data)
121 sync_data = self.config.GetSyncAnalysisStateData()
122 for key, value in TESTS:
123 self.assertEqual(sync_data[f'{git_config.SYNC_STATE_PREFIX}{key}'], value)
124 self.assertTrue(sync_data[f'{git_config.SYNC_STATE_PREFIX}main.synctime'])
125
107 126
108class GitConfigReadWriteTests(unittest.TestCase): 127class GitConfigReadWriteTests(unittest.TestCase):
109 """Read/write tests of the GitConfig class.""" 128 """Read/write tests of the GitConfig class."""
diff --git a/tests/test_git_superproject.py b/tests/test_git_superproject.py
index 9550949b..a24fc7f0 100644
--- a/tests/test_git_superproject.py
+++ b/tests/test_git_superproject.py
@@ -14,6 +14,7 @@
14 14
15"""Unittests for the git_superproject.py module.""" 15"""Unittests for the git_superproject.py module."""
16 16
17import json
17import os 18import os
18import platform 19import platform
19import tempfile 20import tempfile
@@ -21,13 +22,20 @@ import unittest
21from unittest import mock 22from unittest import mock
22 23
23import git_superproject 24import git_superproject
25import git_trace2_event_log
24import manifest_xml 26import manifest_xml
25import platform_utils 27import platform_utils
28from test_manifest_xml import sort_attributes
26 29
27 30
28class SuperprojectTestCase(unittest.TestCase): 31class SuperprojectTestCase(unittest.TestCase):
29 """TestCase for the Superproject module.""" 32 """TestCase for the Superproject module."""
30 33
34 PARENT_SID_KEY = 'GIT_TRACE2_PARENT_SID'
35 PARENT_SID_VALUE = 'parent_sid'
36 SELF_SID_REGEX = r'repo-\d+T\d+Z-.*'
37 FULL_SID_REGEX = r'^%s/%s' % (PARENT_SID_VALUE, SELF_SID_REGEX)
38
31 def setUp(self): 39 def setUp(self):
32 """Set up superproject every time.""" 40 """Set up superproject every time."""
33 self.tempdir = tempfile.mkdtemp(prefix='repo_tests') 41 self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
@@ -37,6 +45,13 @@ class SuperprojectTestCase(unittest.TestCase):
37 os.mkdir(self.repodir) 45 os.mkdir(self.repodir)
38 self.platform = platform.system().lower() 46 self.platform = platform.system().lower()
39 47
48 # By default we initialize with the expected case where
49 # repo launches us (so GIT_TRACE2_PARENT_SID is set).
50 env = {
51 self.PARENT_SID_KEY: self.PARENT_SID_VALUE,
52 }
53 self.git_event_log = git_trace2_event_log.EventLog(env=env)
54
40 # The manifest parsing really wants a git repo currently. 55 # The manifest parsing really wants a git repo currently.
41 gitdir = os.path.join(self.repodir, 'manifests.git') 56 gitdir = os.path.join(self.repodir, 'manifests.git')
42 os.mkdir(gitdir) 57 os.mkdir(gitdir)
@@ -53,7 +68,8 @@ class SuperprojectTestCase(unittest.TestCase):
53 <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """ 68 <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """
54 " /></manifest> 69 " /></manifest>
55""") 70""")
56 self._superproject = git_superproject.Superproject(manifest, self.repodir) 71 self._superproject = git_superproject.Superproject(manifest, self.repodir,
72 self.git_event_log)
57 73
58 def tearDown(self): 74 def tearDown(self):
59 """Tear down superproject every time.""" 75 """Tear down superproject every time."""
@@ -65,14 +81,56 @@ class SuperprojectTestCase(unittest.TestCase):
65 fp.write(data) 81 fp.write(data)
66 return manifest_xml.XmlManifest(self.repodir, self.manifest_file) 82 return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
67 83
84 def verifyCommonKeys(self, log_entry, expected_event_name, full_sid=True):
85 """Helper function to verify common event log keys."""
86 self.assertIn('event', log_entry)
87 self.assertIn('sid', log_entry)
88 self.assertIn('thread', log_entry)
89 self.assertIn('time', log_entry)
90
91 # Do basic data format validation.
92 self.assertEqual(expected_event_name, log_entry['event'])
93 if full_sid:
94 self.assertRegex(log_entry['sid'], self.FULL_SID_REGEX)
95 else:
96 self.assertRegex(log_entry['sid'], self.SELF_SID_REGEX)
97 self.assertRegex(log_entry['time'], r'^\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z$')
98
99 def readLog(self, log_path):
100 """Helper function to read log data into a list."""
101 log_data = []
102 with open(log_path, mode='rb') as f:
103 for line in f:
104 log_data.append(json.loads(line))
105 return log_data
106
107 def verifyErrorEvent(self):
108 """Helper to verify that error event is written."""
109
110 with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
111 log_path = self.git_event_log.Write(path=tempdir)
112 self.log_data = self.readLog(log_path)
113
114 self.assertEqual(len(self.log_data), 2)
115 error_event = self.log_data[1]
116 self.verifyCommonKeys(self.log_data[0], expected_event_name='version')
117 self.verifyCommonKeys(error_event, expected_event_name='error')
118 # Check for 'error' event specific fields.
119 self.assertIn('msg', error_event)
120 self.assertIn('fmt', error_event)
121
68 def test_superproject_get_superproject_no_superproject(self): 122 def test_superproject_get_superproject_no_superproject(self):
69 """Test with no url.""" 123 """Test with no url."""
70 manifest = self.getXmlManifest(""" 124 manifest = self.getXmlManifest("""
71<manifest> 125<manifest>
72</manifest> 126</manifest>
73""") 127""")
74 superproject = git_superproject.Superproject(manifest, self.repodir) 128 superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log)
75 self.assertFalse(superproject.Sync()) 129 # Test that exit condition is false when there is no superproject tag.
130 sync_result = superproject.Sync()
131 self.assertFalse(sync_result.success)
132 self.assertFalse(sync_result.fatal)
133 self.verifyErrorEvent()
76 134
77 def test_superproject_get_superproject_invalid_url(self): 135 def test_superproject_get_superproject_invalid_url(self):
78 """Test with an invalid url.""" 136 """Test with an invalid url."""
@@ -83,8 +141,10 @@ class SuperprojectTestCase(unittest.TestCase):
83 <superproject name="superproject"/> 141 <superproject name="superproject"/>
84</manifest> 142</manifest>
85""") 143""")
86 superproject = git_superproject.Superproject(manifest, self.repodir) 144 superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log)
87 self.assertFalse(superproject.Sync()) 145 sync_result = superproject.Sync()
146 self.assertFalse(sync_result.success)
147 self.assertTrue(sync_result.fatal)
88 148
89 def test_superproject_get_superproject_invalid_branch(self): 149 def test_superproject_get_superproject_invalid_branch(self):
90 """Test with an invalid branch.""" 150 """Test with an invalid branch."""
@@ -95,21 +155,28 @@ class SuperprojectTestCase(unittest.TestCase):
95 <superproject name="superproject"/> 155 <superproject name="superproject"/>
96</manifest> 156</manifest>
97""") 157""")
98 superproject = git_superproject.Superproject(manifest, self.repodir) 158 self._superproject = git_superproject.Superproject(manifest, self.repodir,
99 with mock.patch.object(self._superproject, '_GetBranch', return_value='junk'): 159 self.git_event_log)
100 self.assertFalse(superproject.Sync()) 160 with mock.patch.object(self._superproject, '_branch', 'junk'):
161 sync_result = self._superproject.Sync()
162 self.assertFalse(sync_result.success)
163 self.assertTrue(sync_result.fatal)
101 164
102 def test_superproject_get_superproject_mock_init(self): 165 def test_superproject_get_superproject_mock_init(self):
103 """Test with _Init failing.""" 166 """Test with _Init failing."""
104 with mock.patch.object(self._superproject, '_Init', return_value=False): 167 with mock.patch.object(self._superproject, '_Init', return_value=False):
105 self.assertFalse(self._superproject.Sync()) 168 sync_result = self._superproject.Sync()
169 self.assertFalse(sync_result.success)
170 self.assertTrue(sync_result.fatal)
106 171
107 def test_superproject_get_superproject_mock_fetch(self): 172 def test_superproject_get_superproject_mock_fetch(self):
108 """Test with _Fetch failing.""" 173 """Test with _Fetch failing."""
109 with mock.patch.object(self._superproject, '_Init', return_value=True): 174 with mock.patch.object(self._superproject, '_Init', return_value=True):
110 os.mkdir(self._superproject._superproject_path) 175 os.mkdir(self._superproject._superproject_path)
111 with mock.patch.object(self._superproject, '_Fetch', return_value=False): 176 with mock.patch.object(self._superproject, '_Fetch', return_value=False):
112 self.assertFalse(self._superproject.Sync()) 177 sync_result = self._superproject.Sync()
178 self.assertFalse(sync_result.success)
179 self.assertTrue(sync_result.fatal)
113 180
114 def test_superproject_get_all_project_commit_ids_mock_ls_tree(self): 181 def test_superproject_get_all_project_commit_ids_mock_ls_tree(self):
115 """Test with LsTree being a mock.""" 182 """Test with LsTree being a mock."""
@@ -121,12 +188,13 @@ class SuperprojectTestCase(unittest.TestCase):
121 with mock.patch.object(self._superproject, '_Init', return_value=True): 188 with mock.patch.object(self._superproject, '_Init', return_value=True):
122 with mock.patch.object(self._superproject, '_Fetch', return_value=True): 189 with mock.patch.object(self._superproject, '_Fetch', return_value=True):
123 with mock.patch.object(self._superproject, '_LsTree', return_value=data): 190 with mock.patch.object(self._superproject, '_LsTree', return_value=data):
124 commit_ids = self._superproject._GetAllProjectsCommitIds() 191 commit_ids_result = self._superproject._GetAllProjectsCommitIds()
125 self.assertEqual(commit_ids, { 192 self.assertEqual(commit_ids_result.commit_ids, {
126 'art': '2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea', 193 'art': '2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea',
127 'bootable/recovery': 'e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06', 194 'bootable/recovery': 'e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06',
128 'build/bazel': 'ade9b7a0d874e25fff4bf2552488825c6f111928' 195 'build/bazel': 'ade9b7a0d874e25fff4bf2552488825c6f111928'
129 }) 196 })
197 self.assertFalse(commit_ids_result.fatal)
130 198
131 def test_superproject_write_manifest_file(self): 199 def test_superproject_write_manifest_file(self):
132 """Test with writing manifest to a file after setting revisionId.""" 200 """Test with writing manifest to a file after setting revisionId."""
@@ -135,18 +203,18 @@ class SuperprojectTestCase(unittest.TestCase):
135 project.SetRevisionId('ABCDEF') 203 project.SetRevisionId('ABCDEF')
136 # Create temporary directory so that it can write the file. 204 # Create temporary directory so that it can write the file.
137 os.mkdir(self._superproject._superproject_path) 205 os.mkdir(self._superproject._superproject_path)
138 manifest_path = self._superproject._WriteManfiestFile() 206 manifest_path = self._superproject._WriteManifestFile()
139 self.assertIsNotNone(manifest_path) 207 self.assertIsNotNone(manifest_path)
140 with open(manifest_path, 'r') as fp: 208 with open(manifest_path, 'r') as fp:
141 manifest_xml = fp.read() 209 manifest_xml_data = fp.read()
142 self.assertEqual( 210 self.assertEqual(
143 manifest_xml, 211 sort_attributes(manifest_xml_data),
144 '<?xml version="1.0" ?><manifest>' + 212 '<?xml version="1.0" ?><manifest>'
145 '<remote name="default-remote" fetch="http://localhost"/>' + 213 '<remote fetch="http://localhost" name="default-remote"/>'
146 '<default remote="default-remote" revision="refs/heads/main"/>' + 214 '<default remote="default-remote" revision="refs/heads/main"/>'
147 '<project name="platform/art" path="art" revision="ABCDEF" ' + 215 '<project groups="notdefault,platform-' + self.platform + '" '
148 'groups="notdefault,platform-' + self.platform + '"/>' + 216 'name="platform/art" path="art" revision="ABCDEF" upstream="refs/heads/main"/>'
149 '<superproject name="superproject"/>' + 217 '<superproject name="superproject"/>'
150 '</manifest>') 218 '</manifest>')
151 219
152 def test_superproject_update_project_revision_id(self): 220 def test_superproject_update_project_revision_id(self):
@@ -162,19 +230,145 @@ class SuperprojectTestCase(unittest.TestCase):
162 return_value=data): 230 return_value=data):
163 # Create temporary directory so that it can write the file. 231 # Create temporary directory so that it can write the file.
164 os.mkdir(self._superproject._superproject_path) 232 os.mkdir(self._superproject._superproject_path)
165 manifest_path = self._superproject.UpdateProjectsRevisionId(projects) 233 update_result = self._superproject.UpdateProjectsRevisionId(projects)
166 self.assertIsNotNone(manifest_path) 234 self.assertIsNotNone(update_result.manifest_path)
167 with open(manifest_path, 'r') as fp: 235 self.assertFalse(update_result.fatal)
168 manifest_xml = fp.read() 236 with open(update_result.manifest_path, 'r') as fp:
237 manifest_xml_data = fp.read()
238 self.assertEqual(
239 sort_attributes(manifest_xml_data),
240 '<?xml version="1.0" ?><manifest>'
241 '<remote fetch="http://localhost" name="default-remote"/>'
242 '<default remote="default-remote" revision="refs/heads/main"/>'
243 '<project groups="notdefault,platform-' + self.platform + '" '
244 'name="platform/art" path="art" '
245 'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>'
246 '<superproject name="superproject"/>'
247 '</manifest>')
248
249 def test_superproject_update_project_revision_id_no_superproject_tag(self):
250 """Test update of commit ids of a manifest without superproject tag."""
251 manifest = self.getXmlManifest("""
252<manifest>
253 <remote name="default-remote" fetch="http://localhost" />
254 <default remote="default-remote" revision="refs/heads/main" />
255 <project name="test-name"/>
256</manifest>
257""")
258 self.maxDiff = None
259 self._superproject = git_superproject.Superproject(manifest, self.repodir,
260 self.git_event_log)
261 self.assertEqual(len(self._superproject._manifest.projects), 1)
262 projects = self._superproject._manifest.projects
263 project = projects[0]
264 project.SetRevisionId('ABCDEF')
265 update_result = self._superproject.UpdateProjectsRevisionId(projects)
266 self.assertIsNone(update_result.manifest_path)
267 self.assertFalse(update_result.fatal)
268 self.verifyErrorEvent()
269 self.assertEqual(
270 sort_attributes(manifest.ToXml().toxml()),
271 '<?xml version="1.0" ?><manifest>'
272 '<remote fetch="http://localhost" name="default-remote"/>'
273 '<default remote="default-remote" revision="refs/heads/main"/>'
274 '<project name="test-name" revision="ABCDEF" upstream="refs/heads/main"/>'
275 '</manifest>')
276
277 def test_superproject_update_project_revision_id_from_local_manifest_group(self):
278 """Test update of commit ids of a manifest that have local manifest no superproject group."""
279 local_group = manifest_xml.LOCAL_MANIFEST_GROUP_PREFIX + ':local'
280 manifest = self.getXmlManifest("""
281<manifest>
282 <remote name="default-remote" fetch="http://localhost" />
283 <remote name="goog" fetch="http://localhost2" />
284 <default remote="default-remote" revision="refs/heads/main" />
285 <superproject name="superproject"/>
286 <project path="vendor/x" name="platform/vendor/x" remote="goog"
287 groups=\"""" + local_group + """
288 " revision="master-with-vendor" clone-depth="1" />
289 <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """
290 " /></manifest>
291""")
292 self.maxDiff = None
293 self._superproject = git_superproject.Superproject(manifest, self.repodir,
294 self.git_event_log)
295 self.assertEqual(len(self._superproject._manifest.projects), 2)
296 projects = self._superproject._manifest.projects
297 data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00')
298 with mock.patch.object(self._superproject, '_Init', return_value=True):
299 with mock.patch.object(self._superproject, '_Fetch', return_value=True):
300 with mock.patch.object(self._superproject,
301 '_LsTree',
302 return_value=data):
303 # Create temporary directory so that it can write the file.
304 os.mkdir(self._superproject._superproject_path)
305 update_result = self._superproject.UpdateProjectsRevisionId(projects)
306 self.assertIsNotNone(update_result.manifest_path)
307 self.assertFalse(update_result.fatal)
308 with open(update_result.manifest_path, 'r') as fp:
309 manifest_xml_data = fp.read()
310 # Verify platform/vendor/x's project revision hasn't changed.
311 self.assertEqual(
312 sort_attributes(manifest_xml_data),
313 '<?xml version="1.0" ?><manifest>'
314 '<remote fetch="http://localhost" name="default-remote"/>'
315 '<remote fetch="http://localhost2" name="goog"/>'
316 '<default remote="default-remote" revision="refs/heads/main"/>'
317 '<project groups="notdefault,platform-' + self.platform + '" '
318 'name="platform/art" path="art" '
319 'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>'
320 '<project clone-depth="1" groups="' + local_group + '" '
321 'name="platform/vendor/x" path="vendor/x" remote="goog" '
322 'revision="master-with-vendor"/>'
323 '<superproject name="superproject"/>'
324 '</manifest>')
325
326 def test_superproject_update_project_revision_id_with_pinned_manifest(self):
327 """Test update of commit ids of a pinned manifest."""
328 manifest = self.getXmlManifest("""
329<manifest>
330 <remote name="default-remote" fetch="http://localhost" />
331 <default remote="default-remote" revision="refs/heads/main" />
332 <superproject name="superproject"/>
333 <project path="vendor/x" name="platform/vendor/x" revision="" />
334 <project path="vendor/y" name="platform/vendor/y"
335 revision="52d3c9f7c107839ece2319d077de0cd922aa9d8f" />
336 <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """
337 " /></manifest>
338""")
339 self.maxDiff = None
340 self._superproject = git_superproject.Superproject(manifest, self.repodir,
341 self.git_event_log)
342 self.assertEqual(len(self._superproject._manifest.projects), 3)
343 projects = self._superproject._manifest.projects
344 data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00'
345 '160000 commit e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06\tvendor/x\x00')
346 with mock.patch.object(self._superproject, '_Init', return_value=True):
347 with mock.patch.object(self._superproject, '_Fetch', return_value=True):
348 with mock.patch.object(self._superproject,
349 '_LsTree',
350 return_value=data):
351 # Create temporary directory so that it can write the file.
352 os.mkdir(self._superproject._superproject_path)
353 update_result = self._superproject.UpdateProjectsRevisionId(projects)
354 self.assertIsNotNone(update_result.manifest_path)
355 self.assertFalse(update_result.fatal)
356 with open(update_result.manifest_path, 'r') as fp:
357 manifest_xml_data = fp.read()
358 # Verify platform/vendor/x's project revision hasn't changed.
169 self.assertEqual( 359 self.assertEqual(
170 manifest_xml, 360 sort_attributes(manifest_xml_data),
171 '<?xml version="1.0" ?><manifest>' + 361 '<?xml version="1.0" ?><manifest>'
172 '<remote name="default-remote" fetch="http://localhost"/>' + 362 '<remote fetch="http://localhost" name="default-remote"/>'
173 '<default remote="default-remote" revision="refs/heads/main"/>' + 363 '<default remote="default-remote" revision="refs/heads/main"/>'
174 '<project name="platform/art" path="art" ' + 364 '<project groups="notdefault,platform-' + self.platform + '" '
175 'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" ' + 365 'name="platform/art" path="art" '
176 'groups="notdefault,platform-' + self.platform + '"/>' + 366 'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>'
177 '<superproject name="superproject"/>' + 367 '<project name="platform/vendor/x" path="vendor/x" '
368 'revision="e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06" upstream="refs/heads/main"/>'
369 '<project name="platform/vendor/y" path="vendor/y" '
370 'revision="52d3c9f7c107839ece2319d077de0cd922aa9d8f"/>'
371 '<superproject name="superproject"/>'
178 '</manifest>') 372 '</manifest>')
179 373
180 374
diff --git a/tests/test_git_trace2_event_log.py b/tests/test_git_trace2_event_log.py
index 4a3a4c48..89dcfb92 100644
--- a/tests/test_git_trace2_event_log.py
+++ b/tests/test_git_trace2_event_log.py
@@ -42,7 +42,7 @@ class EventLogTestCase(unittest.TestCase):
42 self._event_log_module = git_trace2_event_log.EventLog(env=env) 42 self._event_log_module = git_trace2_event_log.EventLog(env=env)
43 self._log_data = None 43 self._log_data = None
44 44
45 def verifyCommonKeys(self, log_entry, expected_event_name, full_sid=True): 45 def verifyCommonKeys(self, log_entry, expected_event_name=None, full_sid=True):
46 """Helper function to verify common event log keys.""" 46 """Helper function to verify common event log keys."""
47 self.assertIn('event', log_entry) 47 self.assertIn('event', log_entry)
48 self.assertIn('sid', log_entry) 48 self.assertIn('sid', log_entry)
@@ -50,7 +50,8 @@ class EventLogTestCase(unittest.TestCase):
50 self.assertIn('time', log_entry) 50 self.assertIn('time', log_entry)
51 51
52 # Do basic data format validation. 52 # Do basic data format validation.
53 self.assertEqual(expected_event_name, log_entry['event']) 53 if expected_event_name:
54 self.assertEqual(expected_event_name, log_entry['event'])
54 if full_sid: 55 if full_sid:
55 self.assertRegex(log_entry['sid'], self.FULL_SID_REGEX) 56 self.assertRegex(log_entry['sid'], self.FULL_SID_REGEX)
56 else: 57 else:
@@ -65,6 +66,13 @@ class EventLogTestCase(unittest.TestCase):
65 log_data.append(json.loads(line)) 66 log_data.append(json.loads(line))
66 return log_data 67 return log_data
67 68
69 def remove_prefix(self, s, prefix):
70 """Return a copy string after removing |prefix| from |s|, if present or the original string."""
71 if s.startswith(prefix):
72 return s[len(prefix):]
73 else:
74 return s
75
68 def test_initial_state_with_parent_sid(self): 76 def test_initial_state_with_parent_sid(self):
69 """Test initial state when 'GIT_TRACE2_PARENT_SID' is set by parent.""" 77 """Test initial state when 'GIT_TRACE2_PARENT_SID' is set by parent."""
70 self.assertRegex(self._event_log_module.full_sid, self.FULL_SID_REGEX) 78 self.assertRegex(self._event_log_module.full_sid, self.FULL_SID_REGEX)
@@ -234,6 +242,66 @@ class EventLogTestCase(unittest.TestCase):
234 self.assertEqual(len(self._log_data), 1) 242 self.assertEqual(len(self._log_data), 1)
235 self.verifyCommonKeys(self._log_data[0], expected_event_name='version') 243 self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
236 244
245 def test_data_event_config(self):
246 """Test 'data' event data outputs all config keys.
247
248 Expected event log:
249 <version event>
250 <data event>
251 <data event>
252 """
253 config = {
254 'git.foo': 'bar',
255 'repo.partialclone': 'false',
256 'repo.syncstate.superproject.hassuperprojecttag': 'true',
257 'repo.syncstate.superproject.sys.argv': ['--', 'sync', 'protobuf'],
258 }
259 prefix_value = 'prefix'
260 self._event_log_module.LogDataConfigEvents(config, prefix_value)
261
262 with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
263 log_path = self._event_log_module.Write(path=tempdir)
264 self._log_data = self.readLog(log_path)
265
266 self.assertEqual(len(self._log_data), 5)
267 data_events = self._log_data[1:]
268 self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
269
270 for event in data_events:
271 self.verifyCommonKeys(event)
272 # Check for 'data' event specific fields.
273 self.assertIn('key', event)
274 self.assertIn('value', event)
275 key = event['key']
276 key = self.remove_prefix(key, f'{prefix_value}/')
277 value = event['value']
278 self.assertEqual(self._event_log_module.GetDataEventName(value), event['event'])
279 self.assertTrue(key in config and value == config[key])
280
281 def test_error_event(self):
282 """Test and validate 'error' event data is valid.
283
284 Expected event log:
285 <version event>
286 <error event>
287 """
288 msg = 'invalid option: --cahced'
289 fmt = 'invalid option: %s'
290 self._event_log_module.ErrorEvent(msg, fmt)
291 with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
292 log_path = self._event_log_module.Write(path=tempdir)
293 self._log_data = self.readLog(log_path)
294
295 self.assertEqual(len(self._log_data), 2)
296 error_event = self._log_data[1]
297 self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
298 self.verifyCommonKeys(error_event, expected_event_name='error')
299 # Check for 'error' event specific fields.
300 self.assertIn('msg', error_event)
301 self.assertIn('fmt', error_event)
302 self.assertEqual(error_event['msg'], msg)
303 self.assertEqual(error_event['fmt'], fmt)
304
237 def test_write_with_filename(self): 305 def test_write_with_filename(self):
238 """Test Write() with a path to a file exits with None.""" 306 """Test Write() with a path to a file exits with None."""
239 self.assertIsNone(self._event_log_module.Write(path='path/to/file')) 307 self.assertIsNone(self._event_log_module.Write(path='path/to/file'))
diff --git a/tests/test_manifest_xml.py b/tests/test_manifest_xml.py
index eda06968..cb3eb855 100644
--- a/tests/test_manifest_xml.py
+++ b/tests/test_manifest_xml.py
@@ -16,6 +16,7 @@
16 16
17import os 17import os
18import platform 18import platform
19import re
19import shutil 20import shutil
20import tempfile 21import tempfile
21import unittest 22import unittest
@@ -52,6 +53,9 @@ INVALID_FS_PATHS = (
52 'blah/foo~', 53 'blah/foo~',
53 # Block Unicode characters that get normalized out by filesystems. 54 # Block Unicode characters that get normalized out by filesystems.
54 u'foo\u200Cbar', 55 u'foo\u200Cbar',
56 # Block newlines.
57 'f\n/bar',
58 'f\r/bar',
55) 59)
56 60
57# Make sure platforms that use path separators (e.g. Windows) are also 61# Make sure platforms that use path separators (e.g. Windows) are also
@@ -60,6 +64,30 @@ if os.path.sep != '/':
60 INVALID_FS_PATHS += tuple(x.replace('/', os.path.sep) for x in INVALID_FS_PATHS) 64 INVALID_FS_PATHS += tuple(x.replace('/', os.path.sep) for x in INVALID_FS_PATHS)
61 65
62 66
67def sort_attributes(manifest):
68 """Sort the attributes of all elements alphabetically.
69
70 This is needed because different versions of the toxml() function from
71 xml.dom.minidom outputs the attributes of elements in different orders.
72 Before Python 3.8 they were output alphabetically, later versions preserve
73 the order specified by the user.
74
75 Args:
76 manifest: String containing an XML manifest.
77
78 Returns:
79 The XML manifest with the attributes of all elements sorted alphabetically.
80 """
81 new_manifest = ''
82 # This will find every element in the XML manifest, whether they have
83 # attributes or not. This simplifies recreating the manifest below.
84 matches = re.findall(r'(<[/?]?[a-z-]+\s*)((?:\S+?="[^"]+"\s*?)*)(\s*[/?]?>)', manifest)
85 for head, attrs, tail in matches:
86 m = re.findall(r'\S+?="[^"]+"', attrs)
87 new_manifest += head + ' '.join(sorted(m)) + tail
88 return new_manifest
89
90
63class ManifestParseTestCase(unittest.TestCase): 91class ManifestParseTestCase(unittest.TestCase):
64 """TestCase for parsing manifests.""" 92 """TestCase for parsing manifests."""
65 93
@@ -91,6 +119,11 @@ class ManifestParseTestCase(unittest.TestCase):
91 fp.write(data) 119 fp.write(data)
92 return manifest_xml.XmlManifest(self.repodir, self.manifest_file) 120 return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
93 121
122 @staticmethod
123 def encodeXmlAttr(attr):
124 """Encode |attr| using XML escape rules."""
125 return attr.replace('\r', '&#x000d;').replace('\n', '&#x000a;')
126
94 127
95class ManifestValidateFilePaths(unittest.TestCase): 128class ManifestValidateFilePaths(unittest.TestCase):
96 """Check _ValidateFilePaths helper. 129 """Check _ValidateFilePaths helper.
@@ -232,6 +265,19 @@ class XmlManifestTests(ManifestParseTestCase):
232 self.assertEqual(manifest.repo_hooks_project.name, 'repohooks') 265 self.assertEqual(manifest.repo_hooks_project.name, 'repohooks')
233 self.assertEqual(manifest.repo_hooks_project.enabled_repo_hooks, ['a', 'b']) 266 self.assertEqual(manifest.repo_hooks_project.enabled_repo_hooks, ['a', 'b'])
234 267
268 def test_repo_hooks_unordered(self):
269 """Check repo-hooks settings work even if the project def comes second."""
270 manifest = self.getXmlManifest("""
271<manifest>
272 <remote name="test-remote" fetch="http://localhost" />
273 <default remote="test-remote" revision="refs/heads/main" />
274 <repo-hooks in-project="repohooks" enabled-list="a, b"/>
275 <project name="repohooks" path="src/repohooks"/>
276</manifest>
277""")
278 self.assertEqual(manifest.repo_hooks_project.name, 'repohooks')
279 self.assertEqual(manifest.repo_hooks_project.enabled_repo_hooks, ['a', 'b'])
280
235 def test_unknown_tags(self): 281 def test_unknown_tags(self):
236 """Check superproject settings.""" 282 """Check superproject settings."""
237 manifest = self.getXmlManifest(""" 283 manifest = self.getXmlManifest("""
@@ -246,11 +292,30 @@ class XmlManifestTests(ManifestParseTestCase):
246 self.assertEqual(manifest.superproject['name'], 'superproject') 292 self.assertEqual(manifest.superproject['name'], 'superproject')
247 self.assertEqual(manifest.superproject['remote'].name, 'test-remote') 293 self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
248 self.assertEqual( 294 self.assertEqual(
249 manifest.ToXml().toxml(), 295 sort_attributes(manifest.ToXml().toxml()),
250 '<?xml version="1.0" ?><manifest>' + 296 '<?xml version="1.0" ?><manifest>'
251 '<remote name="test-remote" fetch="http://localhost"/>' + 297 '<remote fetch="http://localhost" name="test-remote"/>'
252 '<default remote="test-remote" revision="refs/heads/main"/>' + 298 '<default remote="test-remote" revision="refs/heads/main"/>'
253 '<superproject name="superproject"/>' + 299 '<superproject name="superproject"/>'
300 '</manifest>')
301
302 def test_remote_annotations(self):
303 """Check remote settings."""
304 manifest = self.getXmlManifest("""
305<manifest>
306 <remote name="test-remote" fetch="http://localhost">
307 <annotation name="foo" value="bar"/>
308 </remote>
309</manifest>
310""")
311 self.assertEqual(manifest.remotes['test-remote'].annotations[0].name, 'foo')
312 self.assertEqual(manifest.remotes['test-remote'].annotations[0].value, 'bar')
313 self.assertEqual(
314 sort_attributes(manifest.ToXml().toxml()),
315 '<?xml version="1.0" ?><manifest>'
316 '<remote fetch="http://localhost" name="test-remote">'
317 '<annotation name="foo" value="bar"/>'
318 '</remote>'
254 '</manifest>') 319 '</manifest>')
255 320
256 321
@@ -303,6 +368,7 @@ class IncludeElementTests(ManifestParseTestCase):
303 def test_allow_bad_name_from_user(self): 368 def test_allow_bad_name_from_user(self):
304 """Check handling of bad name attribute from the user's input.""" 369 """Check handling of bad name attribute from the user's input."""
305 def parse(name): 370 def parse(name):
371 name = self.encodeXmlAttr(name)
306 manifest = self.getXmlManifest(f""" 372 manifest = self.getXmlManifest(f"""
307<manifest> 373<manifest>
308 <remote name="default-remote" fetch="http://localhost" /> 374 <remote name="default-remote" fetch="http://localhost" />
@@ -327,6 +393,7 @@ class IncludeElementTests(ManifestParseTestCase):
327 def test_bad_name_checks(self): 393 def test_bad_name_checks(self):
328 """Check handling of bad name attribute.""" 394 """Check handling of bad name attribute."""
329 def parse(name): 395 def parse(name):
396 name = self.encodeXmlAttr(name)
330 # Setup target of the include. 397 # Setup target of the include.
331 with open(os.path.join(self.manifest_dir, 'target.xml'), 'w') as fp: 398 with open(os.path.join(self.manifest_dir, 'target.xml'), 'w') as fp:
332 fp.write(f'<manifest><include name="{name}"/></manifest>') 399 fp.write(f'<manifest><include name="{name}"/></manifest>')
@@ -398,16 +465,18 @@ class ProjectElementTests(ManifestParseTestCase):
398 project = manifest.projects[0] 465 project = manifest.projects[0]
399 project.SetRevisionId('ABCDEF') 466 project.SetRevisionId('ABCDEF')
400 self.assertEqual( 467 self.assertEqual(
401 manifest.ToXml().toxml(), 468 sort_attributes(manifest.ToXml().toxml()),
402 '<?xml version="1.0" ?><manifest>' + 469 '<?xml version="1.0" ?><manifest>'
403 '<remote name="default-remote" fetch="http://localhost"/>' + 470 '<remote fetch="http://localhost" name="default-remote"/>'
404 '<default remote="default-remote" revision="refs/heads/main"/>' + 471 '<default remote="default-remote" revision="refs/heads/main"/>'
405 '<project name="test-name" revision="ABCDEF"/>' + 472 '<project name="test-name" revision="ABCDEF" upstream="refs/heads/main"/>'
406 '</manifest>') 473 '</manifest>')
407 474
408 def test_trailing_slash(self): 475 def test_trailing_slash(self):
409 """Check handling of trailing slashes in attributes.""" 476 """Check handling of trailing slashes in attributes."""
410 def parse(name, path): 477 def parse(name, path):
478 name = self.encodeXmlAttr(name)
479 path = self.encodeXmlAttr(path)
411 return self.getXmlManifest(f""" 480 return self.getXmlManifest(f"""
412<manifest> 481<manifest>
413 <remote name="default-remote" fetch="http://localhost" /> 482 <remote name="default-remote" fetch="http://localhost" />
@@ -437,6 +506,8 @@ class ProjectElementTests(ManifestParseTestCase):
437 def test_toplevel_path(self): 506 def test_toplevel_path(self):
438 """Check handling of path=. specially.""" 507 """Check handling of path=. specially."""
439 def parse(name, path): 508 def parse(name, path):
509 name = self.encodeXmlAttr(name)
510 path = self.encodeXmlAttr(path)
440 return self.getXmlManifest(f""" 511 return self.getXmlManifest(f"""
441<manifest> 512<manifest>
442 <remote name="default-remote" fetch="http://localhost" /> 513 <remote name="default-remote" fetch="http://localhost" />
@@ -453,6 +524,8 @@ class ProjectElementTests(ManifestParseTestCase):
453 def test_bad_path_name_checks(self): 524 def test_bad_path_name_checks(self):
454 """Check handling of bad path & name attributes.""" 525 """Check handling of bad path & name attributes."""
455 def parse(name, path): 526 def parse(name, path):
527 name = self.encodeXmlAttr(name)
528 path = self.encodeXmlAttr(path)
456 manifest = self.getXmlManifest(f""" 529 manifest = self.getXmlManifest(f"""
457<manifest> 530<manifest>
458 <remote name="default-remote" fetch="http://localhost" /> 531 <remote name="default-remote" fetch="http://localhost" />
@@ -499,12 +572,79 @@ class SuperProjectElementTests(ManifestParseTestCase):
499 self.assertEqual(manifest.superproject['name'], 'superproject') 572 self.assertEqual(manifest.superproject['name'], 'superproject')
500 self.assertEqual(manifest.superproject['remote'].name, 'test-remote') 573 self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
501 self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject') 574 self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
575 self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
502 self.assertEqual( 576 self.assertEqual(
503 manifest.ToXml().toxml(), 577 sort_attributes(manifest.ToXml().toxml()),
504 '<?xml version="1.0" ?><manifest>' + 578 '<?xml version="1.0" ?><manifest>'
505 '<remote name="test-remote" fetch="http://localhost"/>' + 579 '<remote fetch="http://localhost" name="test-remote"/>'
506 '<default remote="test-remote" revision="refs/heads/main"/>' + 580 '<default remote="test-remote" revision="refs/heads/main"/>'
507 '<superproject name="superproject"/>' + 581 '<superproject name="superproject"/>'
582 '</manifest>')
583
584 def test_superproject_revision(self):
585 """Check superproject settings with a different revision attribute"""
586 self.maxDiff = None
587 manifest = self.getXmlManifest("""
588<manifest>
589 <remote name="test-remote" fetch="http://localhost" />
590 <default remote="test-remote" revision="refs/heads/main" />
591 <superproject name="superproject" revision="refs/heads/stable" />
592</manifest>
593""")
594 self.assertEqual(manifest.superproject['name'], 'superproject')
595 self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
596 self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
597 self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
598 self.assertEqual(
599 sort_attributes(manifest.ToXml().toxml()),
600 '<?xml version="1.0" ?><manifest>'
601 '<remote fetch="http://localhost" name="test-remote"/>'
602 '<default remote="test-remote" revision="refs/heads/main"/>'
603 '<superproject name="superproject" revision="refs/heads/stable"/>'
604 '</manifest>')
605
606 def test_superproject_revision_default_negative(self):
607 """Check superproject settings with a same revision attribute"""
608 self.maxDiff = None
609 manifest = self.getXmlManifest("""
610<manifest>
611 <remote name="test-remote" fetch="http://localhost" />
612 <default remote="test-remote" revision="refs/heads/stable" />
613 <superproject name="superproject" revision="refs/heads/stable" />
614</manifest>
615""")
616 self.assertEqual(manifest.superproject['name'], 'superproject')
617 self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
618 self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
619 self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
620 self.assertEqual(
621 sort_attributes(manifest.ToXml().toxml()),
622 '<?xml version="1.0" ?><manifest>'
623 '<remote fetch="http://localhost" name="test-remote"/>'
624 '<default remote="test-remote" revision="refs/heads/stable"/>'
625 '<superproject name="superproject"/>'
626 '</manifest>')
627
628 def test_superproject_revision_remote(self):
629 """Check superproject settings with a same revision attribute"""
630 self.maxDiff = None
631 manifest = self.getXmlManifest("""
632<manifest>
633 <remote name="test-remote" fetch="http://localhost" revision="refs/heads/main" />
634 <default remote="test-remote" />
635 <superproject name="superproject" revision="refs/heads/stable" />
636</manifest>
637""")
638 self.assertEqual(manifest.superproject['name'], 'superproject')
639 self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
640 self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
641 self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
642 self.assertEqual(
643 sort_attributes(manifest.ToXml().toxml()),
644 '<?xml version="1.0" ?><manifest>'
645 '<remote fetch="http://localhost" name="test-remote" revision="refs/heads/main"/>'
646 '<default remote="test-remote"/>'
647 '<superproject name="superproject" revision="refs/heads/stable"/>'
508 '</manifest>') 648 '</manifest>')
509 649
510 def test_remote(self): 650 def test_remote(self):
@@ -520,13 +660,14 @@ class SuperProjectElementTests(ManifestParseTestCase):
520 self.assertEqual(manifest.superproject['name'], 'platform/superproject') 660 self.assertEqual(manifest.superproject['name'], 'platform/superproject')
521 self.assertEqual(manifest.superproject['remote'].name, 'superproject-remote') 661 self.assertEqual(manifest.superproject['remote'].name, 'superproject-remote')
522 self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/platform/superproject') 662 self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/platform/superproject')
663 self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
523 self.assertEqual( 664 self.assertEqual(
524 manifest.ToXml().toxml(), 665 sort_attributes(manifest.ToXml().toxml()),
525 '<?xml version="1.0" ?><manifest>' + 666 '<?xml version="1.0" ?><manifest>'
526 '<remote name="default-remote" fetch="http://localhost"/>' + 667 '<remote fetch="http://localhost" name="default-remote"/>'
527 '<remote name="superproject-remote" fetch="http://localhost"/>' + 668 '<remote fetch="http://localhost" name="superproject-remote"/>'
528 '<default remote="default-remote" revision="refs/heads/main"/>' + 669 '<default remote="default-remote" revision="refs/heads/main"/>'
529 '<superproject name="platform/superproject" remote="superproject-remote"/>' + 670 '<superproject name="platform/superproject" remote="superproject-remote"/>'
530 '</manifest>') 671 '</manifest>')
531 672
532 def test_defalut_remote(self): 673 def test_defalut_remote(self):
@@ -540,10 +681,165 @@ class SuperProjectElementTests(ManifestParseTestCase):
540""") 681""")
541 self.assertEqual(manifest.superproject['name'], 'superproject') 682 self.assertEqual(manifest.superproject['name'], 'superproject')
542 self.assertEqual(manifest.superproject['remote'].name, 'default-remote') 683 self.assertEqual(manifest.superproject['remote'].name, 'default-remote')
684 self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
685 self.assertEqual(
686 sort_attributes(manifest.ToXml().toxml()),
687 '<?xml version="1.0" ?><manifest>'
688 '<remote fetch="http://localhost" name="default-remote"/>'
689 '<default remote="default-remote" revision="refs/heads/main"/>'
690 '<superproject name="superproject"/>'
691 '</manifest>')
692
693
694class ContactinfoElementTests(ManifestParseTestCase):
695 """Tests for <contactinfo>."""
696
697 def test_contactinfo(self):
698 """Check contactinfo settings."""
699 bugurl = 'http://localhost/contactinfo'
700 manifest = self.getXmlManifest(f"""
701<manifest>
702 <contactinfo bugurl="{bugurl}"/>
703</manifest>
704""")
705 self.assertEqual(manifest.contactinfo.bugurl, bugurl)
543 self.assertEqual( 706 self.assertEqual(
544 manifest.ToXml().toxml(), 707 manifest.ToXml().toxml(),
545 '<?xml version="1.0" ?><manifest>' + 708 '<?xml version="1.0" ?><manifest>'
546 '<remote name="default-remote" fetch="http://localhost"/>' + 709 f'<contactinfo bugurl="{bugurl}"/>'
547 '<default remote="default-remote" revision="refs/heads/main"/>' +
548 '<superproject name="superproject"/>' +
549 '</manifest>') 710 '</manifest>')
711
712
713class DefaultElementTests(ManifestParseTestCase):
714 """Tests for <default>."""
715
716 def test_default(self):
717 """Check default settings."""
718 a = manifest_xml._Default()
719 a.revisionExpr = 'foo'
720 a.remote = manifest_xml._XmlRemote(name='remote')
721 b = manifest_xml._Default()
722 b.revisionExpr = 'bar'
723 self.assertEqual(a, a)
724 self.assertNotEqual(a, b)
725 self.assertNotEqual(b, a.remote)
726 self.assertNotEqual(a, 123)
727 self.assertNotEqual(a, None)
728
729
730class RemoteElementTests(ManifestParseTestCase):
731 """Tests for <remote>."""
732
733 def test_remote(self):
734 """Check remote settings."""
735 a = manifest_xml._XmlRemote(name='foo')
736 a.AddAnnotation('key1', 'value1', 'true')
737 b = manifest_xml._XmlRemote(name='foo')
738 b.AddAnnotation('key2', 'value1', 'true')
739 c = manifest_xml._XmlRemote(name='foo')
740 c.AddAnnotation('key1', 'value2', 'true')
741 d = manifest_xml._XmlRemote(name='foo')
742 d.AddAnnotation('key1', 'value1', 'false')
743 self.assertEqual(a, a)
744 self.assertNotEqual(a, b)
745 self.assertNotEqual(a, c)
746 self.assertNotEqual(a, d)
747 self.assertNotEqual(a, manifest_xml._Default())
748 self.assertNotEqual(a, 123)
749 self.assertNotEqual(a, None)
750
751
752class RemoveProjectElementTests(ManifestParseTestCase):
753 """Tests for <remove-project>."""
754
755 def test_remove_one_project(self):
756 manifest = self.getXmlManifest("""
757<manifest>
758 <remote name="default-remote" fetch="http://localhost" />
759 <default remote="default-remote" revision="refs/heads/main" />
760 <project name="myproject" />
761 <remove-project name="myproject" />
762</manifest>
763""")
764 self.assertEqual(manifest.projects, [])
765
766 def test_remove_one_project_one_remains(self):
767 manifest = self.getXmlManifest("""
768<manifest>
769 <remote name="default-remote" fetch="http://localhost" />
770 <default remote="default-remote" revision="refs/heads/main" />
771 <project name="myproject" />
772 <project name="yourproject" />
773 <remove-project name="myproject" />
774</manifest>
775""")
776
777 self.assertEqual(len(manifest.projects), 1)
778 self.assertEqual(manifest.projects[0].name, 'yourproject')
779
780 def test_remove_one_project_doesnt_exist(self):
781 with self.assertRaises(manifest_xml.ManifestParseError):
782 manifest = self.getXmlManifest("""
783<manifest>
784 <remote name="default-remote" fetch="http://localhost" />
785 <default remote="default-remote" revision="refs/heads/main" />
786 <remove-project name="myproject" />
787</manifest>
788""")
789 manifest.projects
790
791 def test_remove_one_optional_project_doesnt_exist(self):
792 manifest = self.getXmlManifest("""
793<manifest>
794 <remote name="default-remote" fetch="http://localhost" />
795 <default remote="default-remote" revision="refs/heads/main" />
796 <remove-project name="myproject" optional="true" />
797</manifest>
798""")
799 self.assertEqual(manifest.projects, [])
800
801
802class ExtendProjectElementTests(ManifestParseTestCase):
803 """Tests for <extend-project>."""
804
805 def test_extend_project_dest_path_single_match(self):
806 manifest = self.getXmlManifest("""
807<manifest>
808 <remote name="default-remote" fetch="http://localhost" />
809 <default remote="default-remote" revision="refs/heads/main" />
810 <project name="myproject" />
811 <extend-project name="myproject" dest-path="bar" />
812</manifest>
813""")
814 self.assertEqual(len(manifest.projects), 1)
815 self.assertEqual(manifest.projects[0].relpath, 'bar')
816
817 def test_extend_project_dest_path_multi_match(self):
818 with self.assertRaises(manifest_xml.ManifestParseError):
819 manifest = self.getXmlManifest("""
820<manifest>
821 <remote name="default-remote" fetch="http://localhost" />
822 <default remote="default-remote" revision="refs/heads/main" />
823 <project name="myproject" path="x" />
824 <project name="myproject" path="y" />
825 <extend-project name="myproject" dest-path="bar" />
826</manifest>
827""")
828 manifest.projects
829
830 def test_extend_project_dest_path_multi_match_path_specified(self):
831 manifest = self.getXmlManifest("""
832<manifest>
833 <remote name="default-remote" fetch="http://localhost" />
834 <default remote="default-remote" revision="refs/heads/main" />
835 <project name="myproject" path="x" />
836 <project name="myproject" path="y" />
837 <extend-project name="myproject" path="x" dest-path="bar" />
838</manifest>
839""")
840 self.assertEqual(len(manifest.projects), 2)
841 if manifest.projects[0].relpath == 'y':
842 self.assertEqual(manifest.projects[1].relpath, 'bar')
843 else:
844 self.assertEqual(manifest.projects[0].relpath, 'bar')
845 self.assertEqual(manifest.projects[1].relpath, 'y')
diff --git a/tests/test_platform_utils.py b/tests/test_platform_utils.py
new file mode 100644
index 00000000..55b7805c
--- /dev/null
+++ b/tests/test_platform_utils.py
@@ -0,0 +1,50 @@
1# Copyright 2021 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""Unittests for the platform_utils.py module."""
16
17import os
18import tempfile
19import unittest
20
21import platform_utils
22
23
24class RemoveTests(unittest.TestCase):
25 """Check remove() helper."""
26
27 def testMissingOk(self):
28 """Check missing_ok handling."""
29 with tempfile.TemporaryDirectory() as tmpdir:
30 path = os.path.join(tmpdir, 'test')
31
32 # Should not fail.
33 platform_utils.remove(path, missing_ok=True)
34
35 # Should fail.
36 self.assertRaises(OSError, platform_utils.remove, path)
37 self.assertRaises(OSError, platform_utils.remove, path, missing_ok=False)
38
39 # Should not fail if it exists.
40 open(path, 'w').close()
41 platform_utils.remove(path, missing_ok=True)
42 self.assertFalse(os.path.exists(path))
43
44 open(path, 'w').close()
45 platform_utils.remove(path)
46 self.assertFalse(os.path.exists(path))
47
48 open(path, 'w').close()
49 platform_utils.remove(path, missing_ok=False)
50 self.assertFalse(os.path.exists(path))
diff --git a/tests/test_ssh.py b/tests/test_ssh.py
new file mode 100644
index 00000000..ffb5cb94
--- /dev/null
+++ b/tests/test_ssh.py
@@ -0,0 +1,74 @@
1# Copyright 2019 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""Unittests for the ssh.py module."""
16
17import multiprocessing
18import subprocess
19import unittest
20from unittest import mock
21
22import ssh
23
24
25class SshTests(unittest.TestCase):
26 """Tests the ssh functions."""
27
28 def test_parse_ssh_version(self):
29 """Check _parse_ssh_version() handling."""
30 ver = ssh._parse_ssh_version('Unknown\n')
31 self.assertEqual(ver, ())
32 ver = ssh._parse_ssh_version('OpenSSH_1.0\n')
33 self.assertEqual(ver, (1, 0))
34 ver = ssh._parse_ssh_version('OpenSSH_6.6.1p1 Ubuntu-2ubuntu2.13, OpenSSL 1.0.1f 6 Jan 2014\n')
35 self.assertEqual(ver, (6, 6, 1))
36 ver = ssh._parse_ssh_version('OpenSSH_7.6p1 Ubuntu-4ubuntu0.3, OpenSSL 1.0.2n 7 Dec 2017\n')
37 self.assertEqual(ver, (7, 6))
38
39 def test_version(self):
40 """Check version() handling."""
41 with mock.patch('ssh._run_ssh_version', return_value='OpenSSH_1.2\n'):
42 self.assertEqual(ssh.version(), (1, 2))
43
44 def test_context_manager_empty(self):
45 """Verify context manager with no clients works correctly."""
46 with multiprocessing.Manager() as manager:
47 with ssh.ProxyManager(manager):
48 pass
49
50 def test_context_manager_child_cleanup(self):
51 """Verify orphaned clients & masters get cleaned up."""
52 with multiprocessing.Manager() as manager:
53 with ssh.ProxyManager(manager) as ssh_proxy:
54 client = subprocess.Popen(['sleep', '964853320'])
55 ssh_proxy.add_client(client)
56 master = subprocess.Popen(['sleep', '964853321'])
57 ssh_proxy.add_master(master)
58 # If the process still exists, these will throw timeout errors.
59 client.wait(0)
60 master.wait(0)
61
62 def test_ssh_sock(self):
63 """Check sock() function."""
64 manager = multiprocessing.Manager()
65 proxy = ssh.ProxyManager(manager)
66 with mock.patch('tempfile.mkdtemp', return_value='/tmp/foo'):
67 # old ssh version uses port
68 with mock.patch('ssh.version', return_value=(6, 6)):
69 self.assertTrue(proxy.sock().endswith('%p'))
70
71 proxy._sock_path = None
72 # new ssh version uses hash
73 with mock.patch('ssh.version', return_value=(6, 7)):
74 self.assertTrue(proxy.sock().endswith('%C'))
diff --git a/tests/test_subcmds.py b/tests/test_subcmds.py
index 2234e646..bc53051a 100644
--- a/tests/test_subcmds.py
+++ b/tests/test_subcmds.py
@@ -14,6 +14,7 @@
14 14
15"""Unittests for the subcmds module (mostly __init__.py than subcommands).""" 15"""Unittests for the subcmds module (mostly __init__.py than subcommands)."""
16 16
17import optparse
17import unittest 18import unittest
18 19
19import subcmds 20import subcmds
@@ -41,3 +42,32 @@ class AllCommands(unittest.TestCase):
41 42
42 # Reject internal python paths like "__init__". 43 # Reject internal python paths like "__init__".
43 self.assertFalse(cmd.startswith('__')) 44 self.assertFalse(cmd.startswith('__'))
45
46 def test_help_desc_style(self):
47 """Force some consistency in option descriptions.
48
49 Python's optparse & argparse has a few default options like --help. Their
50 option description text uses lowercase sentence fragments, so enforce our
51 options follow the same style so UI is consistent.
52
53 We enforce:
54 * Text starts with lowercase.
55 * Text doesn't end with period.
56 """
57 for name, cls in subcmds.all_commands.items():
58 cmd = cls()
59 parser = cmd.OptionParser
60 for option in parser.option_list:
61 if option.help == optparse.SUPPRESS_HELP:
62 continue
63
64 c = option.help[0]
65 self.assertEqual(
66 c.lower(), c,
67 msg=f'subcmds/{name}.py: {option.get_opt_string()}: help text '
68 f'should start with lowercase: "{option.help}"')
69
70 self.assertNotEqual(
71 option.help[-1], '.',
72 msg=f'subcmds/{name}.py: {option.get_opt_string()}: help text '
73 f'should not end in a period: "{option.help}"')
diff --git a/tox.ini b/tox.ini
index 3282de14..aa4e2979 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,11 +15,10 @@
15# https://tox.readthedocs.io/ 15# https://tox.readthedocs.io/
16 16
17[tox] 17[tox]
18envlist = py35, py36, py37, py38, py39 18envlist = py36, py37, py38, py39
19 19
20[gh-actions] 20[gh-actions]
21python = 21python =
22 3.5: py35
23 3.6: py36 22 3.6: py36
24 3.7: py37 23 3.7: py37
25 3.8: py38 24 3.8: py38