Lann Martin | 4fbdf20 | 2018-08-30 12:02:52 -0600 | [diff] [blame] | 1 | # -*- coding: utf-8 -*- |
| 2 | # Copyright 2018 The Chromium OS Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | """Create a manifest snapshot of a repo checkout. |
| 7 | |
| 8 | This starts with the output of `repo manifest -r` and updates the manifest to |
| 9 | account for local changes that may not already be available remotely; for any |
| 10 | commits that aren't already reachable from the upstream tracking branch, push |
| 11 | refs to the remotes so that this snapshot can be reproduced remotely. |
| 12 | """ |
| 13 | |
| 14 | from __future__ import print_function |
| 15 | |
| 16 | import os |
| 17 | import sys |
| 18 | |
| 19 | from chromite.lib import commandline |
| 20 | from chromite.lib import cros_build_lib |
| 21 | from chromite.lib import cros_logging as logging |
| 22 | from chromite.lib import git |
| 23 | from chromite.lib import parallel |
| 24 | from chromite.lib import repo_util |
| 25 | |
| 26 | |
Mike Frysinger | 2688ef6 | 2020-02-16 00:00:46 -0500 | [diff] [blame] | 27 | assert sys.version_info >= (3, 6), 'This module requires Python 3.6+' |
| 28 | |
| 29 | |
Lann Martin | 4fbdf20 | 2018-08-30 12:02:52 -0600 | [diff] [blame] | 30 | BRANCH_REF_PREFIX = 'refs/heads/' |
| 31 | |
| 32 | |
| 33 | def GetParser(): |
| 34 | """Creates the argparse parser.""" |
| 35 | parser = commandline.ArgumentParser(description=__doc__) |
| 36 | parser.add_argument('--repo-path', type='path', default='.', |
| 37 | help='Path to the repo to snapshot.') |
| 38 | parser.add_argument('--snapshot-ref', |
| 39 | help='Remote ref to create for projects whose HEAD is ' |
| 40 | 'not reachable from its current upstream branch. ' |
| 41 | 'Projects with multiple checkouts may have a ' |
| 42 | 'unique suffix appended to this ref.') |
| 43 | parser.add_argument('--output-file', type='path', |
| 44 | help='Path to write the manifest snapshot XML to.') |
| 45 | parser.add_argument('--dry-run', action='store_true', |
| 46 | help='Do not actually push to remotes.') |
| 47 | # This is for limiting network traffic to the git remote(s). |
| 48 | parser.add_argument('--jobs', type=int, default=16, |
| 49 | help='The number of parallel processes to run for ' |
| 50 | 'git push operations.') |
| 51 | return parser |
| 52 | |
| 53 | |
| 54 | def _GetUpstreamBranch(project): |
| 55 | """Return a best guess at the project's upstream branch name.""" |
| 56 | branch = project.upstream |
| 57 | if branch and branch.startswith(BRANCH_REF_PREFIX): |
| 58 | branch = branch[len(BRANCH_REF_PREFIX):] |
| 59 | return branch |
| 60 | |
| 61 | |
| 62 | def _NeedsSnapshot(repo_root, project): |
| 63 | """Test if project's revision is reachable from its upstream ref.""" |
Mike Frysinger | 6e3f23c | 2021-02-17 14:06:19 -0500 | [diff] [blame^] | 64 | # Some projects don't have an upstream set. Try 'main' anyway. |
| 65 | branch = _GetUpstreamBranch(project) or 'main' |
Lann Martin | 4fbdf20 | 2018-08-30 12:02:52 -0600 | [diff] [blame] | 66 | upstream_ref = 'refs/remotes/%s/%s' % (project.Remote().GitName(), branch) |
| 67 | project_path = os.path.join(repo_root, project.Path()) |
| 68 | try: |
| 69 | if git.IsReachable(project_path, project.revision, upstream_ref): |
| 70 | return False |
| 71 | except cros_build_lib.RunCommandError as e: |
| 72 | logging.debug('Reachability check failed: %s', e) |
| 73 | logging.info('Project %s revision %s not reachable from upstream %r.', |
| 74 | project.name, project.revision, upstream_ref) |
| 75 | return True |
| 76 | |
| 77 | |
| 78 | def _MakeUniqueRef(project, base_ref, used_refs): |
| 79 | """Return a git ref for project that isn't in used_refs. |
| 80 | |
| 81 | Args: |
| 82 | project: The Project object to create a ref for. |
| 83 | base_ref: A base ref name; this may be appended to to generate a unique ref. |
| 84 | used_refs: A set of ref names to uniquify against. It is updated with the |
| 85 | newly generated ref. |
| 86 | """ |
| 87 | ref = base_ref |
| 88 | |
Mike Frysinger | 6e3f23c | 2021-02-17 14:06:19 -0500 | [diff] [blame^] | 89 | # If the project upstream is a non-main branch, append it to the ref. |
Lann Martin | 4fbdf20 | 2018-08-30 12:02:52 -0600 | [diff] [blame] | 90 | branch = _GetUpstreamBranch(project) |
Mike Frysinger | 6e3f23c | 2021-02-17 14:06:19 -0500 | [diff] [blame^] | 91 | if branch and branch != 'main': |
Lann Martin | 4fbdf20 | 2018-08-30 12:02:52 -0600 | [diff] [blame] | 92 | ref = '%s/%s' % (ref, branch) |
| 93 | |
| 94 | if ref in used_refs: |
| 95 | # Append incrementing numbers until we find an unused ref. |
| 96 | for i in range(1, len(used_refs) + 2): |
| 97 | numbered = '%s/%d' % (ref, i) |
| 98 | if numbered not in used_refs: |
| 99 | ref = numbered |
| 100 | break |
| 101 | else: |
| 102 | raise AssertionError('failed to make unique ref (ref=%s used_refs=%r)' % |
| 103 | (ref, used_refs)) |
| 104 | |
| 105 | used_refs.add(ref) |
| 106 | return ref |
| 107 | |
| 108 | |
| 109 | def _GitPushProjectUpstream(repo_root, project, dry_run): |
| 110 | """Push the project revision to its remote upstream.""" |
| 111 | git.GitPush( |
| 112 | os.path.join(repo_root, project.Path()), |
| 113 | project.revision, |
| 114 | git.RemoteRef(project.Remote().GitName(), project.upstream), |
| 115 | dry_run=dry_run) |
| 116 | |
| 117 | |
| 118 | def main(argv): |
| 119 | parser = GetParser() |
| 120 | options = parser.parse_args(argv) |
| 121 | options.Freeze() |
| 122 | |
| 123 | snapshot_ref = options.snapshot_ref |
| 124 | if snapshot_ref and not snapshot_ref.startswith('refs/'): |
| 125 | snapshot_ref = BRANCH_REF_PREFIX + snapshot_ref |
| 126 | |
| 127 | repo = repo_util.Repository.Find(options.repo_path) |
| 128 | if repo is None: |
| 129 | cros_build_lib.Die('No repo found in --repo_path %r.', options.repo_path) |
| 130 | |
| 131 | manifest = repo.Manifest(revision_locked=True) |
| 132 | projects = list(manifest.Projects()) |
| 133 | |
| 134 | # Check if projects need snapshots (in parallel). |
| 135 | needs_snapshot_results = parallel.RunTasksInProcessPool( |
| 136 | _NeedsSnapshot, [(repo.root, x) for x in projects]) |
| 137 | |
| 138 | # Group snapshot-needing projects by project name. |
| 139 | snapshot_projects = {} |
| 140 | for project, needs_snapshot in zip(projects, needs_snapshot_results): |
| 141 | if needs_snapshot: |
| 142 | snapshot_projects.setdefault(project.name, []).append(project) |
| 143 | |
| 144 | if snapshot_projects and not snapshot_ref: |
| 145 | cros_build_lib.Die('Some project(s) need snapshot refs but no ' |
| 146 | '--snapshot-ref specified.') |
| 147 | |
| 148 | # Push snapshot refs (in parallel). |
| 149 | with parallel.BackgroundTaskRunner(_GitPushProjectUpstream, |
| 150 | repo.root, dry_run=options.dry_run, |
| 151 | processes=options.jobs) as queue: |
| 152 | for projects in snapshot_projects.values(): |
| 153 | # Since some projects (e.g. chromiumos/third_party/kernel) are checked out |
| 154 | # multiple places, we may need to push each checkout to a unique ref. |
| 155 | need_unique_refs = len(projects) > 1 |
| 156 | used_refs = set() |
| 157 | for project in projects: |
| 158 | if need_unique_refs: |
| 159 | ref = _MakeUniqueRef(project, snapshot_ref, used_refs) |
| 160 | else: |
| 161 | ref = snapshot_ref |
| 162 | # Update the upstream ref both for the push and the output XML. |
| 163 | project.upstream = ref |
| 164 | queue.put([project]) |
| 165 | |
| 166 | dest = options.output_file |
| 167 | if dest is None or dest == '-': |
| 168 | dest = sys.stdout |
| 169 | |
| 170 | manifest.Write(dest) |