blob: 20d744ccd184f880ba1b7e3df39e8cb9b5763e46 [file] [log] [blame]
Mike Frysingerf1ba7ad2022-09-12 05:42:57 -04001# Copyright 2018 The ChromiumOS Authors
Lann Martin4fbdf202018-08-30 12:02:52 -06002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Create a manifest snapshot of a repo checkout.
6
7This starts with the output of `repo manifest -r` and updates the manifest to
8account for local changes that may not already be available remotely; for any
9commits that aren't already reachable from the upstream tracking branch, push
10refs to the remotes so that this snapshot can be reproduced remotely.
11"""
12
Chris McDonald59650c32021-07-20 15:29:28 -060013import logging
Lann Martin4fbdf202018-08-30 12:02:52 -060014import os
15import sys
16
17from chromite.lib import commandline
18from chromite.lib import cros_build_lib
Lann Martin4fbdf202018-08-30 12:02:52 -060019from chromite.lib import git
20from chromite.lib import parallel
21from chromite.lib import repo_util
22
23
Alex Klein1699fab2022-09-08 08:46:06 -060024BRANCH_REF_PREFIX = "refs/heads/"
Lann Martin4fbdf202018-08-30 12:02:52 -060025
26
27def GetParser():
Alex Klein1699fab2022-09-08 08:46:06 -060028 """Creates the argparse parser."""
Mike Frysinger66ac6e32023-02-02 09:03:31 -050029 parser = commandline.ArgumentParser(description=__doc__, dryrun=True)
Alex Klein1699fab2022-09-08 08:46:06 -060030 parser.add_argument(
31 "--repo-path",
32 type="path",
33 default=".",
34 help="Path to the repo to snapshot.",
35 )
36 parser.add_argument(
37 "--snapshot-ref",
38 help="Remote ref to create for projects whose HEAD is "
39 "not reachable from its current upstream branch. "
40 "Projects with multiple checkouts may have a "
41 "unique suffix appended to this ref.",
42 )
43 parser.add_argument(
44 "--output-file",
45 type="path",
46 help="Path to write the manifest snapshot XML to.",
47 )
Alex Klein1699fab2022-09-08 08:46:06 -060048 # This is for limiting network traffic to the git remote(s).
49 parser.add_argument(
50 "--jobs",
51 type=int,
52 default=16,
53 help="The number of parallel processes to run for "
54 "git push operations.",
55 )
56 return parser
Lann Martin4fbdf202018-08-30 12:02:52 -060057
58
59def _GetUpstreamBranch(project):
Alex Klein1699fab2022-09-08 08:46:06 -060060 """Return a best guess at the project's upstream branch name."""
61 branch = project.upstream
62 if branch and branch.startswith(BRANCH_REF_PREFIX):
63 branch = branch[len(BRANCH_REF_PREFIX) :]
64 return branch
Lann Martin4fbdf202018-08-30 12:02:52 -060065
66
67def _NeedsSnapshot(repo_root, project):
Alex Klein1699fab2022-09-08 08:46:06 -060068 """Test if project's revision is reachable from its upstream ref."""
69 # Some projects don't have an upstream set. Try 'main' anyway.
70 branch = _GetUpstreamBranch(project) or "main"
71 upstream_ref = "refs/remotes/%s/%s" % (project.Remote().GitName(), branch)
72 project_path = os.path.join(repo_root, project.Path())
73 try:
74 if git.IsReachable(project_path, project.revision, upstream_ref):
75 return False
76 except cros_build_lib.RunCommandError as e:
77 logging.debug("Reachability check failed: %s", e)
78 logging.info(
79 "Project %s revision %s not reachable from upstream %r.",
80 project.name,
81 project.revision,
82 upstream_ref,
83 )
84 return True
Lann Martin4fbdf202018-08-30 12:02:52 -060085
86
87def _MakeUniqueRef(project, base_ref, used_refs):
Alex Klein1699fab2022-09-08 08:46:06 -060088 """Return a git ref for project that isn't in used_refs.
Lann Martin4fbdf202018-08-30 12:02:52 -060089
Alex Klein1699fab2022-09-08 08:46:06 -060090 Args:
91 project: The Project object to create a ref for.
92 base_ref: A base ref name; this may be appended to to generate a unique ref.
93 used_refs: A set of ref names to uniquify against. It is updated with the
94 newly generated ref.
95 """
96 ref = base_ref
Lann Martin4fbdf202018-08-30 12:02:52 -060097
Alex Klein1699fab2022-09-08 08:46:06 -060098 # If the project upstream is a non-main branch, append it to the ref.
99 branch = _GetUpstreamBranch(project)
100 if branch and branch != "main":
101 ref = "%s/%s" % (ref, branch)
Lann Martin4fbdf202018-08-30 12:02:52 -0600102
Alex Klein1699fab2022-09-08 08:46:06 -0600103 if ref in used_refs:
104 # Append incrementing numbers until we find an unused ref.
105 for i in range(1, len(used_refs) + 2):
106 numbered = "%s/%d" % (ref, i)
107 if numbered not in used_refs:
108 ref = numbered
109 break
110 else:
111 raise AssertionError(
112 "failed to make unique ref (ref=%s used_refs=%r)"
113 % (ref, used_refs)
114 )
Lann Martin4fbdf202018-08-30 12:02:52 -0600115
Alex Klein1699fab2022-09-08 08:46:06 -0600116 used_refs.add(ref)
117 return ref
Lann Martin4fbdf202018-08-30 12:02:52 -0600118
119
Mike Frysinger66ac6e32023-02-02 09:03:31 -0500120def _GitPushProjectUpstream(repo_root, project, dryrun):
Alex Klein1699fab2022-09-08 08:46:06 -0600121 """Push the project revision to its remote upstream."""
122 git.GitPush(
123 os.path.join(repo_root, project.Path()),
124 project.revision,
125 git.RemoteRef(project.Remote().GitName(), project.upstream),
Mike Frysinger66ac6e32023-02-02 09:03:31 -0500126 dry_run=dryrun,
Alex Klein1699fab2022-09-08 08:46:06 -0600127 )
Lann Martin4fbdf202018-08-30 12:02:52 -0600128
129
130def main(argv):
Alex Klein1699fab2022-09-08 08:46:06 -0600131 parser = GetParser()
132 options = parser.parse_args(argv)
133 options.Freeze()
Lann Martin4fbdf202018-08-30 12:02:52 -0600134
Alex Klein1699fab2022-09-08 08:46:06 -0600135 snapshot_ref = options.snapshot_ref
136 if snapshot_ref and not snapshot_ref.startswith("refs/"):
137 snapshot_ref = BRANCH_REF_PREFIX + snapshot_ref
Lann Martin4fbdf202018-08-30 12:02:52 -0600138
Alex Klein1699fab2022-09-08 08:46:06 -0600139 repo = repo_util.Repository.Find(options.repo_path)
140 if repo is None:
141 cros_build_lib.Die(
142 "No repo found in --repo_path %r.", options.repo_path
143 )
Lann Martin4fbdf202018-08-30 12:02:52 -0600144
Alex Klein1699fab2022-09-08 08:46:06 -0600145 manifest = repo.Manifest(revision_locked=True)
146 projects = list(manifest.Projects())
Lann Martin4fbdf202018-08-30 12:02:52 -0600147
Alex Klein1699fab2022-09-08 08:46:06 -0600148 # Check if projects need snapshots (in parallel).
149 needs_snapshot_results = parallel.RunTasksInProcessPool(
150 _NeedsSnapshot, [(repo.root, x) for x in projects]
151 )
Lann Martin4fbdf202018-08-30 12:02:52 -0600152
Alex Klein1699fab2022-09-08 08:46:06 -0600153 # Group snapshot-needing projects by project name.
154 snapshot_projects = {}
155 for project, needs_snapshot in zip(projects, needs_snapshot_results):
156 if needs_snapshot:
157 snapshot_projects.setdefault(project.name, []).append(project)
Lann Martin4fbdf202018-08-30 12:02:52 -0600158
Alex Klein1699fab2022-09-08 08:46:06 -0600159 if snapshot_projects and not snapshot_ref:
160 cros_build_lib.Die(
161 "Some project(s) need snapshot refs but no "
162 "--snapshot-ref specified."
163 )
Lann Martin4fbdf202018-08-30 12:02:52 -0600164
Alex Klein1699fab2022-09-08 08:46:06 -0600165 # Push snapshot refs (in parallel).
166 with parallel.BackgroundTaskRunner(
167 _GitPushProjectUpstream,
168 repo.root,
Mike Frysinger66ac6e32023-02-02 09:03:31 -0500169 dryrun=options.dryrun,
Alex Klein1699fab2022-09-08 08:46:06 -0600170 processes=options.jobs,
171 ) as queue:
172 for projects in snapshot_projects.values():
173 # Since some projects (e.g. chromiumos/third_party/kernel) are checked out
174 # multiple places, we may need to push each checkout to a unique ref.
175 need_unique_refs = len(projects) > 1
176 used_refs = set()
177 for project in projects:
178 if need_unique_refs:
179 ref = _MakeUniqueRef(project, snapshot_ref, used_refs)
180 else:
181 ref = snapshot_ref
182 # Update the upstream ref both for the push and the output XML.
183 project.upstream = ref
184 queue.put([project])
Lann Martin4fbdf202018-08-30 12:02:52 -0600185
Alex Klein1699fab2022-09-08 08:46:06 -0600186 dest = options.output_file
187 if dest is None or dest == "-":
188 dest = sys.stdout
Lann Martin4fbdf202018-08-30 12:02:52 -0600189
Alex Klein1699fab2022-09-08 08:46:06 -0600190 manifest.Write(dest)