blob: 09cae6f501b2503723ef596fdf3be993f1051658 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Raman Tenneti993af5e2021-05-12 12:00:31 -070015import collections
Colin Cross23acdd32012-04-21 00:33:54 -070016import itertools
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import os
Raman Tenneti080877e2021-03-09 15:19:06 -080018import platform
Conley Owensdb728cd2011-09-26 16:34:01 -070019import re
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import sys
David Pursehouse59bbb582013-05-17 10:49:33 +090021import xml.dom.minidom
Mike Frysingeracf63b22019-06-13 02:24:21 -040022import urllib.parse
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070023
Simran Basib9a1b732015-08-20 12:19:28 -070024import gitc_utils
Miguel Gaio1f207762020-07-17 14:09:13 +020025from git_config import GitConfig, IsId
David Pursehousee00aa6b2012-09-11 14:33:51 +090026from git_refs import R_HEADS, HEAD
Renaud Paquayd5cec5e2016-11-01 11:24:03 -070027import platform_utils
David Pursehousee00aa6b2012-09-11 14:33:51 +090028from project import RemoteSpec, Project, MetaProject
Mike Frysinger04122b72019-07-31 23:32:58 -040029from error import (ManifestParseError, ManifestInvalidPathError,
30 ManifestInvalidRevisionError)
Raman Tenneti993af5e2021-05-12 12:00:31 -070031from wrapper import Wrapper
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070032
33MANIFEST_FILE_NAME = 'manifest.xml'
Shawn O. Pearce5cc66792008-10-23 16:19:27 -070034LOCAL_MANIFEST_NAME = 'local_manifest.xml'
David Pursehouse2d5a0df2012-11-13 02:50:36 +090035LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070036
Raman Tenneti78f4dd32021-06-07 13:27:37 -070037# Add all projects from local manifest into a group.
38LOCAL_MANIFEST_GROUP_PREFIX = 'local:'
39
Raman Tenneti993af5e2021-05-12 12:00:31 -070040# ContactInfo has the self-registered bug url, supplied by the manifest authors.
41ContactInfo = collections.namedtuple('ContactInfo', 'bugurl')
42
Anthony Kingcb07ba72015-03-28 23:26:04 +000043# urljoin gets confused if the scheme is not known.
Joe Kilner6e310792016-10-27 15:53:53 -070044urllib.parse.uses_relative.extend([
45 'ssh',
46 'git',
47 'persistent-https',
48 'sso',
49 'rpc'])
50urllib.parse.uses_netloc.extend([
51 'ssh',
52 'git',
53 'persistent-https',
54 'sso',
55 'rpc'])
Conley Owensdb728cd2011-09-26 16:34:01 -070056
David Pursehouse819827a2020-02-12 15:20:19 +090057
Mike Frysingerbb8ee7f2020-02-22 05:30:12 -050058def XmlBool(node, attr, default=None):
59 """Determine boolean value of |node|'s |attr|.
60
61 Invalid values will issue a non-fatal warning.
62
63 Args:
64 node: XML node whose attributes we access.
65 attr: The attribute to access.
66 default: If the attribute is not set (value is empty), then use this.
67
68 Returns:
69 True if the attribute is a valid string representing true.
70 False if the attribute is a valid string representing false.
71 |default| otherwise.
72 """
73 value = node.getAttribute(attr)
74 s = value.lower()
75 if s == '':
76 return default
77 elif s in {'yes', 'true', '1'}:
78 return True
79 elif s in {'no', 'false', '0'}:
80 return False
81 else:
82 print('warning: manifest: %s="%s": ignoring invalid XML boolean' %
83 (attr, value), file=sys.stderr)
84 return default
85
86
87def XmlInt(node, attr, default=None):
88 """Determine integer value of |node|'s |attr|.
89
90 Args:
91 node: XML node whose attributes we access.
92 attr: The attribute to access.
93 default: If the attribute is not set (value is empty), then use this.
94
95 Returns:
96 The number if the attribute is a valid number.
97
98 Raises:
99 ManifestParseError: The number is invalid.
100 """
101 value = node.getAttribute(attr)
102 if not value:
103 return default
104
105 try:
106 return int(value)
107 except ValueError:
108 raise ManifestParseError('manifest: invalid %s="%s" integer' %
109 (attr, value))
110
111
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700112class _Default(object):
113 """Project defaults within the manifest."""
114
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700115 revisionExpr = None
Conley Owensb6a16e62013-09-25 15:06:09 -0700116 destBranchExpr = None
Nasser Grainawida403412018-05-04 12:53:29 -0600117 upstreamExpr = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700118 remote = None
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700119 sync_j = 1
Anatol Pomazau79770d22012-04-20 14:41:59 -0700120 sync_c = False
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800121 sync_s = False
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900122 sync_tags = True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700123
Julien Campergue74879922013-10-09 14:38:46 +0200124 def __eq__(self, other):
Jack Neus5ba21202021-06-09 15:21:25 +0000125 if not isinstance(other, _Default):
126 return False
Julien Campergue74879922013-10-09 14:38:46 +0200127 return self.__dict__ == other.__dict__
128
129 def __ne__(self, other):
Jack Neus5ba21202021-06-09 15:21:25 +0000130 if not isinstance(other, _Default):
131 return True
Julien Campergue74879922013-10-09 14:38:46 +0200132 return self.__dict__ != other.__dict__
133
David Pursehouse819827a2020-02-12 15:20:19 +0900134
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700135class _XmlRemote(object):
136 def __init__(self,
137 name,
Yestin Sunb292b982012-07-02 07:32:50 -0700138 alias=None,
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700139 fetch=None,
Steve Raed6480452016-08-10 15:00:00 -0700140 pushUrl=None,
Conley Owensdb728cd2011-09-26 16:34:01 -0700141 manifestUrl=None,
Anthony King36ea2fb2014-05-06 11:54:01 +0100142 review=None,
Jonathan Nieder93719792015-03-17 11:29:58 -0700143 revision=None):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700144 self.name = name
145 self.fetchUrl = fetch
Steve Raed6480452016-08-10 15:00:00 -0700146 self.pushUrl = pushUrl
Conley Owensdb728cd2011-09-26 16:34:01 -0700147 self.manifestUrl = manifestUrl
Yestin Sunb292b982012-07-02 07:32:50 -0700148 self.remoteAlias = alias
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700149 self.reviewUrl = review
Anthony King36ea2fb2014-05-06 11:54:01 +0100150 self.revision = revision
Conley Owensceea3682011-10-20 10:45:47 -0700151 self.resolvedFetchUrl = self._resolveFetchUrl()
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700152
David Pursehouse717ece92012-11-13 08:49:16 +0900153 def __eq__(self, other):
Jack Neus5ba21202021-06-09 15:21:25 +0000154 if not isinstance(other, _XmlRemote):
155 return False
David Pursehouse717ece92012-11-13 08:49:16 +0900156 return self.__dict__ == other.__dict__
157
158 def __ne__(self, other):
Jack Neus5ba21202021-06-09 15:21:25 +0000159 if not isinstance(other, _XmlRemote):
160 return True
David Pursehouse717ece92012-11-13 08:49:16 +0900161 return self.__dict__ != other.__dict__
162
Conley Owensceea3682011-10-20 10:45:47 -0700163 def _resolveFetchUrl(self):
Jack Neus5ba21202021-06-09 15:21:25 +0000164 if self.fetchUrl is None:
165 return ''
Conley Owensceea3682011-10-20 10:45:47 -0700166 url = self.fetchUrl.rstrip('/')
Conley Owensdb728cd2011-09-26 16:34:01 -0700167 manifestUrl = self.manifestUrl.rstrip('/')
Conley Owens2d0f5082014-01-31 15:03:51 -0800168 # urljoin will gets confused over quite a few things. The ones we care
169 # about here are:
170 # * no scheme in the base url, like <hostname:port>
Anthony Kingcb07ba72015-03-28 23:26:04 +0000171 # We handle no scheme by replacing it with an obscure protocol, gopher
172 # and then replacing it with the original when we are done.
173
Conley Owensdb728cd2011-09-26 16:34:01 -0700174 if manifestUrl.find(':') != manifestUrl.find('/') - 1:
Conley Owens4ccad752015-04-29 10:45:37 -0700175 url = urllib.parse.urljoin('gopher://' + manifestUrl, url)
176 url = re.sub(r'^gopher://', '', url)
Anthony Kingcb07ba72015-03-28 23:26:04 +0000177 else:
178 url = urllib.parse.urljoin(manifestUrl, url)
Shawn Pearcea9f11b32013-01-02 15:40:48 -0800179 return url
Conley Owensceea3682011-10-20 10:45:47 -0700180
181 def ToRemoteSpec(self, projectName):
David Rileye0684ad2017-04-05 00:02:59 -0700182 fetchUrl = self.resolvedFetchUrl.rstrip('/')
183 url = fetchUrl + '/' + projectName
Yestin Sunb292b982012-07-02 07:32:50 -0700184 remoteName = self.name
Conley Owens1e7ab2a2013-10-08 17:26:57 -0700185 if self.remoteAlias:
David Pursehouse37128b62013-10-15 10:48:40 +0900186 remoteName = self.remoteAlias
Dan Willemsen96c2d652016-04-06 16:03:54 -0700187 return RemoteSpec(remoteName,
188 url=url,
Steve Raed6480452016-08-10 15:00:00 -0700189 pushUrl=self.pushUrl,
Dan Willemsen96c2d652016-04-06 16:03:54 -0700190 review=self.reviewUrl,
David Rileye0684ad2017-04-05 00:02:59 -0700191 orig_name=self.name,
192 fetchUrl=self.fetchUrl)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700193
David Pursehouse819827a2020-02-12 15:20:19 +0900194
Shawn O. Pearcec8a300f2009-05-18 13:19:57 -0700195class XmlManifest(object):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700196 """manages the repo configuration file"""
197
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -0400198 def __init__(self, repodir, manifest_file, local_manifests=None):
199 """Initialize.
200
201 Args:
202 repodir: Path to the .repo/ dir for holding all internal checkout state.
203 It must be in the top directory of the repo client checkout.
204 manifest_file: Full path to the manifest file to parse. This will usually
205 be |repodir|/|MANIFEST_FILE_NAME|.
206 local_manifests: Full path to the directory of local override manifests.
207 This will usually be |repodir|/|LOCAL_MANIFESTS_DIR_NAME|.
208 """
209 # TODO(vapier): Move this out of this class.
210 self.globalConfig = GitConfig.ForUser()
211
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700212 self.repodir = os.path.abspath(repodir)
213 self.topdir = os.path.dirname(self.repodir)
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -0400214 self.manifestFile = manifest_file
215 self.local_manifests = local_manifests
Basil Gelloc7453502018-05-25 20:23:52 +0300216 self._load_local_manifests = True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700217
218 self.repoProject = MetaProject(self, 'repo',
David Pursehouseabdf7502020-02-12 14:58:39 +0900219 gitdir=os.path.join(repodir, 'repo/.git'),
220 worktree=os.path.join(repodir, 'repo'))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700221
Mike Frysinger979d5bd2020-02-09 02:28:34 -0500222 mp = MetaProject(self, 'manifests',
223 gitdir=os.path.join(repodir, 'manifests.git'),
224 worktree=os.path.join(repodir, 'manifests'))
225 self.manifestProject = mp
226
227 # This is a bit hacky, but we're in a chicken & egg situation: all the
228 # normal repo settings live in the manifestProject which we just setup
229 # above, so we couldn't easily query before that. We assume Project()
230 # init doesn't care if this changes afterwards.
Mike Frysingerd957ec62020-02-24 14:40:25 -0500231 if os.path.exists(mp.gitdir) and mp.config.GetBoolean('repo.worktree'):
Mike Frysinger979d5bd2020-02-09 02:28:34 -0500232 mp.use_git_worktrees = True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700233
234 self._Unload()
235
Basil Gelloc7453502018-05-25 20:23:52 +0300236 def Override(self, name, load_local_manifests=True):
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700237 """Use a different manifest, just for the current instantiation.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700238 """
Basil Gelloc7453502018-05-25 20:23:52 +0300239 path = None
240
241 # Look for a manifest by path in the filesystem (including the cwd).
242 if not load_local_manifests:
243 local_path = os.path.abspath(name)
244 if os.path.isfile(local_path):
245 path = local_path
246
247 # Look for manifests by name from the manifests repo.
248 if path is None:
249 path = os.path.join(self.manifestProject.worktree, name)
250 if not os.path.isfile(path):
251 raise ManifestParseError('manifest %s not found' % name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700252
253 old = self.manifestFile
254 try:
Basil Gelloc7453502018-05-25 20:23:52 +0300255 self._load_local_manifests = load_local_manifests
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700256 self.manifestFile = path
257 self._Unload()
258 self._Load()
259 finally:
260 self.manifestFile = old
261
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700262 def Link(self, name):
263 """Update the repo metadata to use a different manifest.
264 """
265 self.Override(name)
266
Mike Frysingera269b1c2020-02-21 00:49:41 -0500267 # Old versions of repo would generate symlinks we need to clean up.
268 if os.path.lexists(self.manifestFile):
269 platform_utils.remove(self.manifestFile)
270 # This file is interpreted as if it existed inside the manifest repo.
271 # That allows us to use <include> with the relative file name.
272 with open(self.manifestFile, 'w') as fp:
273 fp.write("""<?xml version="1.0" encoding="UTF-8"?>
274<!--
275DO NOT EDIT THIS FILE! It is generated by repo and changes will be discarded.
276If you want to use a different manifest, use `repo init -m <file>` instead.
277
278If you want to customize your checkout by overriding manifest settings, use
279the local_manifests/ directory instead.
280
281For more information on repo manifests, check out:
282https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
283-->
284<manifest>
285 <include name="%s" />
286</manifest>
287""" % (name,))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700288
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800289 def _RemoteToXml(self, r, doc, root):
290 e = doc.createElement('remote')
291 root.appendChild(e)
292 e.setAttribute('name', r.name)
293 e.setAttribute('fetch', r.fetchUrl)
Steve Raed6480452016-08-10 15:00:00 -0700294 if r.pushUrl is not None:
295 e.setAttribute('pushurl', r.pushUrl)
Conley Owens1e7ab2a2013-10-08 17:26:57 -0700296 if r.remoteAlias is not None:
297 e.setAttribute('alias', r.remoteAlias)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800298 if r.reviewUrl is not None:
299 e.setAttribute('review', r.reviewUrl)
Anthony King36ea2fb2014-05-06 11:54:01 +0100300 if r.revision is not None:
301 e.setAttribute('revision', r.revision)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800302
Mike Frysinger51e39d52020-12-04 05:32:06 -0500303 def _ParseList(self, field):
304 """Parse fields that contain flattened lists.
305
306 These are whitespace & comma separated. Empty elements will be discarded.
307 """
308 return [x for x in re.split(r'[,\s]+', field) if x]
Josh Triplett884a3872014-06-12 14:57:29 -0700309
Mike Frysinger23411d32020-09-02 04:31:10 -0400310 def ToXml(self, peg_rev=False, peg_rev_upstream=True, peg_rev_dest_branch=True, groups=None):
311 """Return the current manifest XML."""
Colin Cross5acde752012-03-28 20:15:45 -0700312 mp = self.manifestProject
313
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700314 if groups is None:
315 groups = mp.config.GetString('manifest.groups')
Matt Gumbel0c635bb2012-12-21 10:14:53 -0800316 if groups:
Mike Frysinger51e39d52020-12-04 05:32:06 -0500317 groups = self._ParseList(groups)
Colin Cross5acde752012-03-28 20:15:45 -0700318
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800319 doc = xml.dom.minidom.Document()
320 root = doc.createElement('manifest')
321 doc.appendChild(root)
322
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700323 # Save out the notice. There's a little bit of work here to give it the
324 # right whitespace, which assumes that the notice is automatically indented
325 # by 4 by minidom.
326 if self.notice:
327 notice_element = root.appendChild(doc.createElement('notice'))
328 notice_lines = self.notice.splitlines()
David Pursehouse54a4e602020-02-12 14:31:05 +0900329 indented_notice = ('\n'.join(" " * 4 + line for line in notice_lines))[4:]
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700330 notice_element.appendChild(doc.createTextNode(indented_notice))
331
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800332 d = self.default
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800333
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530334 for r in sorted(self.remotes):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800335 self._RemoteToXml(self.remotes[r], doc, root)
336 if self.remotes:
337 root.appendChild(doc.createTextNode(''))
338
339 have_default = False
340 e = doc.createElement('default')
341 if d.remote:
342 have_default = True
343 e.setAttribute('remote', d.remote.name)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700344 if d.revisionExpr:
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800345 have_default = True
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700346 e.setAttribute('revision', d.revisionExpr)
Simon Ruggier7e59de22015-07-24 12:50:06 +0200347 if d.destBranchExpr:
348 have_default = True
349 e.setAttribute('dest-branch', d.destBranchExpr)
Nasser Grainawida403412018-05-04 12:53:29 -0600350 if d.upstreamExpr:
351 have_default = True
352 e.setAttribute('upstream', d.upstreamExpr)
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700353 if d.sync_j > 1:
354 have_default = True
355 e.setAttribute('sync-j', '%d' % d.sync_j)
Anatol Pomazau79770d22012-04-20 14:41:59 -0700356 if d.sync_c:
357 have_default = True
358 e.setAttribute('sync-c', 'true')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800359 if d.sync_s:
360 have_default = True
361 e.setAttribute('sync-s', 'true')
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900362 if not d.sync_tags:
363 have_default = True
364 e.setAttribute('sync-tags', 'false')
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800365 if have_default:
366 root.appendChild(e)
367 root.appendChild(doc.createTextNode(''))
368
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700369 if self._manifest_server:
370 e = doc.createElement('manifest-server')
371 e.setAttribute('url', self._manifest_server)
372 root.appendChild(e)
373 root.appendChild(doc.createTextNode(''))
374
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800375 def output_projects(parent, parent_node, projects):
David James8d201162013-10-11 17:03:19 -0700376 for project_name in projects:
377 for project in self._projects[project_name]:
378 output_project(parent, parent_node, project)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800379
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800380 def output_project(parent, parent_node, p):
Colin Cross5acde752012-03-28 20:15:45 -0700381 if not p.MatchesGroups(groups):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800382 return
383
384 name = p.name
385 relpath = p.relpath
386 if parent:
387 name = self._UnjoinName(parent.name, name)
388 relpath = self._UnjoinRelpath(parent.relpath, relpath)
Colin Cross5acde752012-03-28 20:15:45 -0700389
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800390 e = doc.createElement('project')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800391 parent_node.appendChild(e)
392 e.setAttribute('name', name)
393 if relpath != name:
394 e.setAttribute('path', relpath)
Conley Owensa17d7af2013-10-16 14:38:09 -0700395 remoteName = None
396 if d.remote:
Dan Willemsen96c2d652016-04-06 16:03:54 -0700397 remoteName = d.remote.name
398 if not d.remote or p.remote.orig_name != remoteName:
399 remoteName = p.remote.orig_name
Anthony King36ea2fb2014-05-06 11:54:01 +0100400 e.setAttribute('remote', remoteName)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800401 if peg_rev:
402 if self.IsMirror:
Brian Harring14a66742012-09-28 20:21:57 -0700403 value = p.bare_git.rev_parse(p.revisionExpr + '^0')
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800404 else:
Brian Harring14a66742012-09-28 20:21:57 -0700405 value = p.work_git.rev_parse(HEAD + '^0')
406 e.setAttribute('revision', value)
Conley Owens551dfec2015-07-10 14:54:54 -0700407 if peg_rev_upstream:
408 if p.upstream:
409 e.setAttribute('upstream', p.upstream)
410 elif value != p.revisionExpr:
411 # Only save the origin if the origin is not a sha1, and the default
412 # isn't our value
413 e.setAttribute('upstream', p.revisionExpr)
Sean McAllisteraf908cb2020-04-20 08:41:58 -0600414
415 if peg_rev_dest_branch:
416 if p.dest_branch:
417 e.setAttribute('dest-branch', p.dest_branch)
418 elif value != p.revisionExpr:
419 e.setAttribute('dest-branch', p.revisionExpr)
420
Anthony King36ea2fb2014-05-06 11:54:01 +0100421 else:
Dan Willemsen96c2d652016-04-06 16:03:54 -0700422 revision = self.remotes[p.remote.orig_name].revision or d.revisionExpr
Anthony King36ea2fb2014-05-06 11:54:01 +0100423 if not revision or revision != p.revisionExpr:
424 e.setAttribute('revision', p.revisionExpr)
Raman Tennetib5c5a5e2021-02-06 09:44:15 -0800425 elif p.revisionId:
426 e.setAttribute('revision', p.revisionId)
Nasser Grainawida403412018-05-04 12:53:29 -0600427 if (p.upstream and (p.upstream != p.revisionExpr or
428 p.upstream != d.upstreamExpr)):
Mani Chandel7a91d512014-07-24 16:27:08 +0530429 e.setAttribute('upstream', p.upstream)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800430
Simon Ruggier7e59de22015-07-24 12:50:06 +0200431 if p.dest_branch and p.dest_branch != d.destBranchExpr:
432 e.setAttribute('dest-branch', p.dest_branch)
433
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800434 for c in p.copyfiles:
435 ce = doc.createElement('copyfile')
436 ce.setAttribute('src', c.src)
437 ce.setAttribute('dest', c.dest)
438 e.appendChild(ce)
439
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500440 for l in p.linkfiles:
441 le = doc.createElement('linkfile')
442 le.setAttribute('src', l.src)
443 le.setAttribute('dest', l.dest)
444 e.appendChild(le)
445
Conley Owensbb1b5f52012-08-13 13:11:18 -0700446 default_groups = ['all', 'name:%s' % p.name, 'path:%s' % p.relpath]
Dmitry Fink17f85ea2012-08-06 14:52:29 -0700447 egroups = [g for g in p.groups if g not in default_groups]
Conley Owens971de8e2012-04-16 10:36:08 -0700448 if egroups:
449 e.setAttribute('groups', ','.join(egroups))
Colin Cross5acde752012-03-28 20:15:45 -0700450
James W. Mills24c13082012-04-12 15:04:13 -0500451 for a in p.annotations:
452 if a.keep == "true":
453 ae = doc.createElement('annotation')
454 ae.setAttribute('name', a.name)
455 ae.setAttribute('value', a.value)
456 e.appendChild(ae)
457
Anatol Pomazau79770d22012-04-20 14:41:59 -0700458 if p.sync_c:
459 e.setAttribute('sync-c', 'true')
460
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800461 if p.sync_s:
462 e.setAttribute('sync-s', 'true')
463
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900464 if not p.sync_tags:
465 e.setAttribute('sync-tags', 'false')
466
Dan Willemsen88409222015-08-17 15:29:10 -0700467 if p.clone_depth:
468 e.setAttribute('clone-depth', str(p.clone_depth))
469
Simran Basib9a1b732015-08-20 12:19:28 -0700470 self._output_manifest_project_extras(p, e)
471
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800472 if p.subprojects:
David James8d201162013-10-11 17:03:19 -0700473 subprojects = set(subp.name for subp in p.subprojects)
474 output_projects(p, e, list(sorted(subprojects)))
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800475
David James8d201162013-10-11 17:03:19 -0700476 projects = set(p.name for p in self._paths.values() if not p.parent)
477 output_projects(None, root, list(sorted(projects)))
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800478
Doug Anderson37282b42011-03-04 11:54:18 -0800479 if self._repo_hooks_project:
480 root.appendChild(doc.createTextNode(''))
481 e = doc.createElement('repo-hooks')
482 e.setAttribute('in-project', self._repo_hooks_project.name)
483 e.setAttribute('enabled-list',
484 ' '.join(self._repo_hooks_project.enabled_repo_hooks))
485 root.appendChild(e)
486
Raman Tenneti1bb4fb22021-01-07 16:50:45 -0800487 if self._superproject:
488 root.appendChild(doc.createTextNode(''))
489 e = doc.createElement('superproject')
490 e.setAttribute('name', self._superproject['name'])
491 remoteName = None
492 if d.remote:
493 remoteName = d.remote.name
494 remote = self._superproject.get('remote')
495 if not d.remote or remote.orig_name != remoteName:
496 remoteName = remote.orig_name
497 e.setAttribute('remote', remoteName)
498 root.appendChild(e)
499
Raman Tenneti993af5e2021-05-12 12:00:31 -0700500 if self._contactinfo.bugurl != Wrapper().BUG_URL:
Raman Tenneti1c3f57e2021-05-04 12:32:13 -0700501 root.appendChild(doc.createTextNode(''))
502 e = doc.createElement('contactinfo')
Raman Tenneti993af5e2021-05-12 12:00:31 -0700503 e.setAttribute('bugurl', self._contactinfo.bugurl)
Raman Tenneti1c3f57e2021-05-04 12:32:13 -0700504 root.appendChild(e)
505
Mike Frysinger23411d32020-09-02 04:31:10 -0400506 return doc
507
508 def ToDict(self, **kwargs):
509 """Return the current manifest as a dictionary."""
510 # Elements that may only appear once.
511 SINGLE_ELEMENTS = {
512 'notice',
513 'default',
514 'manifest-server',
515 'repo-hooks',
Raman Tenneti1bb4fb22021-01-07 16:50:45 -0800516 'superproject',
Raman Tenneti1c3f57e2021-05-04 12:32:13 -0700517 'contactinfo',
Mike Frysinger23411d32020-09-02 04:31:10 -0400518 }
519 # Elements that may be repeated.
520 MULTI_ELEMENTS = {
521 'remote',
522 'remove-project',
523 'project',
524 'extend-project',
525 'include',
526 # These are children of 'project' nodes.
527 'annotation',
528 'project',
529 'copyfile',
530 'linkfile',
531 }
532
533 doc = self.ToXml(**kwargs)
534 ret = {}
535
536 def append_children(ret, node):
537 for child in node.childNodes:
538 if child.nodeType == xml.dom.Node.ELEMENT_NODE:
539 attrs = child.attributes
540 element = dict((attrs.item(i).localName, attrs.item(i).value)
541 for i in range(attrs.length))
542 if child.nodeName in SINGLE_ELEMENTS:
543 ret[child.nodeName] = element
544 elif child.nodeName in MULTI_ELEMENTS:
545 ret.setdefault(child.nodeName, []).append(element)
546 else:
547 raise ManifestParseError('Unhandled element "%s"' % (child.nodeName,))
548
549 append_children(element, child)
550
551 append_children(ret, doc.firstChild)
552
553 return ret
554
555 def Save(self, fd, **kwargs):
556 """Write the current manifest out to the given file descriptor."""
557 doc = self.ToXml(**kwargs)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800558 doc.writexml(fd, '', ' ', '\n', 'UTF-8')
559
Simran Basib9a1b732015-08-20 12:19:28 -0700560 def _output_manifest_project_extras(self, p, e):
561 """Manifests can modify e if they support extra project attributes."""
Simran Basib9a1b732015-08-20 12:19:28 -0700562
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700563 @property
David James8d201162013-10-11 17:03:19 -0700564 def paths(self):
565 self._Load()
566 return self._paths
567
568 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700569 def projects(self):
570 self._Load()
Anthony Kingd58bfe52014-05-05 23:30:49 +0100571 return list(self._paths.values())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700572
573 @property
574 def remotes(self):
575 self._Load()
576 return self._remotes
577
578 @property
579 def default(self):
580 self._Load()
581 return self._default
582
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800583 @property
Doug Anderson37282b42011-03-04 11:54:18 -0800584 def repo_hooks_project(self):
585 self._Load()
586 return self._repo_hooks_project
587
588 @property
Raman Tenneti1bb4fb22021-01-07 16:50:45 -0800589 def superproject(self):
590 self._Load()
591 return self._superproject
592
593 @property
Raman Tenneti1c3f57e2021-05-04 12:32:13 -0700594 def contactinfo(self):
595 self._Load()
596 return self._contactinfo
597
598 @property
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700599 def notice(self):
600 self._Load()
601 return self._notice
602
603 @property
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700604 def manifest_server(self):
605 self._Load()
Shawn O. Pearce34fb20f2011-11-30 13:41:02 -0800606 return self._manifest_server
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700607
608 @property
Xin Lid79a4bc2020-05-20 16:03:45 -0700609 def CloneBundle(self):
610 clone_bundle = self.manifestProject.config.GetBoolean('repo.clonebundle')
611 if clone_bundle is None:
612 return False if self.manifestProject.config.GetBoolean('repo.partialclone') else True
613 else:
614 return clone_bundle
615
616 @property
Xin Li745be2e2019-06-03 11:24:30 -0700617 def CloneFilter(self):
618 if self.manifestProject.config.GetBoolean('repo.partialclone'):
619 return self.manifestProject.config.GetString('repo.clonefilter')
620 return None
621
622 @property
Raman Tennetif32f2432021-04-12 20:57:25 -0700623 def PartialCloneExclude(self):
624 exclude = self.manifest.manifestProject.config.GetString(
625 'repo.partialcloneexclude') or ''
626 return set(x.strip() for x in exclude.split(','))
627
628 @property
Raman Tennetifeb28912021-05-02 19:47:29 -0700629 def HasLocalManifests(self):
630 return self._load_local_manifests and self.local_manifests
631
632 @property
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800633 def IsMirror(self):
634 return self.manifestProject.config.GetBoolean('repo.mirror')
635
Julien Campergue335f5ef2013-10-16 11:02:35 +0200636 @property
Mike Frysinger979d5bd2020-02-09 02:28:34 -0500637 def UseGitWorktrees(self):
638 return self.manifestProject.config.GetBoolean('repo.worktree')
639
640 @property
Julien Campergue335f5ef2013-10-16 11:02:35 +0200641 def IsArchive(self):
642 return self.manifestProject.config.GetBoolean('repo.archive')
643
Martin Kellye4e94d22017-03-21 16:05:12 -0700644 @property
645 def HasSubmodules(self):
646 return self.manifestProject.config.GetBoolean('repo.submodules')
647
Raman Tenneti080877e2021-03-09 15:19:06 -0800648 def GetDefaultGroupsStr(self):
649 """Returns the default group string for the platform."""
650 return 'default,platform-' + platform.system().lower()
651
652 def GetGroupsStr(self):
653 """Returns the manifest group string that should be synced."""
654 groups = self.manifestProject.config.GetString('manifest.groups')
655 if not groups:
656 groups = self.GetDefaultGroupsStr()
657 return groups
658
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700659 def _Unload(self):
660 self._loaded = False
661 self._projects = {}
David James8d201162013-10-11 17:03:19 -0700662 self._paths = {}
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700663 self._remotes = {}
664 self._default = None
Doug Anderson37282b42011-03-04 11:54:18 -0800665 self._repo_hooks_project = None
Raman Tenneti1bb4fb22021-01-07 16:50:45 -0800666 self._superproject = {}
Raman Tenneti993af5e2021-05-12 12:00:31 -0700667 self._contactinfo = ContactInfo(Wrapper().BUG_URL)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700668 self._notice = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700669 self.branch = None
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700670 self._manifest_server = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700671
672 def _Load(self):
673 if not self._loaded:
Shawn O. Pearce2450a292008-11-04 08:22:07 -0800674 m = self.manifestProject
675 b = m.GetBranch(m.CurrentBranch).merge
Shawn O. Pearce21c5c342009-06-25 16:47:30 -0700676 if b is not None and b.startswith(R_HEADS):
Shawn O. Pearce2450a292008-11-04 08:22:07 -0800677 b = b[len(R_HEADS):]
678 self.branch = b
679
Mike Frysinger54133972021-03-01 21:38:08 -0500680 # The manifestFile was specified by the user which is why we allow include
681 # paths to point anywhere.
Colin Cross23acdd32012-04-21 00:33:54 -0700682 nodes = []
Mike Frysinger54133972021-03-01 21:38:08 -0500683 nodes.append(self._ParseManifestXml(
684 self.manifestFile, self.manifestProject.worktree,
685 restrict_includes=False))
Shawn O. Pearce5cc66792008-10-23 16:19:27 -0700686
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -0400687 if self._load_local_manifests and self.local_manifests:
Basil Gelloc7453502018-05-25 20:23:52 +0300688 try:
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -0400689 for local_file in sorted(platform_utils.listdir(self.local_manifests)):
Basil Gelloc7453502018-05-25 20:23:52 +0300690 if local_file.endswith('.xml'):
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -0400691 local = os.path.join(self.local_manifests, local_file)
Mike Frysinger54133972021-03-01 21:38:08 -0500692 # Since local manifests are entirely managed by the user, allow
693 # them to point anywhere the user wants.
694 nodes.append(self._ParseManifestXml(
Raman Tenneti78f4dd32021-06-07 13:27:37 -0700695 local, self.repodir,
696 parent_groups=f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}',
697 restrict_includes=False))
Basil Gelloc7453502018-05-25 20:23:52 +0300698 except OSError:
699 pass
David Pursehouse2d5a0df2012-11-13 02:50:36 +0900700
Joe Onorato26e24752013-01-11 12:35:53 -0800701 try:
702 self._ParseManifest(nodes)
703 except ManifestParseError as e:
704 # There was a problem parsing, unload ourselves in case they catch
705 # this error and try again later, we will show the correct error
706 self._Unload()
707 raise e
Shawn O. Pearce5cc66792008-10-23 16:19:27 -0700708
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800709 if self.IsMirror:
710 self._AddMetaProjectMirror(self.repoProject)
711 self._AddMetaProjectMirror(self.manifestProject)
712
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700713 self._loaded = True
714
Mike Frysinger54133972021-03-01 21:38:08 -0500715 def _ParseManifestXml(self, path, include_root, parent_groups='',
716 restrict_includes=True):
717 """Parse a manifest XML and return the computed nodes.
718
719 Args:
720 path: The XML file to read & parse.
721 include_root: The path to interpret include "name"s relative to.
722 parent_groups: The groups to apply to this projects.
723 restrict_includes: Whether to constrain the "name" attribute of includes.
724
725 Returns:
726 List of XML nodes.
727 """
David Pursehousef7fc8a92012-11-13 04:00:28 +0900728 try:
729 root = xml.dom.minidom.parse(path)
David Pursehouse2d5a0df2012-11-13 02:50:36 +0900730 except (OSError, xml.parsers.expat.ExpatError) as e:
David Pursehousef7fc8a92012-11-13 04:00:28 +0900731 raise ManifestParseError("error parsing manifest %s: %s" % (path, e))
732
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700733 if not root or not root.childNodes:
Brian Harring26448742011-04-28 05:04:41 -0700734 raise ManifestParseError("no root node in %s" % (path,))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700735
Jooncheol Park34acdd22012-08-27 02:25:59 +0900736 for manifest in root.childNodes:
737 if manifest.nodeName == 'manifest':
738 break
739 else:
Brian Harring26448742011-04-28 05:04:41 -0700740 raise ManifestParseError("no <manifest> in %s" % (path,))
741
Colin Cross23acdd32012-04-21 00:33:54 -0700742 nodes = []
David Pursehouse65b0ba52018-06-24 16:21:51 +0900743 for node in manifest.childNodes:
David Pursehousec1b86a22012-11-14 11:36:51 +0900744 if node.nodeName == 'include':
745 name = self._reqatt(node, 'name')
Mike Frysinger54133972021-03-01 21:38:08 -0500746 if restrict_includes:
747 msg = self._CheckLocalPath(name)
748 if msg:
749 raise ManifestInvalidPathError(
750 '<include> invalid "name": %s: %s' % (name, msg))
Fredrik de Groot352c93b2020-10-06 12:55:14 +0200751 include_groups = ''
752 if parent_groups:
753 include_groups = parent_groups
754 if node.hasAttribute('groups'):
755 include_groups = node.getAttribute('groups') + ',' + include_groups
David Pursehousec1b86a22012-11-14 11:36:51 +0900756 fp = os.path.join(include_root, name)
757 if not os.path.isfile(fp):
Mike Frysinger54133972021-03-01 21:38:08 -0500758 raise ManifestParseError("include [%s/]%s doesn't exist or isn't a file"
759 % (include_root, name))
David Pursehousec1b86a22012-11-14 11:36:51 +0900760 try:
Fredrik de Groot352c93b2020-10-06 12:55:14 +0200761 nodes.extend(self._ParseManifestXml(fp, include_root, include_groups))
David Pursehousec1b86a22012-11-14 11:36:51 +0900762 # should isolate this to the exact exception, but that's
763 # tricky. actual parsing implementation may vary.
Mike Frysinger54133972021-03-01 21:38:08 -0500764 except (KeyboardInterrupt, RuntimeError, SystemExit, ManifestParseError):
David Pursehousec1b86a22012-11-14 11:36:51 +0900765 raise
766 except Exception as e:
767 raise ManifestParseError(
Mike Frysingerec558df2019-07-05 01:38:05 -0400768 "failed parsing included manifest %s: %s" % (name, e))
David Pursehousec1b86a22012-11-14 11:36:51 +0900769 else:
Fredrik de Groot352c93b2020-10-06 12:55:14 +0200770 if parent_groups and node.nodeName == 'project':
771 nodeGroups = parent_groups
772 if node.hasAttribute('groups'):
773 nodeGroups = node.getAttribute('groups') + ',' + nodeGroups
774 node.setAttribute('groups', nodeGroups)
David Pursehousec1b86a22012-11-14 11:36:51 +0900775 nodes.append(node)
Colin Cross23acdd32012-04-21 00:33:54 -0700776 return nodes
Brian Harring26448742011-04-28 05:04:41 -0700777
Colin Cross23acdd32012-04-21 00:33:54 -0700778 def _ParseManifest(self, node_list):
779 for node in itertools.chain(*node_list):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700780 if node.nodeName == 'remote':
781 remote = self._ParseRemote(node)
David Pursehouse717ece92012-11-13 08:49:16 +0900782 if remote:
783 if remote.name in self._remotes:
784 if remote != self._remotes[remote.name]:
785 raise ManifestParseError(
786 'remote %s already exists with different attributes' %
787 (remote.name))
788 else:
789 self._remotes[remote.name] = remote
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700790
Colin Cross23acdd32012-04-21 00:33:54 -0700791 for node in itertools.chain(*node_list):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700792 if node.nodeName == 'default':
Julien Campergue74879922013-10-09 14:38:46 +0200793 new_default = self._ParseDefault(node)
794 if self._default is None:
795 self._default = new_default
796 elif new_default != self._default:
David Pursehouse37128b62013-10-15 10:48:40 +0900797 raise ManifestParseError('duplicate default in %s' %
798 (self.manifestFile))
Julien Campergue74879922013-10-09 14:38:46 +0200799
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700800 if self._default is None:
801 self._default = _Default()
802
Colin Cross23acdd32012-04-21 00:33:54 -0700803 for node in itertools.chain(*node_list):
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700804 if node.nodeName == 'notice':
805 if self._notice is not None:
Doug Anderson37282b42011-03-04 11:54:18 -0800806 raise ManifestParseError(
807 'duplicate notice in %s' %
808 (self.manifestFile))
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700809 self._notice = self._ParseNotice(node)
810
Colin Cross23acdd32012-04-21 00:33:54 -0700811 for node in itertools.chain(*node_list):
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700812 if node.nodeName == 'manifest-server':
813 url = self._reqatt(node, 'url')
814 if self._manifest_server is not None:
David Pursehousec1b86a22012-11-14 11:36:51 +0900815 raise ManifestParseError(
816 'duplicate manifest-server in %s' %
817 (self.manifestFile))
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700818 self._manifest_server = url
819
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800820 def recursively_add_projects(project):
David James8d201162013-10-11 17:03:19 -0700821 projects = self._projects.setdefault(project.name, [])
822 if project.relpath is None:
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800823 raise ManifestParseError(
David James8d201162013-10-11 17:03:19 -0700824 'missing path for %s in %s' %
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800825 (project.name, self.manifestFile))
David James8d201162013-10-11 17:03:19 -0700826 if project.relpath in self._paths:
827 raise ManifestParseError(
828 'duplicate path %s in %s' %
829 (project.relpath, self.manifestFile))
830 self._paths[project.relpath] = project
831 projects.append(project)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800832 for subproject in project.subprojects:
833 recursively_add_projects(subproject)
834
Colin Cross23acdd32012-04-21 00:33:54 -0700835 for node in itertools.chain(*node_list):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700836 if node.nodeName == 'project':
837 project = self._ParseProject(node)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800838 recursively_add_projects(project)
Josh Triplett884a3872014-06-12 14:57:29 -0700839 if node.nodeName == 'extend-project':
840 name = self._reqatt(node, 'name')
841
842 if name not in self._projects:
843 raise ManifestParseError('extend-project element specifies non-existent '
844 'project: %s' % name)
845
846 path = node.getAttribute('path')
847 groups = node.getAttribute('groups')
848 if groups:
Mike Frysinger51e39d52020-12-04 05:32:06 -0500849 groups = self._ParseList(groups)
Luis Hector Chavez7d525852018-03-15 09:54:08 -0700850 revision = node.getAttribute('revision')
Kyunam Jobd0aae92020-02-04 11:38:53 +0900851 remote = node.getAttribute('remote')
852 if remote:
853 remote = self._get_remote(node)
Josh Triplett884a3872014-06-12 14:57:29 -0700854
855 for p in self._projects[name]:
856 if path and p.relpath != path:
857 continue
858 if groups:
859 p.groups.extend(groups)
Luis Hector Chavez7d525852018-03-15 09:54:08 -0700860 if revision:
861 p.revisionExpr = revision
Miguel Gaio1f207762020-07-17 14:09:13 +0200862 if IsId(revision):
863 p.revisionId = revision
864 else:
865 p.revisionId = None
Kyunam Jobd0aae92020-02-04 11:38:53 +0900866 if remote:
867 p.remote = remote.ToRemoteSpec(name)
Doug Anderson37282b42011-03-04 11:54:18 -0800868 if node.nodeName == 'repo-hooks':
869 # Get the name of the project and the (space-separated) list of enabled.
870 repo_hooks_project = self._reqatt(node, 'in-project')
Mike Frysinger51e39d52020-12-04 05:32:06 -0500871 enabled_repo_hooks = self._ParseList(self._reqatt(node, 'enabled-list'))
Doug Anderson37282b42011-03-04 11:54:18 -0800872
873 # Only one project can be the hooks project
874 if self._repo_hooks_project is not None:
875 raise ManifestParseError(
876 'duplicate repo-hooks in %s' %
877 (self.manifestFile))
878
879 # Store a reference to the Project.
880 try:
David James8d201162013-10-11 17:03:19 -0700881 repo_hooks_projects = self._projects[repo_hooks_project]
Doug Anderson37282b42011-03-04 11:54:18 -0800882 except KeyError:
883 raise ManifestParseError(
884 'project %s not found for repo-hooks' %
885 (repo_hooks_project))
886
David James8d201162013-10-11 17:03:19 -0700887 if len(repo_hooks_projects) != 1:
888 raise ManifestParseError(
889 'internal error parsing repo-hooks in %s' %
890 (self.manifestFile))
891 self._repo_hooks_project = repo_hooks_projects[0]
892
Doug Anderson37282b42011-03-04 11:54:18 -0800893 # Store the enabled hooks in the Project object.
894 self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
Raman Tenneti1bb4fb22021-01-07 16:50:45 -0800895 if node.nodeName == 'superproject':
896 name = self._reqatt(node, 'name')
897 # There can only be one superproject.
898 if self._superproject.get('name'):
899 raise ManifestParseError(
900 'duplicate superproject in %s' %
901 (self.manifestFile))
902 self._superproject['name'] = name
903 remote_name = node.getAttribute('remote')
904 if not remote_name:
905 remote = self._default.remote
906 else:
907 remote = self._get_remote(node)
908 if remote is None:
909 raise ManifestParseError("no remote for superproject %s within %s" %
910 (name, self.manifestFile))
911 self._superproject['remote'] = remote.ToRemoteSpec(name)
Raman Tenneti1c3f57e2021-05-04 12:32:13 -0700912 if node.nodeName == 'contactinfo':
913 bugurl = self._reqatt(node, 'bugurl')
914 # This element can be repeated, later entries will clobber earlier ones.
Raman Tenneti993af5e2021-05-12 12:00:31 -0700915 self._contactinfo = ContactInfo(bugurl)
916
Colin Cross23acdd32012-04-21 00:33:54 -0700917 if node.nodeName == 'remove-project':
918 name = self._reqatt(node, 'name')
David Jamesb8433df2014-01-30 10:11:17 -0800919
920 if name not in self._projects:
David Pursehousef9107482012-11-16 19:12:32 +0900921 raise ManifestParseError('remove-project element specifies non-existent '
922 'project: %s' % name)
Colin Cross23acdd32012-04-21 00:33:54 -0700923
David Jamesb8433df2014-01-30 10:11:17 -0800924 for p in self._projects[name]:
925 del self._paths[p.relpath]
926 del self._projects[name]
927
Colin Cross23acdd32012-04-21 00:33:54 -0700928 # If the manifest removes the hooks project, treat it as if it deleted
929 # the repo-hooks element too.
930 if self._repo_hooks_project and (self._repo_hooks_project.name == name):
931 self._repo_hooks_project = None
932
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800933 def _AddMetaProjectMirror(self, m):
934 name = None
935 m_url = m.GetRemote(m.remote.name).url
936 if m_url.endswith('/.git'):
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530937 raise ManifestParseError('refusing to mirror %s' % m_url)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800938
939 if self._default and self._default.remote:
Conley Owensceea3682011-10-20 10:45:47 -0700940 url = self._default.remote.resolvedFetchUrl
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800941 if not url.endswith('/'):
942 url += '/'
943 if m_url.startswith(url):
944 remote = self._default.remote
945 name = m_url[len(url):]
946
947 if name is None:
948 s = m_url.rindex('/') + 1
Conley Owensdb728cd2011-09-26 16:34:01 -0700949 manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
Shawn O. Pearcef35b2d92012-08-02 11:46:22 -0700950 remote = _XmlRemote('origin', fetch=m_url[:s], manifestUrl=manifestUrl)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800951 name = m_url[s:]
952
953 if name.endswith('.git'):
954 name = name[:-4]
955
956 if name not in self._projects:
957 m.PreSync()
958 gitdir = os.path.join(self.topdir, '%s.git' % name)
David Pursehousee5913ae2020-02-12 13:56:59 +0900959 project = Project(manifest=self,
960 name=name,
961 remote=remote.ToRemoteSpec(name),
962 gitdir=gitdir,
963 objdir=gitdir,
964 worktree=None,
965 relpath=name or None,
966 revisionExpr=m.revisionExpr,
967 revisionId=None)
David James8d201162013-10-11 17:03:19 -0700968 self._projects[project.name] = [project]
Kwanhong Leeccd218c2014-02-17 13:07:32 +0900969 self._paths[project.relpath] = project
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800970
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700971 def _ParseRemote(self, node):
972 """
973 reads a <remote> element from the manifest file
974 """
975 name = self._reqatt(node, 'name')
Yestin Sunb292b982012-07-02 07:32:50 -0700976 alias = node.getAttribute('alias')
977 if alias == '':
978 alias = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700979 fetch = self._reqatt(node, 'fetch')
Steve Raed6480452016-08-10 15:00:00 -0700980 pushUrl = node.getAttribute('pushurl')
981 if pushUrl == '':
982 pushUrl = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700983 review = node.getAttribute('review')
Shawn O. Pearceae6e0942008-11-06 10:25:35 -0800984 if review == '':
985 review = None
Anthony King36ea2fb2014-05-06 11:54:01 +0100986 revision = node.getAttribute('revision')
987 if revision == '':
988 revision = None
Conley Owensdb728cd2011-09-26 16:34:01 -0700989 manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
Steve Raed6480452016-08-10 15:00:00 -0700990 return _XmlRemote(name, alias, fetch, pushUrl, manifestUrl, review, revision)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700991
992 def _ParseDefault(self, node):
993 """
994 reads a <default> element from the manifest file
995 """
996 d = _Default()
997 d.remote = self._get_remote(node)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700998 d.revisionExpr = node.getAttribute('revision')
999 if d.revisionExpr == '':
1000 d.revisionExpr = None
Anatol Pomazau79770d22012-04-20 14:41:59 -07001001
Bryan Jacobsf609f912013-05-06 13:36:24 -04001002 d.destBranchExpr = node.getAttribute('dest-branch') or None
Nasser Grainawida403412018-05-04 12:53:29 -06001003 d.upstreamExpr = node.getAttribute('upstream') or None
Bryan Jacobsf609f912013-05-06 13:36:24 -04001004
Mike Frysingerbb8ee7f2020-02-22 05:30:12 -05001005 d.sync_j = XmlInt(node, 'sync-j', 1)
1006 if d.sync_j <= 0:
1007 raise ManifestParseError('%s: sync-j must be greater than 0, not "%s"' %
1008 (self.manifestFile, d.sync_j))
Anatol Pomazau79770d22012-04-20 14:41:59 -07001009
Mike Frysingerbb8ee7f2020-02-22 05:30:12 -05001010 d.sync_c = XmlBool(node, 'sync-c', False)
1011 d.sync_s = XmlBool(node, 'sync-s', False)
1012 d.sync_tags = XmlBool(node, 'sync-tags', True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001013 return d
1014
Doug Anderson2b8db3c2010-11-01 15:08:06 -07001015 def _ParseNotice(self, node):
1016 """
1017 reads a <notice> element from the manifest file
1018
1019 The <notice> element is distinct from other tags in the XML in that the
1020 data is conveyed between the start and end tag (it's not an empty-element
1021 tag).
1022
1023 The white space (carriage returns, indentation) for the notice element is
1024 relevant and is parsed in a way that is based on how python docstrings work.
1025 In fact, the code is remarkably similar to here:
1026 http://www.python.org/dev/peps/pep-0257/
1027 """
1028 # Get the data out of the node...
1029 notice = node.childNodes[0].data
1030
1031 # Figure out minimum indentation, skipping the first line (the same line
1032 # as the <notice> tag)...
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301033 minIndent = sys.maxsize
Doug Anderson2b8db3c2010-11-01 15:08:06 -07001034 lines = notice.splitlines()
1035 for line in lines[1:]:
1036 lstrippedLine = line.lstrip()
1037 if lstrippedLine:
1038 indent = len(line) - len(lstrippedLine)
1039 minIndent = min(indent, minIndent)
1040
1041 # Strip leading / trailing blank lines and also indentation.
1042 cleanLines = [lines[0].strip()]
1043 for line in lines[1:]:
1044 cleanLines.append(line[minIndent:].rstrip())
1045
1046 # Clear completely blank lines from front and back...
1047 while cleanLines and not cleanLines[0]:
1048 del cleanLines[0]
1049 while cleanLines and not cleanLines[-1]:
1050 del cleanLines[-1]
1051
1052 return '\n'.join(cleanLines)
1053
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001054 def _JoinName(self, parent_name, name):
1055 return os.path.join(parent_name, name)
1056
1057 def _UnjoinName(self, parent_name, name):
1058 return os.path.relpath(name, parent_name)
1059
David Pursehousee5913ae2020-02-12 13:56:59 +09001060 def _ParseProject(self, node, parent=None, **extra_proj_attrs):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001061 """
1062 reads a <project> element from the manifest file
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -07001063 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001064 name = self._reqatt(node, 'name')
Mike Frysingera29424e2021-02-25 21:53:49 -05001065 msg = self._CheckLocalPath(name, dir_ok=True)
1066 if msg:
1067 raise ManifestInvalidPathError(
1068 '<project> invalid "name": %s: %s' % (name, msg))
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001069 if parent:
1070 name = self._JoinName(parent.name, name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001071
1072 remote = self._get_remote(node)
1073 if remote is None:
1074 remote = self._default.remote
1075 if remote is None:
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301076 raise ManifestParseError("no remote for project %s within %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +09001077 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001078
Anthony King36ea2fb2014-05-06 11:54:01 +01001079 revisionExpr = node.getAttribute('revision') or remote.revision
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001080 if not revisionExpr:
1081 revisionExpr = self._default.revisionExpr
1082 if not revisionExpr:
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301083 raise ManifestParseError("no revision for project %s within %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +09001084 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001085
1086 path = node.getAttribute('path')
1087 if not path:
1088 path = name
Mike Frysingera29424e2021-02-25 21:53:49 -05001089 else:
Mike Frysinger0458faa2021-03-10 23:35:44 -05001090 # NB: The "." project is handled specially in Project.Sync_LocalHalf.
1091 msg = self._CheckLocalPath(path, dir_ok=True, cwd_dot_ok=True)
Mike Frysingera29424e2021-02-25 21:53:49 -05001092 if msg:
1093 raise ManifestInvalidPathError(
1094 '<project> invalid "path": %s: %s' % (path, msg))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001095
Mike Frysingerbb8ee7f2020-02-22 05:30:12 -05001096 rebase = XmlBool(node, 'rebase', True)
1097 sync_c = XmlBool(node, 'sync-c', False)
1098 sync_s = XmlBool(node, 'sync-s', self._default.sync_s)
1099 sync_tags = XmlBool(node, 'sync-tags', self._default.sync_tags)
Mike Pontillod3153822012-02-28 11:53:24 -08001100
Mike Frysingerbb8ee7f2020-02-22 05:30:12 -05001101 clone_depth = XmlInt(node, 'clone-depth')
1102 if clone_depth is not None and clone_depth <= 0:
1103 raise ManifestParseError('%s: clone-depth must be greater than 0, not "%s"' %
1104 (self.manifestFile, clone_depth))
David Pursehouseede7f122012-11-27 22:25:30 +09001105
Bryan Jacobsf609f912013-05-06 13:36:24 -04001106 dest_branch = node.getAttribute('dest-branch') or self._default.destBranchExpr
1107
Nasser Grainawida403412018-05-04 12:53:29 -06001108 upstream = node.getAttribute('upstream') or self._default.upstreamExpr
Brian Harring14a66742012-09-28 20:21:57 -07001109
Conley Owens971de8e2012-04-16 10:36:08 -07001110 groups = ''
1111 if node.hasAttribute('groups'):
1112 groups = node.getAttribute('groups')
Mike Frysinger51e39d52020-12-04 05:32:06 -05001113 groups = self._ParseList(groups)
Brian Harring7da13142012-06-15 02:24:20 -07001114
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001115 if parent is None:
Mike Frysinger979d5bd2020-02-09 02:28:34 -05001116 relpath, worktree, gitdir, objdir, use_git_worktrees = \
1117 self.GetProjectPaths(name, path)
Shawn O. Pearcecd81dd62012-10-26 12:18:00 -07001118 else:
Mike Frysinger979d5bd2020-02-09 02:28:34 -05001119 use_git_worktrees = False
David James8d201162013-10-11 17:03:19 -07001120 relpath, worktree, gitdir, objdir = \
1121 self.GetSubprojectPaths(parent, name, path)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001122
1123 default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
1124 groups.extend(set(default_groups).difference(groups))
Shawn O. Pearcecd81dd62012-10-26 12:18:00 -07001125
Scott Fandb83b1b2013-02-28 09:34:14 +08001126 if self.IsMirror and node.hasAttribute('force-path'):
Mike Frysingerbb8ee7f2020-02-22 05:30:12 -05001127 if XmlBool(node, 'force-path', False):
Scott Fandb83b1b2013-02-28 09:34:14 +08001128 gitdir = os.path.join(self.topdir, '%s.git' % path)
1129
David Pursehousee5913ae2020-02-12 13:56:59 +09001130 project = Project(manifest=self,
1131 name=name,
1132 remote=remote.ToRemoteSpec(name),
1133 gitdir=gitdir,
1134 objdir=objdir,
1135 worktree=worktree,
1136 relpath=relpath,
1137 revisionExpr=revisionExpr,
1138 revisionId=None,
1139 rebase=rebase,
1140 groups=groups,
1141 sync_c=sync_c,
1142 sync_s=sync_s,
1143 sync_tags=sync_tags,
1144 clone_depth=clone_depth,
1145 upstream=upstream,
1146 parent=parent,
1147 dest_branch=dest_branch,
Mike Frysinger979d5bd2020-02-09 02:28:34 -05001148 use_git_worktrees=use_git_worktrees,
Simran Basib9a1b732015-08-20 12:19:28 -07001149 **extra_proj_attrs)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001150
1151 for n in node.childNodes:
Shawn O. Pearce242b5262009-05-19 13:00:29 -07001152 if n.nodeName == 'copyfile':
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001153 self._ParseCopyFile(project, n)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001154 if n.nodeName == 'linkfile':
1155 self._ParseLinkFile(project, n)
James W. Mills24c13082012-04-12 15:04:13 -05001156 if n.nodeName == 'annotation':
1157 self._ParseAnnotation(project, n)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001158 if n.nodeName == 'project':
David Pursehousee5913ae2020-02-12 13:56:59 +09001159 project.subprojects.append(self._ParseProject(n, parent=project))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001160
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001161 return project
1162
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001163 def GetProjectPaths(self, name, path):
Mike Frysingercebf2272020-05-26 01:02:29 -04001164 # The manifest entries might have trailing slashes. Normalize them to avoid
1165 # unexpected filesystem behavior since we do string concatenation below.
1166 path = path.rstrip('/')
1167 name = name.rstrip('/')
Mike Frysinger979d5bd2020-02-09 02:28:34 -05001168 use_git_worktrees = False
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001169 relpath = path
1170 if self.IsMirror:
1171 worktree = None
1172 gitdir = os.path.join(self.topdir, '%s.git' % name)
David James8d201162013-10-11 17:03:19 -07001173 objdir = gitdir
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001174 else:
1175 worktree = os.path.join(self.topdir, path).replace('\\', '/')
1176 gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
Mike Frysinger979d5bd2020-02-09 02:28:34 -05001177 # We allow people to mix git worktrees & non-git worktrees for now.
1178 # This allows for in situ migration of repo clients.
1179 if os.path.exists(gitdir) or not self.UseGitWorktrees:
1180 objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
1181 else:
1182 use_git_worktrees = True
1183 gitdir = os.path.join(self.repodir, 'worktrees', '%s.git' % name)
1184 objdir = gitdir
1185 return relpath, worktree, gitdir, objdir, use_git_worktrees
David James8d201162013-10-11 17:03:19 -07001186
1187 def GetProjectsWithName(self, name):
1188 return self._projects.get(name, [])
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001189
1190 def GetSubprojectName(self, parent, submodule_path):
1191 return os.path.join(parent.name, submodule_path)
1192
1193 def _JoinRelpath(self, parent_relpath, relpath):
1194 return os.path.join(parent_relpath, relpath)
1195
1196 def _UnjoinRelpath(self, parent_relpath, relpath):
1197 return os.path.relpath(relpath, parent_relpath)
1198
David James8d201162013-10-11 17:03:19 -07001199 def GetSubprojectPaths(self, parent, name, path):
Mike Frysingercebf2272020-05-26 01:02:29 -04001200 # The manifest entries might have trailing slashes. Normalize them to avoid
1201 # unexpected filesystem behavior since we do string concatenation below.
1202 path = path.rstrip('/')
1203 name = name.rstrip('/')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001204 relpath = self._JoinRelpath(parent.relpath, path)
1205 gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
David James8d201162013-10-11 17:03:19 -07001206 objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001207 if self.IsMirror:
1208 worktree = None
1209 else:
1210 worktree = os.path.join(parent.worktree, path).replace('\\', '/')
David James8d201162013-10-11 17:03:19 -07001211 return relpath, worktree, gitdir, objdir
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001212
Mike Frysinger04122b72019-07-31 23:32:58 -04001213 @staticmethod
Mike Frysingera00c5f42021-02-25 18:26:31 -05001214 def _CheckLocalPath(path, dir_ok=False, cwd_dot_ok=False):
1215 """Verify |path| is reasonable for use in filesystem paths.
1216
Mike Frysingera29424e2021-02-25 21:53:49 -05001217 Used with <copyfile> & <linkfile> & <project> elements.
Mike Frysingera00c5f42021-02-25 18:26:31 -05001218
1219 This only validates the |path| in isolation: it does not check against the
1220 current filesystem state. Thus it is suitable as a first-past in a parser.
1221
1222 It enforces a number of constraints:
1223 * No empty paths.
1224 * No "~" in paths.
1225 * No Unicode codepoints that filesystems might elide when normalizing.
1226 * No relative path components like "." or "..".
1227 * No absolute paths.
1228 * No ".git" or ".repo*" path components.
1229
1230 Args:
1231 path: The path name to validate.
1232 dir_ok: Whether |path| may force a directory (e.g. end in a /).
1233 cwd_dot_ok: Whether |path| may be just ".".
1234
1235 Returns:
1236 None if |path| is OK, a failure message otherwise.
1237 """
1238 if not path:
1239 return 'empty paths not allowed'
1240
Mike Frysinger04122b72019-07-31 23:32:58 -04001241 if '~' in path:
1242 return '~ not allowed (due to 8.3 filenames on Windows filesystems)'
1243
Mike Frysingerf69c7ee2021-04-29 23:15:31 -04001244 path_codepoints = set(path)
1245
Mike Frysinger04122b72019-07-31 23:32:58 -04001246 # Some filesystems (like Apple's HFS+) try to normalize Unicode codepoints
1247 # which means there are alternative names for ".git". Reject paths with
1248 # these in it as there shouldn't be any reasonable need for them here.
1249 # The set of codepoints here was cribbed from jgit's implementation:
1250 # https://eclipse.googlesource.com/jgit/jgit/+/9110037e3e9461ff4dac22fee84ef3694ed57648/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java#884
1251 BAD_CODEPOINTS = {
1252 u'\u200C', # ZERO WIDTH NON-JOINER
1253 u'\u200D', # ZERO WIDTH JOINER
1254 u'\u200E', # LEFT-TO-RIGHT MARK
1255 u'\u200F', # RIGHT-TO-LEFT MARK
1256 u'\u202A', # LEFT-TO-RIGHT EMBEDDING
1257 u'\u202B', # RIGHT-TO-LEFT EMBEDDING
1258 u'\u202C', # POP DIRECTIONAL FORMATTING
1259 u'\u202D', # LEFT-TO-RIGHT OVERRIDE
1260 u'\u202E', # RIGHT-TO-LEFT OVERRIDE
1261 u'\u206A', # INHIBIT SYMMETRIC SWAPPING
1262 u'\u206B', # ACTIVATE SYMMETRIC SWAPPING
1263 u'\u206C', # INHIBIT ARABIC FORM SHAPING
1264 u'\u206D', # ACTIVATE ARABIC FORM SHAPING
1265 u'\u206E', # NATIONAL DIGIT SHAPES
1266 u'\u206F', # NOMINAL DIGIT SHAPES
1267 u'\uFEFF', # ZERO WIDTH NO-BREAK SPACE
1268 }
Mike Frysingerf69c7ee2021-04-29 23:15:31 -04001269 if BAD_CODEPOINTS & path_codepoints:
Mike Frysinger04122b72019-07-31 23:32:58 -04001270 # This message is more expansive than reality, but should be fine.
1271 return 'Unicode combining characters not allowed'
1272
Mike Frysingerf69c7ee2021-04-29 23:15:31 -04001273 # Reject newlines as there shouldn't be any legitmate use for them, they'll
1274 # be confusing to users, and they can easily break tools that expect to be
1275 # able to iterate over newline delimited lists. This even applies to our
1276 # own code like .repo/project.list.
1277 if {'\r', '\n'} & path_codepoints:
1278 return 'Newlines not allowed'
1279
Mike Frysinger04122b72019-07-31 23:32:58 -04001280 # Assume paths might be used on case-insensitive filesystems.
1281 path = path.lower()
1282
Mike Frysingerd9254592020-02-19 22:36:26 -05001283 # Split up the path by its components. We can't use os.path.sep exclusively
1284 # as some platforms (like Windows) will convert / to \ and that bypasses all
1285 # our constructed logic here. Especially since manifest authors only use
1286 # / in their paths.
1287 resep = re.compile(r'[/%s]' % re.escape(os.path.sep))
Mike Frysinger0458faa2021-03-10 23:35:44 -05001288 # Strip off trailing slashes as those only produce '' elements, and we use
1289 # parts to look for individual bad components.
1290 parts = resep.split(path.rstrip('/'))
Mike Frysingerd9254592020-02-19 22:36:26 -05001291
Mike Frysingerae625412020-02-10 17:10:03 -05001292 # Some people use src="." to create stable links to projects. Lets allow
1293 # that but reject all other uses of "." to keep things simple.
Mike Frysingera00c5f42021-02-25 18:26:31 -05001294 if not cwd_dot_ok or parts != ['.']:
Mike Frysingerae625412020-02-10 17:10:03 -05001295 for part in set(parts):
1296 if part in {'.', '..', '.git'} or part.startswith('.repo'):
1297 return 'bad component: %s' % (part,)
Mike Frysinger04122b72019-07-31 23:32:58 -04001298
Mike Frysingera00c5f42021-02-25 18:26:31 -05001299 if not dir_ok and resep.match(path[-1]):
Mike Frysinger04122b72019-07-31 23:32:58 -04001300 return 'dirs not allowed'
1301
Mike Frysingerd9254592020-02-19 22:36:26 -05001302 # NB: The two abspath checks here are to handle platforms with multiple
1303 # filesystem path styles (e.g. Windows).
Mike Frysinger04122b72019-07-31 23:32:58 -04001304 norm = os.path.normpath(path)
Mike Frysingerd9254592020-02-19 22:36:26 -05001305 if (norm == '..' or
1306 (len(norm) >= 3 and norm.startswith('..') and resep.match(norm[0])) or
1307 os.path.isabs(norm) or
1308 norm.startswith('/')):
Mike Frysinger04122b72019-07-31 23:32:58 -04001309 return 'path cannot be outside'
1310
1311 @classmethod
1312 def _ValidateFilePaths(cls, element, src, dest):
1313 """Verify |src| & |dest| are reasonable for <copyfile> & <linkfile>.
1314
1315 We verify the path independent of any filesystem state as we won't have a
1316 checkout available to compare to. i.e. This is for parsing validation
1317 purposes only.
1318
1319 We'll do full/live sanity checking before we do the actual filesystem
1320 modifications in _CopyFile/_LinkFile/etc...
1321 """
1322 # |dest| is the file we write to or symlink we create.
1323 # It is relative to the top of the repo client checkout.
1324 msg = cls._CheckLocalPath(dest)
1325 if msg:
1326 raise ManifestInvalidPathError(
1327 '<%s> invalid "dest": %s: %s' % (element, dest, msg))
1328
1329 # |src| is the file we read from or path we point to for symlinks.
1330 # It is relative to the top of the git project checkout.
Mike Frysingera00c5f42021-02-25 18:26:31 -05001331 is_linkfile = element == 'linkfile'
1332 msg = cls._CheckLocalPath(src, dir_ok=is_linkfile, cwd_dot_ok=is_linkfile)
Mike Frysinger04122b72019-07-31 23:32:58 -04001333 if msg:
1334 raise ManifestInvalidPathError(
1335 '<%s> invalid "src": %s: %s' % (element, src, msg))
1336
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001337 def _ParseCopyFile(self, project, node):
1338 src = self._reqatt(node, 'src')
1339 dest = self._reqatt(node, 'dest')
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001340 if not self.IsMirror:
1341 # src is project relative;
Mike Frysinger04122b72019-07-31 23:32:58 -04001342 # dest is relative to the top of the tree.
1343 # We only validate paths if we actually plan to process them.
1344 self._ValidateFilePaths('copyfile', src, dest)
Mike Frysingere6a202f2019-08-02 15:57:57 -04001345 project.AddCopyFile(src, dest, self.topdir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001346
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001347 def _ParseLinkFile(self, project, node):
1348 src = self._reqatt(node, 'src')
1349 dest = self._reqatt(node, 'dest')
1350 if not self.IsMirror:
1351 # src is project relative;
Mike Frysinger04122b72019-07-31 23:32:58 -04001352 # dest is relative to the top of the tree.
1353 # We only validate paths if we actually plan to process them.
1354 self._ValidateFilePaths('linkfile', src, dest)
Mike Frysingere6a202f2019-08-02 15:57:57 -04001355 project.AddLinkFile(src, dest, self.topdir)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001356
James W. Mills24c13082012-04-12 15:04:13 -05001357 def _ParseAnnotation(self, project, node):
1358 name = self._reqatt(node, 'name')
1359 value = self._reqatt(node, 'value')
1360 try:
1361 keep = self._reqatt(node, 'keep').lower()
1362 except ManifestParseError:
1363 keep = "true"
1364 if keep != "true" and keep != "false":
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301365 raise ManifestParseError('optional "keep" attribute must be '
David Pursehouseabdf7502020-02-12 14:58:39 +09001366 '"true" or "false"')
James W. Mills24c13082012-04-12 15:04:13 -05001367 project.AddAnnotation(name, value, keep)
1368
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001369 def _get_remote(self, node):
1370 name = node.getAttribute('remote')
1371 if not name:
1372 return None
1373
1374 v = self._remotes.get(name)
1375 if not v:
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301376 raise ManifestParseError("remote %s not defined in %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +09001377 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001378 return v
1379
1380 def _reqatt(self, node, attname):
1381 """
1382 reads a required attribute from the node.
1383 """
1384 v = node.getAttribute(attname)
1385 if not v:
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301386 raise ManifestParseError("no %s in <%s> within %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +09001387 (attname, node.nodeName, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001388 return v
Julien Camperguedd654222014-01-09 16:21:37 +01001389
1390 def projectsDiff(self, manifest):
1391 """return the projects differences between two manifests.
1392
1393 The diff will be from self to given manifest.
1394
1395 """
1396 fromProjects = self.paths
1397 toProjects = manifest.paths
1398
Anthony King7446c592014-05-06 09:19:39 +01001399 fromKeys = sorted(fromProjects.keys())
1400 toKeys = sorted(toProjects.keys())
Julien Camperguedd654222014-01-09 16:21:37 +01001401
1402 diff = {'added': [], 'removed': [], 'changed': [], 'unreachable': []}
1403
1404 for proj in fromKeys:
David Pursehouseeeff3532020-02-12 11:24:10 +09001405 if proj not in toKeys:
Julien Camperguedd654222014-01-09 16:21:37 +01001406 diff['removed'].append(fromProjects[proj])
1407 else:
1408 fromProj = fromProjects[proj]
1409 toProj = toProjects[proj]
1410 try:
1411 fromRevId = fromProj.GetCommitRevisionId()
1412 toRevId = toProj.GetCommitRevisionId()
1413 except ManifestInvalidRevisionError:
1414 diff['unreachable'].append((fromProj, toProj))
1415 else:
1416 if fromRevId != toRevId:
1417 diff['changed'].append((fromProj, toProj))
1418 toKeys.remove(proj)
1419
1420 for proj in toKeys:
1421 diff['added'].append(toProjects[proj])
1422
1423 return diff
Simran Basib9a1b732015-08-20 12:19:28 -07001424
1425
1426class GitcManifest(XmlManifest):
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -04001427 """Parser for GitC (git-in-the-cloud) manifests."""
Simran Basib9a1b732015-08-20 12:19:28 -07001428
David Pursehousee5913ae2020-02-12 13:56:59 +09001429 def _ParseProject(self, node, parent=None):
Simran Basib9a1b732015-08-20 12:19:28 -07001430 """Override _ParseProject and add support for GITC specific attributes."""
Mike Frysinger5d9c4972021-02-19 13:34:09 -05001431 return super()._ParseProject(
Simran Basib9a1b732015-08-20 12:19:28 -07001432 node, parent=parent, old_revision=node.getAttribute('old-revision'))
1433
1434 def _output_manifest_project_extras(self, p, e):
1435 """Output GITC Specific Project attributes"""
1436 if p.old_revision:
Stefan Beller66851062016-06-17 16:40:08 -07001437 e.setAttribute('old-revision', str(p.old_revision))
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -04001438
1439
1440class RepoClient(XmlManifest):
1441 """Manages a repo client checkout."""
1442
1443 def __init__(self, repodir, manifest_file=None):
1444 self.isGitcClient = False
1445
1446 if os.path.exists(os.path.join(repodir, LOCAL_MANIFEST_NAME)):
1447 print('error: %s is not supported; put local manifests in `%s` instead' %
1448 (LOCAL_MANIFEST_NAME, os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME)),
1449 file=sys.stderr)
1450 sys.exit(1)
1451
1452 if manifest_file is None:
1453 manifest_file = os.path.join(repodir, MANIFEST_FILE_NAME)
1454 local_manifests = os.path.abspath(os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME))
Mike Frysinger5d9c4972021-02-19 13:34:09 -05001455 super().__init__(repodir, manifest_file, local_manifests)
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -04001456
1457 # TODO: Completely separate manifest logic out of the client.
1458 self.manifest = self
1459
1460
1461class GitcClient(RepoClient, GitcManifest):
1462 """Manages a GitC client checkout."""
1463
1464 def __init__(self, repodir, gitc_client_name):
1465 """Initialize the GitcManifest object."""
1466 self.gitc_client_name = gitc_client_name
1467 self.gitc_client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
1468 gitc_client_name)
1469
Mike Frysinger5d9c4972021-02-19 13:34:09 -05001470 super().__init__(repodir, os.path.join(self.gitc_client_dir, '.manifest'))
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -04001471 self.isGitcClient = True